Skip to content

Open source CapCut alternative.

TypeScript

chore(evals): baseline on the routed pipeline — 100% pass on every bucket

David Le73032ce24562
diff --git a/evals/cut-chat.latest-report.json b/evals/cut-chat.latest-report.json
index 182eadec..91c7c5f3 100644
--- a/evals/cut-chat.latest-report.json
+++ b/evals/cut-chat.latest-report.json
@@ -1,82 +1,82 @@
{
"schema": "cut-chat-eval/v1",
- "generatedAt": "2026-08-04T14:24:58.006Z",
+ "generatedAt": "2026-08-05T00:16:04.051Z",
"base": "http://localhost:3000",
"runsPerCase": 3,
"configs": [
{
"label": "baseline",
- "chatModel": "gemini-3.5-flash",
+ "chatModel": "gemini-3.1-flash-lite | complex\u2192gemini-3.5-flash",
"gateModel": "gemini-3.1-flash-lite",
"buckets": {
"chat": {
"cases": 7,
"passRate": 1,
"gateMs": {
- "p50": 1074,
- "p95": 1316,
- "mean": 834,
+ "p50": 1054,
+ "p95": 1347,
+ "mean": 780,
"n": 7
},
"ttftMs": {
- "p50": 3968,
- "p95": 6025,
- "mean": 4109,
+ "p50": 2427,
+ "p95": 8570,
+ "mean": 3713,
"n": 7
},
"totalMs": {
- "p50": 4702,
- "p95": 6600,
- "mean": 4605,
+ "p50": 3120,
+ "p95": 9027,
+ "mean": 4210,
"n": 7
},
"meanRounds": 1
},
"multi-tool": {
"cases": 4,
- "passRate": 0.83,
+ "passRate": 1,
"gateMs": {
"p50": 0,
- "p95": 1273,
- "mean": 600,
+ "p95": 1079,
+ "mean": 534,
"n": 4
},
"ttftMs": {
- "p50": 16525,
- "p95": 65085,
- "mean": 30342,
+ "p50": 19115,
+ "p95": 41693,
+ "mean": 21126,
"n": 4
},
"totalMs": {
- "p50": 16888,
- "p95": 65717,
- "mean": 30853,
+ "p50": 19820,
+ "p95": 42450,
+ "mean": 21702,
"n": 4
},
- "meanRounds": 8.9
+ "meanRounds": 5.9
},
"single-tool": {
"cases": 15,
"passRate": 1,
"gateMs": {
- "p50": 1142,
- "p95": 1543,
- "mean": 1104,
+ "p50": 1156,
+ "p95": 1556,
+ "mean": 1089,
"n": 15
},
"ttftMs": {
- "p50": 7640,
- "p95": 24161,
- "mean": 8617,
+ "p50": 4573,
+ "p95": 8879,
+ "mean": 5520,
"n": 15
},
"totalMs": {
- "p50": 8019,
- "p95": 24686,
- "mean": 9007,
+ "p50": 5257,
+ "p95": 9300,
+ "mean": 5906,
"n": 15
},
- "meanRounds": 2.2
+ "meanRounds": 2
}
},
"cases": [
@@ -93,21 +93,21 @@
"n": 3
},
"ttftMs": {
- "p50": 4466,
- "p95": 4745,
- "mean": 4523,
+ "p50": 4194,
+ "p95": 7794,
+ "mean": 5330,
"n": 3
},
"totalMs": {
- "p50": 4702,
- "p95": 5112,
- "mean": 4818,
+ "p50": 4528,
+ "p95": 7794,
+ "mean": 5464,
"n": 3
},
"modelMs": {
- "p50": 4702,
- "p95": 5112,
- "mean": 4818,
+ "p50": 4528,
+ "p95": 7794,
+ "mean": 5464,
"n": 3
},
"toolMs": {
@@ -124,22 +124,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4359.609083,
- "modelMs": 4639.969749999999,
+ "ttftMs": 7793.889541,
+ "modelMs": 7794.0265420000005,
"toolMs": 0,
- "totalMs": 4639.969749999999,
+ "totalMs": 7794.0265420000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4639.969749999999,
- "firstDeltaMs": 4359.194458,
+ "roundMs": 7794.0265420000005,
+ "firstDeltaMs": 7793.380333,
"calls": []
}
]
@@ -147,22 +148,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4744.642833,
- "modelMs": 5112.328084,
+ "ttftMs": 4194.385209,
+ "modelMs": 4527.521083000001,
"toolMs": 0,
- "totalMs": 5112.328084,
+ "totalMs": 4527.521083000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5112.328084,
- "firstDeltaMs": 4744.552584,
+ "roundMs": 4527.521083000001,
+ "firstDeltaMs": 4194.338916000001,
"calls": []
}
]
@@ -170,22 +172,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4465.872958999998,
- "modelMs": 4701.575957999999,
+ "ttftMs": 4001.5467499999995,
+ "modelMs": 4070.346458000002,
"toolMs": 0,
- "totalMs": 4701.575957999999,
+ "totalMs": 4070.346458000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4701.575957999999,
- "firstDeltaMs": 4464.514791,
+ "roundMs": 4070.346458000002,
+ "firstDeltaMs": 4001.3762499999993,
"calls": []
}
]
@@ -200,27 +203,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1316,
- "p95": 1960,
- "mean": 1494,
+ "p50": 1066,
+ "p95": 1970,
+ "mean": 1358,
"n": 3
},
"ttftMs": {
- "p50": 3136,
- "p95": 3591,
- "mean": 3257,
+ "p50": 2058,
+ "p95": 5865,
+ "mean": 3325,
"n": 3
},
"totalMs": {
- "p50": 3360,
- "p95": 3759,
- "mean": 3456,
+ "p50": 2318,
+ "p95": 6432,
+ "mean": 3683,
"n": 3
},
"modelMs": {
- "p50": 2043,
- "p95": 2044,
- "mean": 1962,
+ "p50": 1261,
+ "p95": 4462,
+ "mean": 2325,
"n": 3
},
"toolMs": {
@@ -238,21 +241,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1959.7404999999999,
+ "gateMs": 1969.5097920000007,
"gateSkipped": false,
- "ttftMs": 3590.546042,
- "modelMs": 1799.5951249999998,
+ "ttftMs": 5864.654458999998,
+ "modelMs": 4462.196416999999,
"toolMs": 0,
- "totalMs": 3759.3356249999997,
+ "totalMs": 6431.706209,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1799.5951249999998,
- "firstDeltaMs": 1630.70075,
+ "roundMs": 4462.196416999999,
+ "firstDeltaMs": 3893.8072089999987,
"calls": []
}
]
@@ -261,21 +265,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1205.728583,
+ "gateMs": 1038.349166,
"gateSkipped": false,
- "ttftMs": 3043.5315840000003,
- "modelMs": 2043.2952499999992,
+ "ttftMs": 2051.231208000001,
+ "modelMs": 1260.6797499999993,
"toolMs": 0,
- "totalMs": 3249.0238329999993,
+ "totalMs": 2299.0289159999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2043.2952499999992,
- "firstDeltaMs": 1837.6844999999994,
+ "roundMs": 1260.6797499999993,
+ "firstDeltaMs": 1012.7811249999977,
"calls": []
}
]
@@ -284,21 +289,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1315.8584169999995,
+ "gateMs": 1066.0253329999978,
"gateSkipped": false,
- "ttftMs": 3136.0234170000003,
- "modelMs": 2044.2198750000025,
+ "ttftMs": 2058.1978340000023,
+ "modelMs": 1251.575832999999,
"toolMs": 0,
- "totalMs": 3360.078292000002,
+ "totalMs": 2317.6011659999967,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2044.2198750000025,
- "firstDeltaMs": 1820.080917000003,
+ "roundMs": 1251.575832999999,
+ "firstDeltaMs": 991.9768749999967,
"calls": []
}
]
@@ -313,27 +319,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1243,
- "p95": 1646,
- "mean": 1350,
+ "p50": 1060,
+ "p95": 2070,
+ "mean": 1394,
"n": 3
},
"ttftMs": {
- "p50": 2887,
- "p95": 3499,
- "mean": 2972,
+ "p50": 2427,
+ "p95": 3058,
+ "mean": 2630,
"n": 3
},
"totalMs": {
- "p50": 3010,
- "p95": 3828,
- "mean": 3189,
+ "p50": 3120,
+ "p95": 3341,
+ "mean": 3020,
"n": 3
},
"modelMs": {
- "p50": 1768,
- "p95": 2182,
- "mean": 1840,
+ "p50": 1546,
+ "p95": 2060,
+ "mean": 1626,
"n": 3
},
"toolMs": {
@@ -351,21 +357,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1242.6463750000003,
+ "gateMs": 1059.795917000003,
"gateSkipped": false,
- "ttftMs": 2887.0092079999995,
- "modelMs": 1767.6925420000007,
+ "ttftMs": 2426.6803330000002,
+ "modelMs": 2060.026750000001,
"toolMs": 0,
- "totalMs": 3010.338917000001,
+ "totalMs": 3119.822667000004,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1767.6925420000007,
- "firstDeltaMs": 1644.2118750000009,
+ "roundMs": 2060.026750000001,
+ "firstDeltaMs": 1366.6626670000005,
"calls": []
}
]
@@ -374,21 +381,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1159.5506249999999,
+ "gateMs": 2069.903000000002,
"gateSkipped": false,
- "ttftMs": 2529.8635840000024,
- "modelMs": 1570.502042,
+ "ttftMs": 3057.9860420000005,
+ "modelMs": 1271.451000000001,
"toolMs": 0,
- "totalMs": 2730.052667,
+ "totalMs": 3341.354000000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1570.502042,
- "firstDeltaMs": 1370.2050000000017,
+ "roundMs": 1271.451000000001,
+ "firstDeltaMs": 987.9754580000008,
"calls": []
}
]
@@ -397,21 +405,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1646.3154579999973,
+ "gateMs": 1051.824041999993,
"gateSkipped": false,
- "ttftMs": 3498.555457999999,
- "modelMs": 2181.766875000001,
+ "ttftMs": 2404.611334000001,
+ "modelMs": 1546.1673749999973,
"toolMs": 0,
- "totalMs": 3828.0823329999985,
+ "totalMs": 2597.99141699999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2181.766875000001,
- "firstDeltaMs": 1852.1642920000013,
+ "roundMs": 1546.1673749999973,
+ "firstDeltaMs": 1352.4513329999972,
"calls": []
}
]
@@ -426,86 +435,87 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1128,
- "p95": 1960,
- "mean": 1370,
+ "p50": 1056,
+ "p95": 1177,
+ "mean": 1056,
"n": 3
},
"ttftMs": {
- "p50": 13155,
- "p95": 17306,
- "mean": 13249,
+ "p50": 3584,
+ "p95": 3686,
+ "mean": 3607,
"n": 3
},
"totalMs": {
- "p50": 13648,
- "p95": 17812,
- "mean": 13753,
+ "p50": 3909,
+ "p95": 3951,
+ "mean": 3887,
"n": 3
},
"modelMs": {
- "p50": 12519,
- "p95": 15851,
- "mean": 12383,
+ "p50": 2867,
+ "p95": 2895,
+ "mean": 2831,
"n": 3
},
"toolMs": {
- "p50": 1,
+ "p50": 0,
"p95": 1,
- "mean": 1,
+ "mean": 0,
"n": 3
},
"rounds": {
- "mean": 2.7,
- "max": 4
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 7780.846042000005,
- "execMs": 0.6418750000011642
+ "atMs": 2509.3868749999965,
+ "execMs": 0.5657500000015716
},
{
"name": "update_cue",
"round": 0,
- "atMs": 7781.501833000002,
- "execMs": 0.008457999996608123
+ "atMs": 2509.9682089999988,
+ "execMs": 0.00620800000615418
}
],
"timings": {
- "gateMs": 1021.4395409999997,
+ "gateMs": 1177.3011659999975,
"gateSkipped": false,
- "ttftMs": 9287.144792000006,
- "modelMs": 8778.869458000001,
- "toolMs": 0.6503329999977723,
- "totalMs": 9800.959331999999,
+ "ttftMs": 3686.2987090000024,
+ "modelMs": 2731.3125839999993,
+ "toolMs": 0.5719580000077258,
+ "totalMs": 3909.1857080000045,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6758.751916000001,
+ "roundMs": 1331.0242919999946,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.6418750000011642
+ "execMs": 0.5657500000015716
},
{
"name": "update_cue",
- "execMs": 0.008457999996608123
+ "execMs": 0.00620800000615418
}
]
},
{
- "roundMs": 2020.117542,
- "firstDeltaMs": 1505.6138339999961,
+ "roundMs": 1400.2882920000047,
+ "firstDeltaMs": 1176.3052080000052,
"calls": []
}
]
@@ -513,49 +523,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 11591.891791999995,
- "execMs": 0.11795900000288384
+ "atMs": 2519.7390839999935,
+ "execMs": 0.03016599999682512
},
{
"name": "update_cue",
"round": 0,
- "atMs": 11592.036957999997,
- "execMs": 0.011624999999185093
+ "atMs": 2519.7765839999993,
+ "execMs": 0.010000000002037268
}
],
"timings": {
- "gateMs": 1128.4081250000017,
+ "gateMs": 1056.0058750000026,
"gateSkipped": false,
- "ttftMs": 13154.570999999996,
- "modelMs": 12519.247458000005,
- "toolMs": 0.12958400000206893,
- "totalMs": 13647.78516700001,
+ "ttftMs": 3584.456624999999,
+ "modelMs": 2894.6320410000044,
+ "toolMs": 0.04016599999886239,
+ "totalMs": 3950.678082000006,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 10462.905958000003,
+ "roundMs": 1463.4017500000045,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.11795900000288384
+ "execMs": 0.03016599999682512
},
{
"name": "update_cue",
- "execMs": 0.011624999999185093
+ "execMs": 0.010000000002037268
}
]
},
{
- "roundMs": 2056.3415000000023,
- "firstDeltaMs": 1562.4829170000012,
+ "roundMs": 1431.2302909999999,
+ "firstDeltaMs": 1064.6322910000017,
"calls": []
}
]
@@ -563,81 +574,70 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 11419.927625000004,
- "execMs": 0.01366700000653509
+ "atMs": 2458.4587500000052,
+ "execMs": 0.07804100000066683
},
{
- "name": "merge_cue",
+ "name": "update_cue",
"round": 0,
- "atMs": 11419.945082999991,
- "execMs": 0.18620900000678375
+ "atMs": 2458.551833000005,
+ "execMs": 0.010792000000947155
},
{
- "name": "get_state",
- "round": 1,
- "atMs": 13291.628000000004,
- "execMs": 0.6385830000072019
+ "name": "delete_cue",
+ "round": 0,
+ "atMs": 2458.5675830000037,
+ "execMs": 0.18429199999809498
},
{
"name": "update_cue",
- "round": 2,
- "atMs": 15302.830832999993,
- "execMs": 0.025374999997438863
+ "round": 0,
+ "atMs": 2458.755833000003,
+ "execMs": 0.005708000004233327
}
],
"timings": {
- "gateMs": 1960.0664589999942,
+ "gateMs": 934.7073749999981,
"gateSkipped": false,
- "ttftMs": 17306.47091699999,
- "modelMs": 15850.66187399999,
- "toolMs": 0.8638340000179596,
- "totalMs": 17811.592167000003,
+ "ttftMs": 3548.945375000003,
+ "modelMs": 2867.4186249999984,
+ "toolMs": 0.2788330000039423,
+ "totalMs": 3802.4048330000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 9459.606458000002,
+ "roundMs": 1523.1478750000024,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.01366700000653509
+ "execMs": 0.07804100000066683
},
{
- "name": "merge_cue",
- "execMs": 0.18620900000678375
- }
- ]
- },
- {
- "roundMs": 1871.310540999999,
- "firstDeltaMs": null,
- "calls": [
+ "name": "update_cue",
+ "execMs": 0.010792000000947155
+ },
{
- "name": "get_state",
- "execMs": 0.6385830000072019
- }
- ]
- },
- {
- "roundMs": 2010.419041999994,
- "firstDeltaMs": null,
- "calls": [
+ "name": "delete_cue",
+ "execMs": 0.18429199999809498
+ },
{
"name": "update_cue",
- "execMs": 0.025374999997438863
+ "execMs": 0.005708000004233327
}
]
},
{
- "roundMs": 2509.3258329999953,
- "firstDeltaMs": 2003.5387080000073,
+ "roundMs": 1344.270749999996,
+ "firstDeltaMs": 1090.153790999997,
"calls": []
}
]
@@ -652,27 +652,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1059,
- "p95": 1172,
- "mean": 1051,
+ "p50": 932,
+ "p95": 1152,
+ "mean": 995,
"n": 3
},
"ttftMs": {
- "p50": 3034,
- "p95": 3410,
- "mean": 3124,
+ "p50": 2195,
+ "p95": 2289,
+ "mean": 2136,
"n": 3
},
"totalMs": {
- "p50": 3208,
- "p95": 3614,
- "mean": 3320,
+ "p50": 2439,
+ "p95": 2513,
+ "mean": 2380,
"n": 3
},
"modelMs": {
- "p50": 2216,
- "p95": 2441,
- "mean": 2269,
+ "p50": 1289,
+ "p95": 1580,
+ "mean": 1385,
"n": 3
},
"toolMs": {
@@ -689,22 +689,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 921.1869169999991,
+ "gateMs": 932.4662919999973,
"gateSkipped": false,
- "ttftMs": 2926.0624580000003,
- "modelMs": 2216.0313339999993,
+ "ttftMs": 2288.5164579999982,
+ "modelMs": 1580.4839580000043,
"toolMs": 0,
- "totalMs": 3137.2182509999984,
+ "totalMs": 2512.9502500000017,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2216.0313339999993,
- "firstDeltaMs": 2004.5246670000051,
+ "roundMs": 1580.4839580000043,
+ "firstDeltaMs": 1355.7090000000026,
"calls": []
}
]
@@ -712,22 +713,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1059.0913750000036,
+ "gateMs": 1151.6104160000032,
"gateSkipped": false,
- "ttftMs": 3034.106707999992,
- "modelMs": 2149.300500000012,
+ "ttftMs": 2194.624499999998,
+ "modelMs": 1286.9646670000002,
"toolMs": 0,
- "totalMs": 3208.3918750000157,
+ "totalMs": 2438.5750830000034,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2149.300500000012,
- "firstDeltaMs": 1974.9127500000031,
+ "roundMs": 1286.9646670000002,
+ "firstDeltaMs": 1042.804833000002,
"calls": []
}
]
@@ -735,22 +737,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1172.2600419999944,
+ "gateMs": 899.5664579999939,
"gateSkipped": false,
- "ttftMs": 3410.4349169999914,
- "modelMs": 2441.36708299999,
+ "ttftMs": 1925.6463330000042,
+ "modelMs": 1288.8934999999983,
"toolMs": 0,
- "totalMs": 3613.6271249999845,
+ "totalMs": 2188.459957999992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2441.36708299999,
- "firstDeltaMs": 2238.0697910000017,
+ "roundMs": 1288.8934999999983,
+ "firstDeltaMs": 1025.7225420000032,
"calls": []
}
]
@@ -771,96 +774,113 @@
"n": 3
},
"ttftMs": {
- "p50": 16525,
- "p95": 24932,
- "mean": 18107,
+ "p50": 20113,
+ "p95": 31438,
+ "mean": 19807,
"n": 3
},
"totalMs": {
- "p50": 16888,
- "p95": 25321,
- "mean": 18528,
+ "p50": 20630,
+ "p95": 31766,
+ "mean": 20179,
"n": 3
},
"modelMs": {
- "p50": 16888,
- "p95": 25321,
- "mean": 18528,
+ "p50": 20629,
+ "p95": 31766,
+ "mean": 20179,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 0,
+ "p95": 1,
"mean": 0,
"n": 3
},
"rounds": {
- "mean": 5,
- "max": 9
+ "mean": 5.3,
+ "max": 10
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 6393.775084000008,
- "execMs": 0.001041000010445714
+ "atMs": 7084.217583000005,
+ "execMs": 0.014458000005106442
},
{
"name": "add_clip",
"round": 0,
- "atMs": 6393.784083999999,
- "execMs": 0.002082999999402091
+ "atMs": 7084.266457999998,
+ "execMs": 0.02899999999499414
},
{
"name": "get_state",
"round": 1,
- "atMs": 9776.628250000009,
- "execMs": 0.31791700000758283
+ "atMs": 13292.531583000004,
+ "execMs": 0.44070900000224356
+ },
+ {
+ "name": "get_state",
+ "round": 2,
+ "atMs": 16965.996958000003,
+ "execMs": 0.015916000003926456
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 12864.125834000006,
- "modelMs": 13374.06295800001,
- "toolMs": 0.32104100001743063,
- "totalMs": 13374.383999000027,
+ "ttftMs": 20112.57475,
+ "modelMs": 20629.039458999978,
+ "toolMs": 0.5000830000062706,
+ "totalMs": 20629.539541999984,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6393.267000000007,
+ "roundMs": 7083.9570419999945,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.001041000010445714
+ "execMs": 0.014458000005106442
},
{
"name": "add_clip",
- "execMs": 0.002082999999402091
+ "execMs": 0.02899999999499414
+ }
+ ]
+ },
+ {
+ "roundMs": 6208.099875,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.44070900000224356
}
]
},
{
- "roundMs": 3382.747292,
+ "roundMs": 3672.95204199999,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.31791700000758283
+ "execMs": 0.015916000003926456
}
]
},
{
- "roundMs": 3598.0486660000024,
- "firstDeltaMs": 3086.9170000000013,
+ "roundMs": 3664.0304999999935,
+ "firstDeltaMs": 3146.5378749999945,
"calls": []
}
]
@@ -868,65 +888,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 5519.512749999994,
- "execMs": 0.005832999988342635
+ "atMs": 4574.28850000001,
+ "execMs": 0.0036249999975552782
},
{
"name": "add_clip",
"round": 0,
- "atMs": 5519.553417000003,
- "execMs": 0.015291999996406958
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 7457.497124999994,
- "execMs": 0.007083000004058704
+ "atMs": 4574.317334000007,
+ "execMs": 0.0002500000118743628
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 16524.546791999994,
- "modelMs": 16887.89370700001,
- "toolMs": 0.028207999988808297,
- "totalMs": 16887.921915,
+ "ttftMs": 7870.962792000006,
+ "modelMs": 8141.328873999984,
+ "toolMs": 0.003875000009429641,
+ "totalMs": 8141.332748999994,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5519.0919580000045,
+ "roundMs": 4573.873582999993,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.005832999988342635
+ "execMs": 0.0036249999975552782
},
{
"name": "add_clip",
- "execMs": 0.015291999996406958
- }
- ]
- },
- {
- "roundMs": 1937.8609580000048,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.007083000004058704
+ "execMs": 0.0002500000118743628
}
]
},
{
- "roundMs": 9430.940791,
- "firstDeltaMs": 9067.000249999997,
+ "roundMs": 3567.455290999991,
+ "firstDeltaMs": 3296.6022499999963,
"calls": []
}
]
@@ -934,161 +939,178 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 5463.008208999992,
- "execMs": 0.004209000006085262
+ "atMs": 5157.393249999994,
+ "execMs": 0.0019170000014128163
},
{
"name": "add_clip",
"round": 0,
- "atMs": 5463.023958999998,
- "execMs": 0.00016700000560376793
+ "atMs": 5157.403832999989,
+ "execMs": 0.003416999999899417
},
{
"name": "get_state",
"round": 1,
- "atMs": 7691.296083999987,
- "execMs": 0.04091700000572018
+ "atMs": 7497.159791999991,
+ "execMs": 0.03262499999254942
},
{
"name": "add_clip",
"round": 2,
- "atMs": 10309.034999999989,
- "execMs": 0.0010839999886229634
+ "atMs": 12609.253125000003,
+ "execMs": 0.001791999995475635
},
{
"name": "add_clip",
"round": 3,
- "atMs": 11862.874333999993,
- "execMs": 0.0033750000002328306
+ "atMs": 14647.699957999997,
+ "execMs": 0.001416999992216006
},
{
"name": "get_state",
"round": 4,
- "atMs": 13772.919499999989,
- "execMs": 0.015083000005688518
+ "atMs": 16497.609792000003,
+ "execMs": 0.015708000006270595
},
{
"name": "add_clip",
"round": 5,
- "atMs": 16198.954708999998,
- "execMs": 0.001459000020986423
+ "atMs": 18667.497749999995,
+ "execMs": 0.001125000009778887
},
{
- "name": "add_clip",
+ "name": "get_state",
"round": 6,
- "atMs": 18514.299374999988,
- "execMs": 0.0009170000266749412
+ "atMs": 20309.080667000002,
+ "execMs": 0.01129199999559205
},
{
- "name": "get_state",
+ "name": "add_clip",
"round": 7,
- "atMs": 20172.82987499998,
- "execMs": 0.006500000017695129
+ "atMs": 23717.474625000003,
+ "execMs": 0.0013749999925494194
+ },
+ {
+ "name": "set_transition",
+ "round": 8,
+ "atMs": 29087.995958,
+ "execMs": 0.003958999994210899
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 24932.335791999983,
- "modelMs": 25321.250706999985,
- "toolMs": 0.07371100007731002,
- "totalMs": 25321.324418000062,
+ "ttftMs": 31438.012208,
+ "modelMs": 31766.286167999977,
+ "toolMs": 0.07462699997995514,
+ "totalMs": 31766.360794999957,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5462.821874999994,
+ "roundMs": 5157.0201249999955,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.004209000006085262
+ "execMs": 0.0019170000014128163
},
{
"name": "add_clip",
- "execMs": 0.00016700000560376793
+ "execMs": 0.003416999999899417
}
]
},
{
- "roundMs": 2228.1981249999953,
+ "roundMs": 2339.702250000002,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.04091700000572018
+ "execMs": 0.03262499999254942
}
]
},
{
- "roundMs": 2617.6444580000098,
+ "roundMs": 5112.023249999998,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.0010839999886229634
+ "execMs": 0.001791999995475635
}
]
},
{
- "roundMs": 1553.792957999991,
+ "roundMs": 2038.4170000000013,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.0033750000002328306
+ "execMs": 0.001416999992216006
}
]
},
{
- "roundMs": 1909.955667000002,
+ "roundMs": 1849.8805830000056,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.015083000005688518
+ "execMs": 0.015708000006270595
}
]
},
{
- "roundMs": 2425.9573750000127,
+ "roundMs": 2169.840083999996,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.001459000020986423
+ "execMs": 0.001125000009778887
+ }
+ ]
+ },
+ {
+ "roundMs": 1641.5517919999984,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.01129199999559205
}
]
},
{
- "roundMs": 2315.3177499999874,
+ "roundMs": 3408.2985419999895,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.0009170000266749412
+ "execMs": 0.0013749999925494194
}
]
},
{
- "roundMs": 1658.507083000004,
+ "roundMs": 5370.464416999996,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.006500000017695129
+ "name": "set_transition",
+ "execMs": 0.003958999994210899
}
]
},
{
- "roundMs": 5149.055415999988,
- "firstDeltaMs": 4759.478457999998,
+ "roundMs": 2679.0881249999948,
+ "firstDeltaMs": 2349.9932499999995,
"calls": []
}
]
@@ -1109,21 +1131,21 @@
"n": 3
},
"ttftMs": {
- "p50": 3968,
- "p95": 4794,
- "mean": 4191,
+ "p50": 4632,
+ "p95": 7567,
+ "mean": 5389,
"n": 3
},
"totalMs": {
- "p50": 5533,
- "p95": 6019,
- "mean": 5535,
+ "p50": 5569,
+ "p95": 8069,
+ "mean": 6213,
"n": 3
},
"modelMs": {
- "p50": 5533,
- "p95": 6019,
- "mean": 5535,
+ "p50": 5569,
+ "p95": 8069,
+ "mean": 6213,
"n": 3
},
"toolMs": {
@@ -1140,22 +1162,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3811.9108330000017,
- "modelMs": 5054.131958000013,
+ "ttftMs": 3968.5312919999997,
+ "modelMs": 4999.656541999997,
"toolMs": 0,
- "totalMs": 5054.131958000013,
+ "totalMs": 4999.656541999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5054.131958000013,
- "firstDeltaMs": 3811.4652080000087,
+ "roundMs": 4999.656541999997,
+ "firstDeltaMs": 3968.389666999996,
"calls": []
}
]
@@ -1163,22 +1186,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3967.5332500000077,
- "modelMs": 5533.016624999989,
+ "ttftMs": 4631.911708999993,
+ "modelMs": 5569.370624999996,
"toolMs": 0,
- "totalMs": 5533.016624999989,
+ "totalMs": 5569.370624999996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5533.016624999989,
- "firstDeltaMs": 3967.478000000003,
+ "roundMs": 5569.370624999996,
+ "firstDeltaMs": 4631.630459000007,
"calls": []
}
]
@@ -1186,22 +1210,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4793.673792000016,
- "modelMs": 6019.273333000019,
+ "ttftMs": 7566.532707999999,
+ "modelMs": 8068.67279099999,
"toolMs": 0,
- "totalMs": 6019.273333000019,
+ "totalMs": 8068.67279099999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6019.273333000019,
- "firstDeltaMs": 4793.495374999999,
+ "roundMs": 8068.67279099999,
+ "firstDeltaMs": 7566.259749999997,
"calls": []
}
]
@@ -1216,27 +1241,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1149,
- "p95": 1297,
- "mean": 1157,
+ "p50": 1054,
+ "p95": 1244,
+ "mean": 1098,
"n": 3
},
"ttftMs": {
- "p50": 5245,
- "p95": 8261,
- "mean": 6049,
+ "p50": 1913,
+ "p95": 4718,
+ "mean": 2847,
"n": 3
},
"totalMs": {
- "p50": 5820,
- "p95": 8671,
- "mean": 6576,
+ "p50": 2469,
+ "p95": 5167,
+ "mean": 3346,
"n": 3
},
"modelMs": {
- "p50": 4796,
- "p95": 7522,
- "mean": 5420,
+ "p50": 1473,
+ "p95": 3923,
+ "mean": 2248,
"n": 3
},
"toolMs": {
@@ -1253,22 +1278,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1296.504333000019,
+ "gateMs": 1244.1021670000046,
"gateSkipped": false,
- "ttftMs": 4640.529625000025,
- "modelMs": 3940.535999999993,
+ "ttftMs": 4717.7975840000145,
+ "modelMs": 3922.6515409999993,
"toolMs": 0,
- "totalMs": 5237.040333000012,
+ "totalMs": 5166.753708000004,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3940.535999999993,
- "firstDeltaMs": 3343.6559169999964,
+ "roundMs": 3922.6515409999993,
+ "firstDeltaMs": 3473.4761249999865,
"calls": []
}
]
@@ -1276,22 +1302,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1024.778541000007,
+ "gateMs": 1053.6942080000008,
"gateSkipped": false,
- "ttftMs": 5245.285042000003,
- "modelMs": 4795.638958999974,
+ "ttftMs": 1912.8308340000222,
+ "modelMs": 1348.214082999999,
"toolMs": 0,
- "totalMs": 5820.417499999981,
+ "totalMs": 2401.9082909999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4795.638958999974,
- "firstDeltaMs": 4220.248208999983,
+ "roundMs": 1348.214082999999,
+ "firstDeltaMs": 859.0292499999923,
"calls": []
}
]
@@ -1299,22 +1326,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1148.797917000018,
+ "gateMs": 996.209124999994,
"gateSkipped": false,
- "ttftMs": 8260.782624999993,
- "modelMs": 7522.421875,
+ "ttftMs": 1911.5884169999918,
+ "modelMs": 1473.1731250000012,
"toolMs": 0,
- "totalMs": 8671.219792000018,
+ "totalMs": 2469.382249999995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7522.421875,
- "firstDeltaMs": 7111.815207999985,
+ "roundMs": 1473.1731250000012,
+ "firstDeltaMs": 915.1882089999854,
"calls": []
}
]
@@ -1329,27 +1357,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1148,
- "p95": 1268,
- "mean": 1077,
+ "p50": 1215,
+ "p95": 2504,
+ "mean": 1641,
"n": 3
},
"ttftMs": {
- "p50": 7981,
- "p95": 8081,
- "mean": 7874,
+ "p50": 4948,
+ "p95": 4966,
+ "mean": 4639,
"n": 3
},
"totalMs": {
- "p50": 8409,
- "p95": 8514,
- "mean": 8351,
+ "p50": 5257,
+ "p95": 5329,
+ "mean": 4992,
"n": 3
},
"modelMs": {
- "p50": 7316,
- "p95": 7367,
- "mean": 7274,
+ "p50": 3174,
+ "p95": 4054,
+ "mean": 3351,
"n": 3
},
"toolMs": {
@@ -1366,39 +1394,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 6108.19883400001,
- "execMs": 0.04412499998579733
+ "atMs": 2798.792084000015,
+ "execMs": 0.003208000009180978
}
],
"timings": {
- "gateMs": 1147.740374999994,
+ "gateMs": 1215.193333000003,
"gateSkipped": false,
- "ttftMs": 7980.638834000012,
- "modelMs": 7366.637457999954,
- "toolMs": 0.04412499998579733,
- "totalMs": 8514.421957999934,
+ "ttftMs": 4003.6869590000133,
+ "modelMs": 3173.979498999979,
+ "toolMs": 0.003208000009180978,
+ "totalMs": 4389.176039999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4960.037166999973,
+ "roundMs": 1583.2567080000008,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.04412499998579733
+ "execMs": 0.003208000009180978
}
]
},
{
- "roundMs": 2406.6002909999806,
- "firstDeltaMs": 1872.361790999974,
+ "roundMs": 1590.7227909999783,
+ "firstDeltaMs": 1204.8768749999872,
"calls": []
}
]
@@ -1406,39 +1435,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 6506.390333999996,
- "execMs": 0.010666999995009974
+ "atMs": 3669.561916000006,
+ "execMs": 0.025957999983802438
}
],
"timings": {
- "gateMs": 1268.361957999994,
+ "gateMs": 1203.2232920000097,
"gateSkipped": false,
- "ttftMs": 8080.815042000002,
- "modelMs": 7140.563958999992,
- "toolMs": 0.010666999995009974,
- "totalMs": 8408.936583999981,
+ "ttftMs": 4965.603582999989,
+ "modelMs": 4054.1048759999976,
+ "toolMs": 0.025957999983802438,
+ "totalMs": 5257.354125999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5237.909709,
+ "roundMs": 2466.0621669999964,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.010666999995009974
+ "execMs": 0.025957999983802438
}
]
},
{
- "roundMs": 1902.6542499999923,
- "firstDeltaMs": 1574.384625000006,
+ "roundMs": 1588.0427090000012,
+ "firstDeltaMs": 1295.946666999982,
"calls": []
}
]
@@ -1446,39 +1476,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 5983.193708000006,
- "execMs": 0.010459000011906028
+ "atMs": 3831.0110000000277,
+ "execMs": 0.009332999994512647
}
],
"timings": {
- "gateMs": 814.9560829999973,
+ "gateMs": 2504.232250000001,
"gateSkipped": false,
- "ttftMs": 7561.60504200001,
- "modelMs": 7315.718292000005,
- "toolMs": 0.010459000011906028,
- "totalMs": 8130.684834000014,
+ "ttftMs": 4947.781083000009,
+ "modelMs": 2824.8459999999905,
+ "toolMs": 0.009332999994512647,
+ "totalMs": 5329.087582999986,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5168.1398749999935,
+ "roundMs": 1326.559917000006,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.010459000011906028
+ "execMs": 0.009332999994512647
}
]
},
{
- "roundMs": 2147.5784170000115,
- "firstDeltaMs": 1578.370041999995,
+ "roundMs": 1498.2860829999845,
+ "firstDeltaMs": 1116.7051659999997,
"calls": []
}
]
@@ -1493,27 +1524,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1073,
- "p95": 1894,
- "mean": 1342,
+ "p50": 1132,
+ "p95": 1839,
+ "mean": 1302,
"n": 3
},
"ttftMs": {
- "p50": 8585,
- "p95": 9329,
- "mean": 8742,
+ "p50": 8681,
+ "p95": 9903,
+ "mean": 7350,
"n": 3
},
"totalMs": {
- "p50": 9055,
- "p95": 10031,
- "mean": 9262,
+ "p50": 9300,
+ "p95": 10343,
+ "mean": 7797,
"n": 3
},
"modelMs": {
- "p50": 7995,
- "p95": 8137,
- "mean": 7920,
+ "p50": 8168,
+ "p95": 8503,
+ "mean": 6495,
"n": 3
},
"toolMs": {
@@ -1530,39 +1561,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 6451.531749999995,
- "execMs": 0.21308300000964664
+ "atMs": 2240.694125000009,
+ "execMs": 0.11583300001802854
}
],
"timings": {
- "gateMs": 1072.8555409999972,
+ "gateMs": 934.6835000000137,
"gateSkipped": false,
- "ttftMs": 8312.538124999992,
- "modelMs": 7627.367208000011,
- "toolMs": 0.21308300000964664,
- "totalMs": 8700.435832000017,
+ "ttftMs": 3466.508791,
+ "modelMs": 2814.417501000018,
+ "toolMs": 0.11583300001802854,
+ "totalMs": 3749.21683400005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5378.34483300001,
+ "roundMs": 1305.627959000005,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.21308300000964664
+ "execMs": 0.11583300001802854
}
]
},
{
- "roundMs": 2249.0223750000005,
- "firstDeltaMs": 1860.7366669999901,
+ "roundMs": 1508.7895420000132,
+ "firstDeltaMs": 1225.668209000025,
"calls": []
}
]
@@ -1570,39 +1602,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 7167.555582999979,
- "execMs": 0.009707999997772276
+ "atMs": 6301.520542000013,
+ "execMs": 0.013875000004190952
}
],
"timings": {
- "gateMs": 1059.5804579999822,
+ "gateMs": 1131.8551670000015,
"gateSkipped": false,
- "ttftMs": 8585.167666999972,
- "modelMs": 7995.094417000015,
- "toolMs": 0.009707999997772276,
- "totalMs": 9054.684582999995,
+ "ttftMs": 8680.913541999995,
+ "modelMs": 8168.447208999976,
+ "toolMs": 0.013875000004190952,
+ "totalMs": 9300.316250999982,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6107.616250000021,
+ "roundMs": 5169.3279589999875,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.009707999997772276
+ "execMs": 0.013875000004190952
}
]
},
{
- "roundMs": 1887.4781669999938,
- "firstDeltaMs": 1417.5500420000171,
+ "roundMs": 2999.119249999989,
+ "firstDeltaMs": 2379.3384169999918,
"calls": []
}
]
@@ -1610,39 +1643,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 7622.366624999995,
- "execMs": 0.009749999997438863
+ "atMs": 7861.739708000008,
+ "execMs": 0.0232079999987036
}
],
"timings": {
- "gateMs": 1893.688541999989,
+ "gateMs": 1839.3672080000106,
"gateSkipped": false,
- "ttftMs": 9329.472083,
- "modelMs": 8137.384124999982,
- "toolMs": 0.009749999997438863,
- "totalMs": 10031.082416999969,
+ "ttftMs": 9902.65058300001,
+ "modelMs": 8503.224123999971,
+ "toolMs": 0.0232079999987036,
+ "totalMs": 10342.61453999998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5728.396624999994,
+ "roundMs": 6022.078165999992,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.009749999997438863
+ "execMs": 0.0232079999987036
}
]
},
{
- "roundMs": 2408.9874999999884,
- "firstDeltaMs": 1707.069583000004,
+ "roundMs": 2481.145957999979,
+ "firstDeltaMs": 2040.8328329999931,
"calls": []
}
]
@@ -1657,27 +1691,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1142,
- "p95": 1750,
- "mean": 1319,
+ "p50": 1156,
+ "p95": 1942,
+ "mean": 1382,
"n": 3
},
"ttftMs": {
- "p50": 7624,
- "p95": 9146,
- "mean": 7434,
+ "p50": 3825,
+ "p95": 10269,
+ "mean": 5959,
"n": 3
},
"totalMs": {
- "p50": 7862,
- "p95": 9362,
- "mean": 7763,
+ "p50": 4092,
+ "p95": 10387,
+ "mean": 6160,
"n": 3
},
"modelMs": {
- "p50": 6112,
- "p95": 8219,
- "mean": 6444,
+ "p50": 3045,
+ "p95": 8445,
+ "mean": 4778,
"n": 3
},
"toolMs": {
@@ -1694,39 +1728,60 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
- "name": "set_track_muted",
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 2654.5346659999923,
+ "execMs": 0.17337499998393469
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 2654.7189999999828,
+ "execMs": 0.0019169999868609011
+ },
+ {
+ "name": "set_clip_muted",
"round": 0,
- "atMs": 7134.758042000001,
- "execMs": 0.25383400000282563
+ "atMs": 2654.7270410000056,
+ "execMs": 0.0005829999863635749
}
],
"timings": {
- "gateMs": 1142.4588750000112,
+ "gateMs": 1047.563374999998,
"gateSkipped": false,
- "ttftMs": 9145.853292000014,
- "modelMs": 8219.014500999998,
- "toolMs": 0.25383400000282563,
- "totalMs": 9361.727210000012,
+ "ttftMs": 3824.9181249999965,
+ "modelMs": 3044.6330420000013,
+ "toolMs": 0.17587499995715916,
+ "totalMs": 4092.3722919999564,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5991.719666999998,
+ "roundMs": 1604.7194590000145,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_track_muted",
- "execMs": 0.25383400000282563
+ "name": "set_clip_muted",
+ "execMs": 0.17337499998393469
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0019169999868609011
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0005829999863635749
}
]
},
{
- "roundMs": 2227.2948340000003,
- "firstDeltaMs": 2010.8004170000204,
+ "roundMs": 1439.9135829999868,
+ "firstDeltaMs": 1170.1677909999853,
"calls": []
}
]
@@ -1734,59 +1789,60 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 5840.146375000011,
- "execMs": 0.04145799999241717
+ "atMs": 2372.383124999993,
+ "execMs": 0.024250000016763806
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 5840.211624999996,
- "execMs": 0.001875000016298145
+ "atMs": 2372.4187500000116,
+ "execMs": 0.0014999999839346856
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 5840.220834000007,
- "execMs": 0.0005000000237487257
+ "atMs": 2372.4214999999967,
+ "execMs": 0.0012500000011641532
}
],
"timings": {
- "gateMs": 1749.7133750000212,
+ "gateMs": 1155.6971670000057,
"gateSkipped": false,
- "ttftMs": 7624.221042000019,
- "modelMs": 6112.404208000022,
- "toolMs": 0.04383300003246404,
- "totalMs": 7862.161416000075,
+ "ttftMs": 3784.795084000012,
+ "modelMs": 2844.2671240000054,
+ "toolMs": 0.027000000001862645,
+ "totalMs": 3999.991291000013,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4090.0643330000166,
+ "roundMs": 1216.215333,
"firstDeltaMs": null,
"calls": [
{
"name": "set_clip_muted",
- "execMs": 0.04145799999241717
+ "execMs": 0.024250000016763806
},
{
"name": "set_clip_muted",
- "execMs": 0.001875000016298145
+ "execMs": 0.0014999999839346856
},
{
"name": "set_clip_muted",
- "execMs": 0.0005000000237487257
+ "execMs": 0.0012500000011641532
}
]
},
{
- "roundMs": 2022.3398750000051,
- "firstDeltaMs": 1783.8961660000205,
+ "roundMs": 1628.0517910000053,
+ "firstDeltaMs": 1412.339540999994,
"calls": []
}
]
@@ -1794,39 +1850,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_track_muted",
"round": 0,
- "atMs": 3947.5991670000076,
- "execMs": 0.023792000021785498
+ "atMs": 8037.547374999995,
+ "execMs": 0.01900000000023283
}
],
"timings": {
- "gateMs": 1064.6930830000201,
+ "gateMs": 1942.0038329999952,
"gateSkipped": false,
- "ttftMs": 5531.234874999995,
- "modelMs": 5001.5371249999735,
- "toolMs": 0.023792000021785498,
- "totalMs": 6066.254000000015,
+ "ttftMs": 10268.727583,
+ "modelMs": 8445.240665999998,
+ "toolMs": 0.01900000000023283,
+ "totalMs": 10387.263498999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2882.4801249999728,
+ "roundMs": 6095.147832999995,
"firstDeltaMs": null,
"calls": [
{
"name": "set_track_muted",
- "execMs": 0.023792000021785498
+ "execMs": 0.01900000000023283
}
]
},
{
- "roundMs": 2119.0570000000007,
- "firstDeltaMs": 1583.548375000013,
+ "roundMs": 2350.0928330000024,
+ "firstDeltaMs": 2231.136208000011,
"calls": []
}
]
@@ -1841,27 +1898,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1136,
- "p95": 4933,
- "mean": 2369,
+ "p50": 1170,
+ "p95": 1171,
+ "mean": 1113,
"n": 3
},
"ttftMs": {
- "p50": 7941,
- "p95": 15425,
- "mean": 9923,
+ "p50": 7070,
+ "p95": 7375,
+ "mean": 6196,
"n": 3
},
"totalMs": {
- "p50": 8407,
- "p95": 15711,
- "mean": 10315,
+ "p50": 7262,
+ "p95": 7588,
+ "mean": 6390,
"n": 3
},
"modelMs": {
- "p50": 7271,
- "p95": 10778,
- "mean": 7946,
+ "p50": 6091,
+ "p95": 6418,
+ "mean": 5277,
"n": 3
},
"toolMs": {
@@ -1871,46 +1928,47 @@
"n": 3
},
"rounds": {
- "mean": 2.3,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_image",
"round": 0,
- "atMs": 4939.554417000007,
- "execMs": 0.1386249999923166
+ "atMs": 5858.840958999994,
+ "execMs": 0.10562499999650754
}
],
"timings": {
- "gateMs": 1038.3640829999931,
+ "gateMs": 1170.7471249999944,
"gateSkipped": false,
- "ttftMs": 6403.125083999999,
- "modelMs": 5788.804416999978,
- "toolMs": 0.1386249999923166,
- "totalMs": 6827.307124999963,
+ "ttftMs": 7070.281209000008,
+ "modelMs": 6091.013457999972,
+ "toolMs": 0.10562499999650754,
+ "totalMs": 7261.866207999963,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3900.551124999998,
+ "roundMs": 4687.624916999979,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.1386249999923166
+ "execMs": 0.10562499999650754
}
]
},
{
- "roundMs": 1888.2532919999794,
- "firstDeltaMs": 1463.4157499999856,
+ "roundMs": 1403.388540999993,
+ "firstDeltaMs": 1211.319416000013,
"calls": []
}
]
@@ -1918,55 +1976,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
- {
- "name": "watch_video",
- "round": 0,
- "atMs": 10371.176875000005,
- "execMs": 0.015000000013969839
- },
{
"name": "generate_image",
- "round": 1,
- "atMs": 13370.286458000017,
- "execMs": 0.039416000014171004
+ "round": 0,
+ "atMs": 3106.2200840000005,
+ "execMs": 0.011042000027373433
}
],
"timings": {
- "gateMs": 4933.261333000031,
+ "gateMs": 997.6964999999909,
"gateSkipped": false,
- "ttftMs": 15424.643374999985,
- "modelMs": 10777.872375000035,
- "toolMs": 0.05441600002814084,
- "totalMs": 15711.188124000095,
+ "ttftMs": 4142.683374999993,
+ "modelMs": 3320.870457000012,
+ "toolMs": 0.011042000027373433,
+ "totalMs": 4318.57799900003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5437.577916999988,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.015000000013969839
- }
- ]
- },
- {
- "roundMs": 2999.0269580000313,
+ "roundMs": 2108.4222909999953,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.039416000014171004
+ "execMs": 0.011042000027373433
}
]
},
{
- "roundMs": 2341.2675000000163,
- "firstDeltaMs": 2054.278875000018,
+ "roundMs": 1212.4481660000165,
+ "firstDeltaMs": 1036.439041000005,
"calls": []
}
]
@@ -1974,39 +2017,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_image",
"round": 0,
- "atMs": 6519.279957999999,
- "execMs": 0.018709000025410205
+ "atMs": 5580.244208000018,
+ "execMs": 0.011375000001862645
}
],
"timings": {
- "gateMs": 1136.0272910000058,
+ "gateMs": 1170.364750000008,
"gateSkipped": false,
- "ttftMs": 7940.869041999977,
- "modelMs": 7270.545667000057,
- "toolMs": 0.018709000025410205,
- "totalMs": 8406.591667000088,
+ "ttftMs": 7374.501666000026,
+ "modelMs": 6417.717375000007,
+ "toolMs": 0.011375000001862645,
+ "totalMs": 7588.093500000017,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5383.059125000029,
+ "roundMs": 4409.635833000008,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.018709000025410205
+ "execMs": 0.011375000001862645
}
]
},
{
- "roundMs": 1887.4865420000278,
- "firstDeltaMs": 1421.5110420000274,
+ "roundMs": 2008.081542,
+ "firstDeltaMs": 1794.230459000013,
"calls": []
}
]
@@ -2021,27 +2065,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1543,
- "p95": 1839,
- "mean": 1521,
+ "p50": 1202,
+ "p95": 1548,
+ "mean": 1262,
"n": 3
},
"ttftMs": {
- "p50": 7640,
- "p95": 10361,
- "mean": 8344,
+ "p50": 4813,
+ "p95": 4996,
+ "mean": 4489,
"n": 3
},
"totalMs": {
- "p50": 8019,
- "p95": 10751,
- "mean": 8699,
+ "p50": 5437,
+ "p95": 5609,
+ "mean": 4986,
"n": 3
},
"modelMs": {
- "p50": 6477,
- "p95": 9569,
- "mean": 7178,
+ "p50": 3889,
+ "p95": 4572,
+ "mean": 3723,
"n": 3
},
"toolMs": {
@@ -2058,39 +2102,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 5219.905582999985,
- "execMs": 0.02720799995586276
+ "atMs": 3317.751458999992,
+ "execMs": 0.01591699998243712
}
],
"timings": {
- "gateMs": 1838.833290999988,
+ "gateMs": 1548.1645000000135,
"gateSkipped": false,
- "ttftMs": 7032.250416000024,
- "modelMs": 5488.268874000001,
- "toolMs": 0.02720799995586276,
- "totalMs": 7327.129372999945,
+ "ttftMs": 4813.314041999984,
+ "modelMs": 3888.6234159999876,
+ "toolMs": 0.01591699998243712,
+ "totalMs": 5436.803832999984,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3380.7117909999797,
+ "roundMs": 1769.1314999999886,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.02720799995586276
+ "execMs": 0.01591699998243712
}
]
},
{
- "roundMs": 2107.5570830000215,
- "firstDeltaMs": 1812.2983329999843,
+ "roundMs": 2119.491915999999,
+ "firstDeltaMs": 1495.5120830000087,
"calls": []
}
]
@@ -2098,39 +2143,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 8432.478166000044,
- "execMs": 0.021500000031664968
+ "atMs": 2517.0193749999744,
+ "execMs": 0.00483300001360476
}
],
"timings": {
- "gateMs": 1181.716208000027,
+ "gateMs": 1202.1698750000214,
"gateSkipped": false,
- "ttftMs": 10360.772625000041,
- "modelMs": 9568.843999999983,
- "toolMs": 0.021500000031664968,
- "totalMs": 10750.581708000042,
+ "ttftMs": 3658.505083999975,
+ "modelMs": 2708.255625000049,
+ "toolMs": 0.00483300001360476,
+ "totalMs": 3910.430333000084,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7250.637125000008,
+ "roundMs": 1314.7808330000262,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.021500000031664968
+ "execMs": 0.00483300001360476
}
]
},
{
- "roundMs": 2318.2068749999744,
- "firstDeltaMs": 1928.2345830000122,
+ "roundMs": 1393.4747920000227,
+ "firstDeltaMs": 1141.4635000000126,
"calls": []
}
]
@@ -2138,39 +2184,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 5584.626875000016,
- "execMs": 0.017792000027839094
+ "atMs": 3213.5513330000103,
+ "execMs": 0.01187499999650754
}
],
"timings": {
- "gateMs": 1542.525874999992,
+ "gateMs": 1037.066541999986,
"gateSkipped": false,
- "ttftMs": 7639.901375000016,
- "modelMs": 6476.692500000005,
- "toolMs": 0.017792000027839094,
- "totalMs": 8019.236167000025,
+ "ttftMs": 4995.602207999997,
+ "modelMs": 4572.304958000022,
+ "toolMs": 0.01187499999650754,
+ "totalMs": 5609.383375000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4041.9540830000187,
+ "roundMs": 2176.307208000013,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.017792000027839094
+ "execMs": 0.01187499999650754
}
]
},
{
- "roundMs": 2434.738416999986,
- "firstDeltaMs": 2055.2293749999953,
+ "roundMs": 2395.9977500000095,
+ "firstDeltaMs": 1782.0001670000202,
"calls": []
}
]
@@ -2185,27 +2232,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1103,
- "p95": 1811,
- "mean": 1325,
+ "p50": 1067,
+ "p95": 1443,
+ "mean": 1191,
"n": 3
},
"ttftMs": {
- "p50": 7076,
- "p95": 7586,
- "mean": 7196,
+ "p50": 3925,
+ "p95": 4723,
+ "mean": 4123,
"n": 3
},
"totalMs": {
- "p50": 7481,
- "p95": 8069,
- "mean": 7623,
+ "p50": 4219,
+ "p95": 4989,
+ "mean": 4403,
"n": 3
},
"modelMs": {
- "p50": 6378,
- "p95": 7009,
- "mean": 6298,
+ "p50": 3152,
+ "p95": 3546,
+ "mean": 3212,
"n": 3
},
"toolMs": {
@@ -2222,39 +2269,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 5418.593874999962,
- "execMs": 0.13454200001433492
+ "atMs": 2562.8527919999906,
+ "execMs": 0.23016600002301857
}
],
"timings": {
- "gateMs": 1810.9544170000008,
+ "gateMs": 1063.0403339999903,
"gateSkipped": false,
- "ttftMs": 6924.253749999974,
- "modelMs": 5507.017416000017,
- "toolMs": 0.13454200001433492,
- "totalMs": 7318.106375000032,
+ "ttftMs": 3720.4709999999905,
+ "modelMs": 2937.3021249999874,
+ "toolMs": 0.23016600002301857,
+ "totalMs": 4000.5726250000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3607.2092500000144,
+ "roundMs": 1499.3732080000045,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.13454200001433492
+ "execMs": 0.23016600002301857
}
]
},
{
- "roundMs": 1899.8081660000025,
- "firstDeltaMs": 1505.5104580000043,
+ "roundMs": 1437.928916999983,
+ "firstDeltaMs": 1157.3436669999792,
"calls": []
}
]
@@ -2262,39 +2310,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 5122.811624999973,
- "execMs": 0.011624999984633178
+ "atMs": 3412.1935419999936,
+ "execMs": 0.0318340000230819
}
],
"timings": {
- "gateMs": 1102.9226250000065,
+ "gateMs": 1442.9397500000196,
"gateSkipped": false,
- "ttftMs": 7076.451582999958,
- "modelMs": 6377.800665999937,
- "toolMs": 0.011624999984633178,
- "totalMs": 7480.7349159999285,
+ "ttftMs": 4722.995166999986,
+ "modelMs": 3546.428749999992,
+ "toolMs": 0.0318340000230819,
+ "totalMs": 4989.4003340000345,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4019.7760409999755,
+ "roundMs": 1969.08924999999,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.011624999984633178
+ "execMs": 0.0318340000230819
}
]
},
{
- "roundMs": 2358.024624999962,
- "firstDeltaMs": 1953.5955829999875,
+ "roundMs": 1577.3395000000019,
+ "firstDeltaMs": 1310.7056250000023,
"calls": []
}
]
@@ -2302,39 +2351,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 6010.5027080000145,
- "execMs": 0.009874999988824129
+ "atMs": 2769.0024580000027,
+ "execMs": 0.012916000006953254
}
],
"timings": {
- "gateMs": 1060.1811670000316,
+ "gateMs": 1066.6668340000033,
"gateSkipped": false,
- "ttftMs": 7586.211624999996,
- "modelMs": 7009.091958999983,
- "toolMs": 0.009874999988824129,
- "totalMs": 8069.283001000003,
+ "ttftMs": 3925.0844170000055,
+ "modelMs": 3152.4496250000084,
+ "toolMs": 0.012916000006953254,
+ "totalMs": 4219.129375000019,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4950.139916999964,
+ "roundMs": 1702.1625000000058,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.009874999988824129
+ "execMs": 0.012916000006953254
}
]
},
{
- "roundMs": 2058.952042000019,
- "firstDeltaMs": 1575.6807920000283,
+ "roundMs": 1450.2871250000026,
+ "firstDeltaMs": 1156.0587079999968,
"calls": []
}
]
@@ -2349,27 +2399,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1227,
- "p95": 1300,
- "mean": 1195,
+ "p50": 1278,
+ "p95": 1299,
+ "mean": 1228,
"n": 3
},
"ttftMs": {
- "p50": 6126,
- "p95": 6521,
- "mean": 6146,
+ "p50": 4143,
+ "p95": 4654,
+ "mean": 4100,
"n": 3
},
"totalMs": {
- "p50": 6494,
- "p95": 6809,
- "mean": 6474,
+ "p50": 4391,
+ "p95": 5427,
+ "mean": 4533,
"n": 3
},
"modelMs": {
- "p50": 5267,
- "p95": 5508,
- "mean": 5279,
+ "p50": 3092,
+ "p95": 4150,
+ "mean": 3304,
"n": 3
},
"toolMs": {
@@ -2386,39 +2436,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 4717.80124999996,
- "execMs": 0.008042000001296401
+ "atMs": 2424.817708000017,
+ "execMs": 0.004165999998804182
}
],
"timings": {
- "gateMs": 1300.4646249999641,
+ "gateMs": 1107.7597499999974,
"gateSkipped": false,
- "ttftMs": 6521.297250000003,
- "modelMs": 5508.392541999929,
- "toolMs": 0.008042000001296401,
- "totalMs": 6808.865208999894,
+ "ttftMs": 3504.153750000027,
+ "modelMs": 2671.6176250000135,
+ "toolMs": 0.004165999998804182,
+ "totalMs": 3779.3815410000097,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3417.126707999967,
+ "roundMs": 1316.8862500000105,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.008042000001296401
+ "execMs": 0.004165999998804182
}
]
},
{
- "roundMs": 2091.2658339999616,
- "firstDeltaMs": 1803.478333999985,
+ "roundMs": 1354.731375000003,
+ "firstDeltaMs": 1079.3208330000052,
"calls": []
}
]
@@ -2426,39 +2477,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 4333.488624999998,
- "execMs": 0.003832999966107309
+ "atMs": 3258.9722500000207,
+ "execMs": 0.007542000006651506
}
],
"timings": {
- "gateMs": 1057.1188749999856,
+ "gateMs": 1299.2805830000143,
"gateSkipped": false,
- "ttftMs": 5792.1630000000005,
- "modelMs": 5060.955957999919,
- "toolMs": 0.003832999966107309,
- "totalMs": 6118.07866599987,
+ "ttftMs": 4143.358042000007,
+ "modelMs": 3091.8085419999843,
+ "toolMs": 0.007542000006651506,
+ "totalMs": 4391.096667000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3276.253999999957,
+ "roundMs": 1959.5318329999864,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.003832999966107309
+ "execMs": 0.007542000006651506
}
]
},
{
- "roundMs": 1784.7019579999615,
- "firstDeltaMs": 1458.643374999985,
+ "roundMs": 1132.276708999998,
+ "firstDeltaMs": 884.3690420000057,
"calls": []
}
]
@@ -2466,39 +2518,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 3940.6454169999924,
- "execMs": 0.003041999996639788
+ "atMs": 3341.0136659999844,
+ "execMs": 0.0037500000034924597
}
],
"timings": {
- "gateMs": 1227.4472500000265,
+ "gateMs": 1277.602165999997,
"gateSkipped": false,
- "ttftMs": 6125.757708000019,
- "modelMs": 5266.778290999995,
- "toolMs": 0.003041999996639788,
- "totalMs": 6494.228583000018,
+ "ttftMs": 4653.908790999994,
+ "modelMs": 4149.747874999972,
+ "toolMs": 0.0037500000034924597,
+ "totalMs": 5427.353790999972,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2713.1101250000065,
+ "roundMs": 2063.0629999999946,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.003041999996639788
+ "execMs": 0.0037500000034924597
}
]
},
{
- "roundMs": 2553.6681659999886,
- "firstDeltaMs": 2185.09262499999,
+ "roundMs": 2086.684874999977,
+ "firstDeltaMs": 1312.8543749999953,
"calls": []
}
]
@@ -2513,27 +2566,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1284,
- "p95": 1305,
- "mean": 1212,
+ "p50": 1083,
+ "p95": 2019,
+ "mean": 1354,
"n": 3
},
"ttftMs": {
- "p50": 7607,
- "p95": 13046,
- "mean": 8987,
+ "p50": 4544,
+ "p95": 6804,
+ "mean": 5234,
"n": 3
},
"totalMs": {
- "p50": 7640,
- "p95": 13063,
- "mean": 9098,
+ "p50": 4762,
+ "p95": 7398,
+ "mean": 5574,
"n": 3
},
"modelMs": {
- "p50": 6593,
- "p95": 11759,
- "mean": 7886,
+ "p50": 3679,
+ "p95": 6438,
+ "mean": 4219,
"n": 3
},
"toolMs": {
@@ -2550,39 +2603,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 10717.367666999984,
- "execMs": 0.30683300003875047
+ "atMs": 5210.1022500000545,
+ "execMs": 0.08199999999487773
}
],
"timings": {
- "gateMs": 1304.5941250000033,
+ "gateMs": 960.6567910000449,
"gateSkipped": false,
- "ttftMs": 13045.894583999994,
- "modelMs": 11758.523332999961,
- "toolMs": 0.30683300003875047,
- "totalMs": 13063.424291000003,
+ "ttftMs": 6803.918666000012,
+ "modelMs": 6437.507000999991,
+ "toolMs": 0.08199999999487773,
+ "totalMs": 7398.245792000031,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 9411.792582999973,
+ "roundMs": 4249.233708999993,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.30683300003875047
+ "execMs": 0.08199999999487773
}
]
},
{
- "roundMs": 2346.730749999988,
- "firstDeltaMs": 2328.176541999972,
+ "roundMs": 2188.273291999998,
+ "firstDeltaMs": 1593.7093749999767,
"calls": []
}
]
@@ -2590,39 +2644,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 4500.049500000023,
- "execMs": 0.006832999992184341
+ "atMs": 3354.18520800001,
+ "execMs": 0.018459000042639673
}
],
"timings": {
- "gateMs": 1047.8389169999864,
+ "gateMs": 1083.053917000012,
"gateSkipped": false,
- "ttftMs": 7607.273083999986,
- "modelMs": 6592.628499999992,
- "toolMs": 0.006832999992184341,
- "totalMs": 7640.47424999997,
+ "ttftMs": 4543.855916999979,
+ "modelMs": 3678.727542000008,
+ "toolMs": 0.018459000042639673,
+ "totalMs": 4761.7999180000625,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3452.0544579999987,
+ "roundMs": 2270.9358340000035,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.006832999992184341
+ "execMs": 0.018459000042639673
}
]
},
{
- "roundMs": 3140.574041999993,
- "firstDeltaMs": 3107.2032499999623,
+ "roundMs": 1407.7917080000043,
+ "firstDeltaMs": 1189.5909580000443,
"calls": []
}
]
@@ -2630,39 +2685,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 4729.622541000019,
- "execMs": 0.00504200003342703
+ "atMs": 3299.816792000027,
+ "execMs": 0.01433299999916926
}
],
"timings": {
- "gateMs": 1284.0215000000317,
+ "gateMs": 2019.3209170000046,
"gateSkipped": false,
- "ttftMs": 6306.44616599998,
- "modelMs": 5307.502916999976,
- "toolMs": 0.00504200003342703,
- "totalMs": 6591.529459000041,
+ "ttftMs": 4353.434208000021,
+ "modelMs": 2541.2725829999545,
+ "toolMs": 0.01433299999916926,
+ "totalMs": 4560.607832999958,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3445.441916999989,
+ "roundMs": 1280.3408329999656,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.00504200003342703
+ "execMs": 0.01433299999916926
}
]
},
{
- "roundMs": 1862.060999999987,
- "firstDeltaMs": 1576.8071670000209,
+ "roundMs": 1260.9317499999888,
+ "firstDeltaMs": 1053.5772080000024,
"calls": []
}
]
@@ -2677,27 +2733,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1097,
- "p95": 1118,
- "mean": 1037,
+ "p50": 958,
+ "p95": 1060,
+ "mean": 988,
"n": 3
},
"ttftMs": {
- "p50": 6369,
- "p95": 6554,
- "mean": 6089,
+ "p50": 3874,
+ "p95": 5067,
+ "mean": 4206,
"n": 3
},
"totalMs": {
- "p50": 6808,
- "p95": 6879,
- "mean": 6446,
+ "p50": 4506,
+ "p95": 5770,
+ "mean": 4706,
"n": 3
},
"modelMs": {
- "p50": 5711,
- "p95": 5984,
- "mean": 5409,
+ "p50": 3560,
+ "p95": 4711,
+ "mean": 3718,
"n": 3
},
"toolMs": {
@@ -2714,39 +2770,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 4670.480083000031,
- "execMs": 0.3286669999943115
+ "atMs": 2411.452708000026,
+ "execMs": 0.2817919999943115
}
],
"timings": {
- "gateMs": 1097.24099999998,
+ "gateMs": 944.8640000000014,
"gateSkipped": false,
- "ttftMs": 6368.549791000027,
- "modelMs": 5710.624499999976,
- "toolMs": 0.3286669999943115,
- "totalMs": 6808.1941669999505,
+ "ttftMs": 3874.2368750000023,
+ "modelMs": 3560.483041999978,
+ "toolMs": 0.2817919999943115,
+ "totalMs": 4505.628833999974,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3572.4103749999776,
+ "roundMs": 1465.8879999999772,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.3286669999943115
+ "execMs": 0.2817919999943115
}
]
},
{
- "roundMs": 2138.2141249999986,
- "firstDeltaMs": 1697.7074160000193,
+ "roundMs": 2094.595042000001,
+ "firstDeltaMs": 1462.421417000005,
"calls": []
}
]
@@ -2754,39 +2811,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 3898.695624999993,
- "execMs": 0.01304099999833852
+ "atMs": 3582.630332999979,
+ "execMs": 0.013207999989390373
}
],
"timings": {
- "gateMs": 1118.1052079999936,
+ "gateMs": 1059.5932499999763,
"gateSkipped": false,
- "ttftMs": 5345.394708000007,
- "modelMs": 4531.543917000003,
- "toolMs": 0.01304099999833852,
- "totalMs": 5649.662165999995,
+ "ttftMs": 5066.88608299999,
+ "modelMs": 4710.623917000019,
+ "toolMs": 0.013207999989390373,
+ "totalMs": 5770.230374999985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2780.351250000007,
+ "roundMs": 2522.8824170000153,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.01304099999833852
+ "execMs": 0.013207999989390373
}
]
},
{
- "roundMs": 1751.1926669999957,
- "firstDeltaMs": 1446.6560840000166,
+ "roundMs": 2187.7415000000037,
+ "firstDeltaMs": 1484.2061669999966,
"calls": []
}
]
@@ -2794,39 +2852,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 5196.993332999991,
- "execMs": 0.006209000013768673
+ "atMs": 2499.7888750000275,
+ "execMs": 0.008250000013504177
}
],
"timings": {
- "gateMs": 895.320250000048,
+ "gateMs": 958.4654999999912,
"gateSkipped": false,
- "ttftMs": 6553.5641250000335,
- "modelMs": 5984.105582999997,
- "toolMs": 0.006209000013768673,
- "totalMs": 6879.432042000059,
+ "ttftMs": 3676.999208000023,
+ "modelMs": 2883.5358750000014,
+ "toolMs": 0.008250000013504177,
+ "totalMs": 3842.009625000006,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4301.423874999979,
+ "roundMs": 1541.2072499999776,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.006209000013768673
+ "execMs": 0.008250000013504177
}
]
},
{
- "roundMs": 1682.6817080000183,
- "firstDeltaMs": 1356.5495419999934,
+ "roundMs": 1342.3286250000237,
+ "firstDeltaMs": 1177.173084000009,
"calls": []
}
]
@@ -2841,32 +2900,32 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1157,
- "p95": 1625,
- "mean": 1226,
+ "p50": 1556,
+ "p95": 2034,
+ "mean": 1504,
"n": 3
},
"ttftMs": {
- "p50": 5924,
- "p95": 9780,
- "mean": 7126,
+ "p50": 3901,
+ "p95": 5306,
+ "mean": 4139,
"n": 3
},
"totalMs": {
- "p50": 6468,
- "p95": 10178,
- "mean": 7575,
+ "p50": 4086,
+ "p95": 5533,
+ "mean": 4396,
"n": 3
},
"modelMs": {
- "p50": 5311,
- "p95": 8553,
- "mean": 6349,
+ "p50": 2644,
+ "p95": 3499,
+ "mean": 2891,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
@@ -2878,39 +2937,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 4129.579417000001,
- "execMs": 0.5039579999865964
+ "atMs": 2236.439500000037,
+ "execMs": 0.11050000000977889
}
],
"timings": {
- "gateMs": 896.3682909999625,
+ "gateMs": 923.8095410000533,
"gateSkipped": false,
- "ttftMs": 5673.626209000009,
- "modelMs": 5181.163750000007,
- "toolMs": 0.5039579999865964,
- "totalMs": 6078.035998999956,
+ "ttftMs": 3211.9254580000415,
+ "modelMs": 2644.183958999987,
+ "toolMs": 0.11050000000977889,
+ "totalMs": 3568.1040000000503,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3232.4959999999846,
+ "roundMs": 1311.9515000000247,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.5039579999865964
+ "execMs": 0.11050000000977889
}
]
},
{
- "roundMs": 1948.6677500000224,
- "firstDeltaMs": 1543.50329100003,
+ "roundMs": 1332.2324589999625,
+ "firstDeltaMs": 975.3477089999942,
"calls": []
}
]
@@ -2918,39 +2978,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 4437.8222500000265,
- "execMs": 0.01187499996740371
+ "atMs": 4120.633999999962,
+ "execMs": 0.019041999999899417
}
],
"timings": {
- "gateMs": 1157.1306249999907,
+ "gateMs": 2033.900958000042,
"gateSkipped": false,
- "ttftMs": 5923.862207999977,
- "modelMs": 5311.076124000014,
- "toolMs": 0.01187499996740371,
- "totalMs": 6468.218623999972,
+ "ttftMs": 5305.671875,
+ "modelMs": 3499.17404100002,
+ "toolMs": 0.019041999999899417,
+ "totalMs": 5533.094041000062,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3280.501333000022,
+ "roundMs": 2086.494624999992,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.01187499996740371
+ "execMs": 0.019041999999899417
}
]
},
{
- "roundMs": 2030.5747909999918,
- "firstDeltaMs": 1486.0137080000131,
+ "roundMs": 1412.6794160000281,
+ "firstDeltaMs": 1184.983165999991,
"calls": []
}
]
@@ -2958,39 +3019,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 5072.579124999989,
- "execMs": 0.023291999998036772
+ "atMs": 2836.1005420000292,
+ "execMs": 0.02312499994877726
}
],
"timings": {
- "gateMs": 1624.956750000012,
+ "gateMs": 1555.6860829999787,
"gateSkipped": false,
- "ttftMs": 9779.61154100002,
- "modelMs": 8553.49654100003,
- "toolMs": 0.023291999998036772,
- "totalMs": 10178.47658300004,
+ "ttftMs": 3900.8055420000455,
+ "modelMs": 2530.169249999977,
+ "toolMs": 0.02312499994877726,
+ "totalMs": 4085.8784579999046,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3447.4397500000196,
+ "roundMs": 1280.3110829999787,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.023291999998036772
+ "execMs": 0.02312499994877726
}
]
},
{
- "roundMs": 5106.05679100001,
- "firstDeltaMs": 4706.97112500004,
+ "roundMs": 1249.8581669999985,
+ "firstDeltaMs": 1064.6512499999953,
"calls": []
}
]
@@ -3005,27 +3067,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1198,
- "p95": 1580,
- "mean": 1249,
+ "p50": 1013,
+ "p95": 1726,
+ "mean": 1228,
"n": 3
},
"ttftMs": {
- "p50": 8568,
- "p95": 8787,
- "mean": 8001,
+ "p50": 4573,
+ "p95": 4617,
+ "mean": 4347,
"n": 3
},
"totalMs": {
- "p50": 8928,
- "p95": 9111,
- "mean": 8330,
+ "p50": 5329,
+ "p95": 5398,
+ "mean": 5112,
"n": 3
},
"modelMs": {
- "p50": 7531,
- "p95": 7730,
- "mean": 7081,
+ "p50": 3672,
+ "p95": 4316,
+ "mean": 3883,
"n": 3
},
"toolMs": {
@@ -3042,49 +3104,50 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 4512.662624999997,
- "execMs": 0.280709000013303
+ "atMs": 2229.2905829999945,
+ "execMs": 0.2688330000382848
},
{
"name": "set_animation",
"round": 0,
- "atMs": 4513.004083000007,
- "execMs": 0.0094999999855645
+ "atMs": 2229.5893750000396,
+ "execMs": 0.008083999971859157
}
],
"timings": {
- "gateMs": 968.0012500000303,
+ "gateMs": 947.0815419999999,
"gateSkipped": false,
- "ttftMs": 6650.153665999998,
- "modelMs": 5981.667999999947,
- "toolMs": 0.2902089999988675,
- "totalMs": 6949.959458999976,
+ "ttftMs": 3851.668417000037,
+ "modelMs": 3661.5612909999327,
+ "toolMs": 0.27691700001014397,
+ "totalMs": 4608.919749999943,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3544.199790999992,
+ "roundMs": 1281.7654999999795,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.280709000013303
+ "execMs": 0.2688330000382848
},
{
"name": "set_animation",
- "execMs": 0.0094999999855645
+ "execMs": 0.008083999971859157
}
]
},
{
- "roundMs": 2437.468208999955,
- "firstDeltaMs": 2137.087083999999,
+ "roundMs": 2379.795790999953,
+ "firstDeltaMs": 1622.0372079999652,
"calls": []
}
]
@@ -3092,49 +3155,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 6960.538375000004,
- "execMs": 0.0047919999924488366
+ "atMs": 3206.7353749999893,
+ "execMs": 0.0030410000472329557
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6960.549208000011,
- "execMs": 0.003416999999899417
+ "atMs": 3206.748499999987,
+ "execMs": 0.0020409999997355044
}
],
"timings": {
- "gateMs": 1198.1542499999632,
+ "gateMs": 1012.6722910000244,
"gateSkipped": false,
- "ttftMs": 8567.801582999993,
- "modelMs": 7729.84637600003,
- "toolMs": 0.008208999992348254,
- "totalMs": 8928.008834999986,
+ "ttftMs": 4617.4910419999505,
+ "modelMs": 4316.3850410000305,
+ "toolMs": 0.00508200004696846,
+ "totalMs": 5329.062414000102,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5762.119209000026,
+ "roundMs": 2193.7603750000126,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.0047919999924488366
+ "execMs": 0.0030410000472329557
},
{
"name": "set_animation",
- "execMs": 0.003416999999899417
+ "execMs": 0.0020409999997355044
}
]
},
{
- "roundMs": 1967.7271670000046,
- "firstDeltaMs": 1607.2284590000054,
+ "roundMs": 2122.624666000018,
+ "firstDeltaMs": 1410.7180830000434,
"calls": []
}
]
@@ -3142,49 +3206,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 6841.130208000017,
- "execMs": 0.030041999998502433
+ "atMs": 3166.8885829999927,
+ "execMs": 0.012708000023849308
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6841.164166999981,
- "execMs": 0.0047080000513233244
+ "atMs": 3166.9154580000322,
+ "execMs": 0.0030409999890252948
}
],
"timings": {
- "gateMs": 1580.042415999982,
+ "gateMs": 1725.7000419999822,
"gateSkipped": false,
- "ttftMs": 8786.54125000001,
- "modelMs": 7531.031750000024,
- "toolMs": 0.03475000004982576,
- "totalMs": 9111.108916000056,
+ "ttftMs": 4572.919667000009,
+ "modelMs": 3671.994916000054,
+ "toolMs": 0.015749000012874603,
+ "totalMs": 5397.710707000049,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5260.89850000001,
+ "roundMs": 1440.7828750000335,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.030041999998502433
+ "execMs": 0.012708000023849308
},
{
"name": "set_animation",
- "execMs": 0.0047080000513233244
+ "execMs": 0.0030409999890252948
}
]
},
{
- "roundMs": 2270.1332500000135,
- "firstDeltaMs": 1945.3536249999888,
+ "roundMs": 2231.2120410000207,
+ "firstDeltaMs": 1405.9436659999774,
"calls": []
}
]
@@ -3195,368 +3260,255 @@
{
"name": "filler-words-cut-with-editor",
"bucket": "multi-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1273,
- "p95": 1725,
- "mean": 1499,
- "n": 2
+ "p50": 1079,
+ "p95": 2091,
+ "mean": 1368,
+ "n": 3
},
"ttftMs": {
- "p50": 65085,
- "p95": 72333,
- "mean": 68709,
- "n": 2
+ "p50": 41693,
+ "p95": 42980,
+ "mean": 41904,
+ "n": 3
},
"totalMs": {
- "p50": 65717,
- "p95": 72846,
- "mean": 69282,
- "n": 2
+ "p50": 42450,
+ "p95": 43339,
+ "mean": 42449,
+ "n": 3
},
"modelMs": {
- "p50": 64443,
- "p95": 71118,
- "mean": 67780,
- "n": 2
+ "p50": 41246,
+ "p95": 41370,
+ "mean": 41079,
+ "n": 3
},
"toolMs": {
"p50": 1,
"p95": 3,
"mean": 2,
- "n": 2
+ "n": 3
},
"rounds": {
- "mean": 20.5,
- "max": 21
+ "mean": 11.7,
+ "max": 14
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "list_skills",
"round": 0,
- "atMs": 4299.932249999954,
- "execMs": 0.18424999999115244
+ "atMs": 7477.966750000021,
+ "execMs": 0.13133299996843562
},
{
"name": "read_skill",
"round": 1,
- "atMs": 5770.035916999972,
- "execMs": 0.037040999974124134
+ "atMs": 9308.037041000032,
+ "execMs": 0.020334000000730157
},
{
"name": "detect_silence",
"round": 2,
- "atMs": 9339.20749999996,
- "execMs": 0.06945900002028793
+ "atMs": 11897.303332999989,
+ "execMs": 0.017583000008016825
},
{
"name": "listen_audio",
"round": 3,
- "atMs": 12716.74099999998,
- "execMs": 0.019166999962180853
+ "atMs": 14389.883124999993,
+ "execMs": 0.05599999998230487
},
{
- "name": "split_at",
+ "name": "detect_silence",
"round": 4,
- "atMs": 25550.665874999948,
- "execMs": 1.1067909999983385
+ "atMs": 18822.201082999993,
+ "execMs": 0.07779200002551079
},
{
"name": "split_at",
"round": 5,
- "atMs": 27520.15608399996,
- "execMs": 0.11779099999694154
+ "atMs": 28750.56683299999,
+ "execMs": 1.0205840000417084
},
{
"name": "split_at",
- "round": 6,
- "atMs": 29320.683958999987,
- "execMs": 0.07520799996564165
+ "round": 5,
+ "atMs": 28751.643541000027,
+ "execMs": 0.02666600001975894
},
{
- "name": "get_state",
- "round": 7,
- "atMs": 31046.69841699995,
- "execMs": 0.03912499995203689
+ "name": "split_at",
+ "round": 5,
+ "atMs": 28751.67283300002,
+ "execMs": 0.01170899998396635
},
{
"name": "delete_item",
- "round": 8,
- "atMs": 36901.216083999956,
- "execMs": 0.6472499999799766
+ "round": 6,
+ "atMs": 32291.880249999987,
+ "execMs": 0.5122919999994338
},
{
"name": "delete_item",
- "round": 9,
- "atMs": 38839.807458999974,
- "execMs": 0.07820799999171868
- },
- {
- "name": "get_state",
- "round": 10,
- "atMs": 40569.47420899995,
- "execMs": 0.08937500003958121
+ "round": 6,
+ "atMs": 32292.405458000023,
+ "execMs": 0.21804200002225116
},
{
"name": "refine_speech_cuts",
- "round": 11,
- "atMs": 42777.071958999964,
- "execMs": 0.15474999998696148
+ "round": 7,
+ "atMs": 34470.35404100001,
+ "execMs": 0.41058299998985603
},
{
"name": "get_state",
- "round": 12,
- "atMs": 45430.02370899997,
- "execMs": 0.07841600000392646
- },
- {
- "name": "delete_cue",
- "round": 13,
- "atMs": 52090.893333999964,
- "execMs": 0.2989590000361204
- },
- {
- "name": "update_cue",
- "round": 14,
- "atMs": 54127.52866699995,
- "execMs": 0.041415999992750585
- },
- {
- "name": "update_cue",
- "round": 15,
- "atMs": 56189.87349999999,
- "execMs": 0.0542089999653399
- },
- {
- "name": "update_cue",
- "round": 16,
- "atMs": 59006.62637499999,
- "execMs": 0.009250000002793968
- },
- {
- "name": "update_cue",
- "round": 17,
- "atMs": 61198.58891699999,
- "execMs": 0.013124999997671694
+ "round": 8,
+ "atMs": 36657.148916000035,
+ "execMs": 0.19670899998163804
},
{
- "name": "get_state",
- "round": 18,
- "atMs": 70369.75570899999,
- "execMs": 0.2763330000452697
+ "name": "subtitles_generate",
+ "round": 9,
+ "atMs": 38855.45441599999,
+ "execMs": 0.04962499998509884
}
],
"timings": {
- "gateMs": 1724.8017080000136,
+ "gateMs": 933.3387089999742,
"gateSkipped": false,
- "ttftMs": 72333.46783399995,
- "modelMs": 71118.06941700011,
- "toolMs": 3.3901229999028146,
- "totalMs": 72846.26124800002,
+ "ttftMs": 41038.22195799998,
+ "modelMs": 40621.97733600001,
+ "toolMs": 2.7492520000087097,
+ "totalMs": 41558.065296999994,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2574.3982500000275,
+ "roundMs": 6544.272042000026,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.18424999999115244
+ "execMs": 0.13133299996843562
}
]
},
{
- "roundMs": 1469.785333000007,
+ "roundMs": 1829.8839170000283,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.037040999974124134
+ "execMs": 0.020334000000730157
}
]
},
{
- "roundMs": 3569.0324169999803,
+ "roundMs": 2589.2102500000037,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.06945900002028793
+ "execMs": 0.017583000008016825
}
]
},
{
- "roundMs": 3377.423417000042,
+ "roundMs": 2492.4999170000083,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.019166999962180853
+ "execMs": 0.05599999998230487
}
]
},
{
- "roundMs": 12833.862250000006,
+ "roundMs": 4432.07037500001,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 1.1067909999983385
+ "name": "detect_silence",
+ "execMs": 0.07779200002551079
}
]
},
{
- "roundMs": 1968.3169160000398,
+ "roundMs": 9928.146375000011,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.11779099999694154
- }
- ]
- },
- {
- "roundMs": 1800.3382499999716,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 1.0205840000417084
+ },
{
"name": "split_at",
- "execMs": 0.07520799996564165
- }
- ]
- },
- {
- "roundMs": 1725.8930830000318,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.02666600001975894
+ },
{
- "name": "get_state",
- "execMs": 0.03912499995203689
+ "name": "split_at",
+ "execMs": 0.01170899998396635
}
]
},
{
- "roundMs": 5854.428542000009,
+ "roundMs": 3540.1294589999598,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.6472499999799766
- }
- ]
- },
- {
- "roundMs": 1937.8502909999806,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.5122919999994338
+ },
{
"name": "delete_item",
- "execMs": 0.07820799999171868
- }
- ]
- },
- {
- "roundMs": 1729.5197920000064,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.08937500003958121
+ "execMs": 0.21804200002225116
}
]
},
{
- "roundMs": 2207.4635000000126,
+ "roundMs": 2177.6717920000083,
"firstDeltaMs": null,
"calls": [
{
"name": "refine_speech_cuts",
- "execMs": 0.15474999998696148
+ "execMs": 0.41058299998985603
}
]
},
{
- "roundMs": 2652.7430000000168,
+ "roundMs": 2186.3420840000035,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.07841600000392646
+ "execMs": 0.19670899998163804
}
]
},
{
- "roundMs": 6660.717917000002,
+ "roundMs": 2198.0681249999907,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_cue",
- "execMs": 0.2989590000361204
+ "name": "subtitles_generate",
+ "execMs": 0.04962499998509884
}
]
},
{
- "roundMs": 2036.2597500000265,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.041415999992750585
- }
- ]
- },
- {
- "roundMs": 2062.2546250000014,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.0542089999653399
- }
- ]
- },
- {
- "roundMs": 2816.6539169999887,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.009250000002793968
- }
- ]
- },
- {
- "roundMs": 2191.934332999983,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.013124999997671694
- }
- ]
- },
- {
- "roundMs": 9171.109624999983,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.2763330000452697
- }
- ]
- },
- {
- "roundMs": 2478.0842089999933,
- "firstDeltaMs": 1963.3939590000082,
+ "roundMs": 2703.682999999961,
+ "firstDeltaMs": 2182.680249999976,
"calls": []
}
]
@@ -3564,749 +3516,472 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "list_skills",
"round": 0,
- "atMs": 5541.374124999973,
- "execMs": 0.06500000006053597
+ "atMs": 5881.015000000014,
+ "execMs": 0.11337500001536682
},
{
"name": "read_skill",
"round": 1,
- "atMs": 7141.275125000044,
- "execMs": 0.06266699999105185
+ "atMs": 7550.783874999965,
+ "execMs": 0.022499999962747097
},
{
"name": "detect_silence",
"round": 2,
- "atMs": 10870.257208000054,
- "execMs": 0.03154200001154095
+ "atMs": 10481.856291999982,
+ "execMs": 0.038791999977547675
},
{
"name": "listen_audio",
"round": 3,
- "atMs": 13208.644666000037,
- "execMs": 0.03662499994970858
- },
- {
- "name": "read_skill",
- "round": 4,
- "atMs": 19521.68437500007,
- "execMs": 0.028583000064827502
+ "atMs": 12788.841249999998,
+ "execMs": 0.03491699998266995
},
{
"name": "split_at",
- "round": 5,
- "atMs": 29778.48462500004,
- "execMs": 0.053874999983236194
+ "round": 4,
+ "atMs": 24369.635833000008,
+ "execMs": 0.12466599995968863
},
{
"name": "split_at",
- "round": 6,
- "atMs": 31471.45304100006,
- "execMs": 0.255583000020124
+ "round": 4,
+ "atMs": 24369.770917000016,
+ "execMs": 0.007875000010244548
},
{
"name": "split_at",
- "round": 7,
- "atMs": 33480.92087500007,
- "execMs": 0.12783300003502518
- },
- {
- "name": "get_state",
- "round": 8,
- "atMs": 35041.35033300007,
- "execMs": 0.06837499991524965
+ "round": 4,
+ "atMs": 24369.78133299999,
+ "execMs": 0.003832999966107309
},
{
"name": "delete_item",
- "round": 9,
- "atMs": 38398.36408299999,
- "execMs": 0.027708000037819147
+ "round": 5,
+ "atMs": 30239.63400000002,
+ "execMs": 0.24837500002468005
},
{
"name": "delete_item",
- "round": 10,
- "atMs": 39970.54225000006,
- "execMs": 0.08479200000874698
- },
- {
- "name": "get_state",
- "round": 11,
- "atMs": 41676.916582999984,
- "execMs": 0.02283300005365163
+ "round": 5,
+ "atMs": 30239.89595799998,
+ "execMs": 0.03045899997232482
},
{
"name": "refine_speech_cuts",
- "round": 12,
- "atMs": 44321.467291000066,
- "execMs": 0.058041000040248036
+ "round": 6,
+ "atMs": 32720.613416999986,
+ "execMs": 0.14070800004992634
},
{
"name": "get_state",
- "round": 13,
- "atMs": 47999.318791000056,
- "execMs": 0.025040999986231327
+ "round": 7,
+ "atMs": 34392.04954199999,
+ "execMs": 0.15887499996460974
},
{
"name": "update_cue",
- "round": 14,
- "atMs": 53451.320540999994,
- "execMs": 0.0388329999987036
+ "round": 8,
+ "atMs": 40922.11391700001,
+ "execMs": 0.06941599998390302
},
{
"name": "update_cue",
- "round": 15,
- "atMs": 55060.823875,
- "execMs": 0.08908300008624792
+ "round": 8,
+ "atMs": 40922.19541699998,
+ "execMs": 0.17699999996693805
},
{
"name": "delete_cue",
- "round": 16,
- "atMs": 57308.43975000002,
- "execMs": 0.019874999998137355
+ "round": 8,
+ "atMs": 40922.37754199997,
+ "execMs": 0.05937499995343387
},
{
"name": "update_cue",
- "round": 17,
- "atMs": 59091.295166000025,
- "execMs": 0.07829099998343736
+ "round": 8,
+ "atMs": 40922.438957999984,
+ "execMs": 0.009541000006720424
},
{
"name": "update_cue",
- "round": 18,
- "atMs": 60881.07612500002,
- "execMs": 0.026000000070780516
- },
- {
- "name": "get_state",
- "round": 19,
- "atMs": 63410.946291,
- "execMs": 0.04562500002793968
+ "round": 8,
+ "atMs": 40922.44949999999,
+ "execMs": 0.005292000016197562
}
],
"timings": {
- "gateMs": 1273.0824160000193,
+ "gateMs": 2091.3710420000134,
"gateSkipped": false,
- "ttftMs": 65085.010499999975,
- "modelMs": 64442.74074899993,
- "toolMs": 1.2462050003232434,
- "totalMs": 65717.06937000027,
+ "ttftMs": 42980.38937499997,
+ "modelMs": 41245.99137599999,
+ "toolMs": 1.244998999813106,
+ "totalMs": 43338.60741699982,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4268.075291000016,
+ "roundMs": 3789.4030839999905,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.06500000006053597
+ "execMs": 0.11337500001536682
}
]
},
{
- "roundMs": 1599.7701249999227,
+ "roundMs": 1669.497457999969,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.06266699999105185
+ "execMs": 0.022499999962747097
}
]
},
{
- "roundMs": 3728.8537080000388,
+ "roundMs": 2930.953291999991,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.03154200001154095
+ "execMs": 0.038791999977547675
}
]
},
{
- "roundMs": 2338.2905419999734,
+ "roundMs": 2306.904917000036,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.03662499994970858
- }
- ]
- },
- {
- "roundMs": 6312.925874999957,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.028583000064827502
+ "execMs": 0.03491699998266995
}
]
},
{
- "roundMs": 10256.710458000074,
+ "roundMs": 11580.69312499999,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.053874999983236194
- }
- ]
- },
- {
- "roundMs": 1692.8655830000062,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.12466599995968863
+ },
{
"name": "split_at",
- "execMs": 0.255583000020124
- }
- ]
- },
- {
- "roundMs": 2009.1369580000173,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.007875000010244548
+ },
{
"name": "split_at",
- "execMs": 0.12783300003502518
- }
- ]
- },
- {
- "roundMs": 1560.2351250000065,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.06837499991524965
+ "execMs": 0.003832999966107309
}
]
},
{
- "roundMs": 3356.901959000039,
+ "roundMs": 5869.80037500005,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.027708000037819147
- }
- ]
- },
- {
- "roundMs": 1572.107333999942,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.24837500002468005
+ },
{
"name": "delete_item",
- "execMs": 0.08479200000874698
- }
- ]
- },
- {
- "roundMs": 1706.2426660000347,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.02283300005365163
+ "execMs": 0.03045899997232482
}
]
},
{
- "roundMs": 2644.466291000019,
+ "roundMs": 2480.5850830000127,
"firstDeltaMs": null,
"calls": [
{
"name": "refine_speech_cuts",
- "execMs": 0.058041000040248036
+ "execMs": 0.14070800004992634
}
]
},
{
- "roundMs": 3677.737749999971,
+ "roundMs": 1671.1985419999692,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.025040999986231327
+ "execMs": 0.15887499996460974
}
]
},
{
- "roundMs": 5451.931958000059,
+ "roundMs": 6529.8252090000315,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.0388329999987036
- }
- ]
- },
- {
- "roundMs": 1609.404416999896,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.06941599998390302
+ },
{
"name": "update_cue",
- "execMs": 0.08908300008624792
- }
- ]
- },
- {
- "roundMs": 2247.483125000028,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.17699999996693805
+ },
{
"name": "delete_cue",
- "execMs": 0.019874999998137355
- }
- ]
- },
- {
- "roundMs": 1782.7879999999423,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.05937499995343387
+ },
{
"name": "update_cue",
- "execMs": 0.07829099998343736
- }
- ]
- },
- {
- "roundMs": 1789.6252919998951,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.009541000006720424
+ },
{
"name": "update_cue",
- "execMs": 0.026000000070780516
- }
- ]
- },
- {
- "roundMs": 2529.7782500000903,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.04562500002793968
+ "execMs": 0.005292000016197562
}
]
},
{
- "roundMs": 2307.410042000003,
- "firstDeltaMs": 1673.9886249999981,
+ "roundMs": 2417.1302909999504,
+ "firstDeltaMs": 2057.876249999972,
"calls": []
}
]
}
},
{
- "pass": false,
- "intent": "work",
- "notes": [
- "reply did not match /filler|um|uh|cut|remove|trim/i"
- ],
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [],
"trace": [
{
"name": "list_skills",
"round": 0,
- "atMs": 5113.2021670000395,
- "execMs": 0.04179199994541705
+ "atMs": 5107.901584000036,
+ "execMs": 0.023999999975785613
},
{
"name": "read_skill",
"round": 1,
- "atMs": 6832.653792000026,
- "execMs": 0.01762499997857958
+ "atMs": 6620.963125000009,
+ "execMs": 0.05904199997894466
},
{
"name": "detect_silence",
"round": 2,
- "atMs": 9442.567750000046,
- "execMs": 0.04550000000745058
+ "atMs": 9157.826459000004,
+ "execMs": 0.007625000027474016
},
{
"name": "listen_audio",
"round": 3,
- "atMs": 13052.46575000009,
- "execMs": 0.042625000001862645
+ "atMs": 11257.563375000027,
+ "execMs": 0.008333000005222857
},
{
"name": "detect_silence",
"round": 4,
- "atMs": 16251.032125000027,
- "execMs": 0.021000000066123903
+ "atMs": 18514.550917000044,
+ "execMs": 0.025083000015001744
},
{
- "name": "read_skill",
+ "name": "split_at",
"round": 5,
- "atMs": 31743.619292000076,
- "execMs": 0.020791999995708466
+ "atMs": 23668.270042000047,
+ "execMs": 0.15054200001759455
},
{
"name": "split_at",
"round": 6,
- "atMs": 49973.84158400004,
- "execMs": 0.1360839999979362
+ "atMs": 25868.38620900002,
+ "execMs": 0.28104099998017773
},
{
- "name": "get_state",
+ "name": "split_at",
"round": 7,
- "atMs": 51902.381584000075,
- "execMs": 0.06941700004972517
+ "atMs": 27650.57462500001,
+ "execMs": 0.18975000001955777
},
{
- "name": "split_at",
+ "name": "delete_item",
"round": 8,
- "atMs": 53872.70766700001,
- "execMs": 0.07970899995416403
+ "atMs": 30577.760459000012,
+ "execMs": 0.04512499994598329
},
{
- "name": "split_at",
+ "name": "delete_item",
"round": 9,
- "atMs": 56449.399124999996,
- "execMs": 0.059125000028871
+ "atMs": 32673.995084000053,
+ "execMs": 0.024291999987326562
},
{
- "name": "get_state",
+ "name": "refine_speech_cuts",
"round": 10,
- "atMs": 58272.82829199999,
- "execMs": 0.05162499996367842
+ "atMs": 34768.73200000002,
+ "execMs": 0.054207999957725406
},
{
- "name": "delete_item",
+ "name": "get_state",
"round": 11,
- "atMs": 68101.73600000003,
- "execMs": 0.03837500000372529
+ "atMs": 36874.84729200002,
+ "execMs": 0.13575000001583248
},
{
- "name": "get_state",
+ "name": "subtitles_generate",
"round": 12,
- "atMs": 69922.40833400004,
- "execMs": 0.20095800003036857
- },
- {
- "name": "delete_item",
- "round": 13,
- "atMs": 78200.44741700008,
- "execMs": 0.04804199992213398
- },
- {
- "name": "get_state",
- "round": 14,
- "atMs": 87772.47508400003,
- "execMs": 0.0789580000564456
- },
- {
- "name": "refine_speech_cuts",
- "round": 15,
- "atMs": 94272.03116700007,
- "execMs": 0.1033330000936985
- },
- {
- "name": "get_state",
- "round": 16,
- "atMs": 100179.66833400005,
- "execMs": 0.047916000010445714
- },
- {
- "name": "delete_cue",
- "round": 17,
- "atMs": 106503.91487500002,
- "execMs": 0.09116700000595301
- },
- {
- "name": "update_cue",
- "round": 18,
- "atMs": 108862.15312500007,
- "execMs": 0.11591699998825788
- },
- {
- "name": "update_cue",
- "round": 19,
- "atMs": 111612.83379200008,
- "execMs": 0.1373750000493601
- },
- {
- "name": "update_cue",
- "round": 20,
- "atMs": 118492.89508400008,
- "execMs": 0.04650000005494803
- },
- {
- "name": "update_cue",
- "round": 21,
- "atMs": 120639.65754200006,
- "execMs": 0.02804199990350753
- },
- {
- "name": "delete_item",
- "round": 22,
- "atMs": 123972.09708400001,
- "execMs": 0.16987500002142042
- },
- {
- "name": "delete_item",
- "round": 23,
- "atMs": 126180.20666700008,
- "execMs": 0.015499999979510903
+ "atMs": 39168.492959000054,
+ "execMs": 0.012584000010974705
}
],
"timings": {
- "gateMs": 1105.7318750000559,
+ "gateMs": 1078.7151659999508,
"gateSkipped": false,
- "ttftMs": null,
- "modelMs": 125070.92979099974,
- "toolMs": 1.7072520001092926,
- "totalMs": 126178.3689179999,
+ "ttftMs": 41693.456042000034,
+ "modelMs": 41369.84000099992,
+ "toolMs": 1.0173749999376014,
+ "totalMs": 42449.572541999805,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4007.3619999999646,
+ "roundMs": 4028.8258340000175,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.04179199994541705
+ "execMs": 0.023999999975785613
}
]
},
{
- "roundMs": 1719.3523750000168,
+ "roundMs": 1512.990582999948,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.01762499997857958
+ "execMs": 0.05904199997894466
}
]
},
{
- "roundMs": 2609.8331659999676,
+ "roundMs": 2536.7522500000196,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.04550000000745058
+ "execMs": 0.007625000027474016
}
]
},
{
- "roundMs": 3609.7822500000475,
+ "roundMs": 2099.7122079999535,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.042625000001862645
+ "execMs": 0.008333000005222857
}
]
},
{
- "roundMs": 3198.473665999947,
+ "roundMs": 7256.931333000015,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.021000000066123903
- }
- ]
- },
- {
- "roundMs": 15492.513708000071,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.020791999995708466
+ "execMs": 0.025083000015001744
}
]
},
{
- "roundMs": 18230.14266699995,
+ "roundMs": 5153.652249999985,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.1360839999979362
- }
- ]
- },
- {
- "roundMs": 1928.3363749999553,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.06941700004972517
+ "execMs": 0.15054200001759455
}
]
},
{
- "roundMs": 1970.18833299994,
+ "roundMs": 2199.7816250000033,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.07970899995416403
+ "execMs": 0.28104099998017773
}
]
},
{
- "roundMs": 2576.565207999898,
+ "roundMs": 1781.7388339999598,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.059125000028871
- }
- ]
- },
- {
- "roundMs": 1823.3292920000385,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.05162499996367842
+ "execMs": 0.18975000001955777
}
]
},
{
- "roundMs": 9828.787167000002,
+ "roundMs": 2926.91720799997,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.03837500000372529
+ "execMs": 0.04512499994598329
}
]
},
{
- "roundMs": 1820.5054579999996,
+ "roundMs": 2096.171833999979,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.20095800003036857
+ "name": "delete_item",
+ "execMs": 0.024291999987326562
}
]
},
{
- "roundMs": 8277.689041999984,
+ "roundMs": 2094.696667000011,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.04804199992213398
+ "name": "refine_speech_cuts",
+ "execMs": 0.054207999957725406
}
]
},
{
- "roundMs": 9571.89370799996,
+ "roundMs": 2106.023583000002,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.0789580000564456
+ "execMs": 0.13575000001583248
}
]
},
{
- "roundMs": 6499.387125000008,
+ "roundMs": 2293.4708330000285,
"firstDeltaMs": null,
"calls": [
{
- "name": "refine_speech_cuts",
- "execMs": 0.1033330000936985
+ "name": "subtitles_generate",
+ "execMs": 0.012584000010974705
}
]
},
{
- "roundMs": 5907.490624999977,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.047916000010445714
- }
- ]
- },
- {
- "roundMs": 6324.150707999943,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_cue",
- "execMs": 0.09116700000595301
- }
- ]
- },
- {
- "roundMs": 2357.9922500000102,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.11591699998825788
- }
- ]
- },
- {
- "roundMs": 2750.441749999998,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.1373750000493601
- }
- ]
- },
- {
- "roundMs": 6879.854334000032,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.04650000005494803
- }
- ]
- },
- {
- "roundMs": 2146.6089589999756,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.02804199990350753
- }
- ]
- },
- {
- "roundMs": 3332.3504590000957,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.16987500002142042
- }
- ]
- },
- {
- "roundMs": 2207.899165999959,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.015499999979510903
- }
- ]
+ "roundMs": 3282.1749590000254,
+ "firstDeltaMs": 2524.9222500000033,
+ "calls": []
}
]
}
@@ -4320,27 +3995,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1247,
- "p95": 1867,
- "mean": 1362,
+ "p50": 1239,
+ "p95": 1839,
+ "mean": 1392,
"n": 3
},
"ttftMs": {
- "p50": 6278,
- "p95": 9251,
- "mean": 7181,
+ "p50": 4221,
+ "p95": 8361,
+ "mean": 5378,
"n": 3
},
"totalMs": {
- "p50": 6669,
- "p95": 9681,
- "mean": 7586,
+ "p50": 4631,
+ "p95": 8697,
+ "mean": 5755,
"n": 3
},
"modelMs": {
- "p50": 5423,
- "p95": 8709,
- "mean": 6225,
+ "p50": 2840,
+ "p95": 7457,
+ "mean": 4363,
"n": 3
},
"toolMs": {
@@ -4357,39 +4032,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 4519.682167000021,
- "execMs": 0.04450000007636845
+ "atMs": 6875.302167000016,
+ "execMs": 0.004625000001396984
}
],
"timings": {
- "gateMs": 1246.770416000043,
+ "gateMs": 1239.3268340000068,
"gateSkipped": false,
- "ttftMs": 6278.209167000023,
- "modelMs": 5422.533707999974,
- "toolMs": 0.04450000007636845,
- "totalMs": 6669.348624000093,
+ "ttftMs": 8361.200750000018,
+ "modelMs": 7457.415333000012,
+ "toolMs": 0.004625000001396984,
+ "totalMs": 8696.74679200002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3272.6996250000084,
+ "roundMs": 5635.262042000017,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.04450000007636845
+ "execMs": 0.004625000001396984
}
]
},
{
- "roundMs": 2149.834082999965,
- "firstDeltaMs": 1758.4317909999518,
+ "roundMs": 1822.153290999995,
+ "firstDeltaMs": 1485.8286250000237,
"calls": []
}
]
@@ -4397,39 +4073,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 4121.114708999987,
- "execMs": 0.025375000084750354
+ "atMs": 2340.7985839999747,
+ "execMs": 0.03441600000951439
}
],
"timings": {
- "gateMs": 1866.5903339999495,
+ "gateMs": 1098.4453750000102,
"gateSkipped": false,
- "ttftMs": 6015.095124999993,
- "modelMs": 4541.697334000026,
- "toolMs": 0.025375000084750354,
- "totalMs": 6408.31304300006,
+ "ttftMs": 3552.5767500000075,
+ "modelMs": 2840.120166999928,
+ "toolMs": 0.03441600000951439,
+ "totalMs": 3938.599957999948,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2254.2087090000277,
+ "roundMs": 1242.0259999999544,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.025375000084750354
+ "execMs": 0.03441600000951439
}
]
},
{
- "roundMs": 2287.488624999998,
- "firstDeltaMs": 1893.8966249999357,
+ "roundMs": 1598.0941669999738,
+ "firstDeltaMs": 1211.675500000012,
"calls": []
}
]
@@ -4437,39 +4114,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 7131.981333000003,
- "execMs": 0.012708000023849308
+ "atMs": 3299.9940829999978,
+ "execMs": 0.008000000030733645
}
],
"timings": {
- "gateMs": 971.9428329999791,
+ "gateMs": 1838.9000829999568,
"gateSkipped": false,
- "ttftMs": 9250.59933300002,
- "modelMs": 8709.499083000002,
- "toolMs": 0.012708000023849308,
- "totalMs": 9681.454624000005,
+ "ttftMs": 4220.94854100002,
+ "modelMs": 2792.112124999927,
+ "toolMs": 0.008000000030733645,
+ "totalMs": 4631.020207999914,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6159.858291000011,
+ "roundMs": 1460.918040999968,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.012708000023849308
+ "execMs": 0.008000000030733645
}
]
},
{
- "roundMs": 2549.640791999991,
- "firstDeltaMs": 2118.556666999939,
+ "roundMs": 1331.1940839999588,
+ "firstDeltaMs": 920.904458999983,
"calls": []
}
]
@@ -4484,124 +4162,77 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1078,
- "p95": 1358,
- "mean": 1163,
+ "p50": 855,
+ "p95": 1166,
+ "mean": 956,
"n": 3
},
"ttftMs": {
- "p50": 24161,
- "p95": 33704,
- "mean": 21939,
+ "p50": 7863,
+ "p95": 11121,
+ "mean": 8567,
"n": 3
},
"totalMs": {
- "p50": 24686,
- "p95": 34069,
- "mean": 22394,
+ "p50": 8240,
+ "p95": 11476,
+ "mean": 8931,
"n": 3
},
"modelMs": {
- "p50": 23631,
- "p95": 32991,
- "mean": 21231,
+ "p50": 7074,
+ "p95": 10621,
+ "mean": 7975,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
"rounds": {
- "mean": 4,
- "max": 5
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 4679.521625000052,
- "execMs": 0.264915999956429
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 6377.977083000005,
- "execMs": 0.02504199999384582
- },
- {
- "name": "listen_audio",
- "round": 2,
- "atMs": 8848.066208000062,
- "execMs": 0.04312500008381903
- },
{
"name": "generate_scene",
- "round": 3,
- "atMs": 22139.729082999984,
- "execMs": 0.28633400006219745
+ "round": 0,
+ "atMs": 9128.564042000042,
+ "execMs": 0.4116249999497086
}
],
"timings": {
- "gateMs": 1054.172540999949,
+ "gateMs": 854.8156249999884,
"gateSkipped": false,
- "ttftMs": 24161.236291000037,
- "modelMs": 23631.441375000053,
- "toolMs": 0.6194170000962913,
- "totalMs": 24686.2333330001,
+ "ttftMs": 11121.02129200002,
+ "modelMs": 10620.653667000006,
+ "toolMs": 0.4116249999497086,
+ "totalMs": 11475.880916999944,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3624.7014999999665,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.264915999956429
- }
- ]
- },
- {
- "roundMs": 1698.1151250000112,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.02504199999384582
- }
- ]
- },
- {
- "roundMs": 2469.890417000046,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.04312500008381903
- }
- ]
- },
- {
- "roundMs": 13291.51312499994,
+ "roundMs": 8272.880917000002,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.28633400006219745
+ "execMs": 0.4116249999497086
}
]
},
{
- "roundMs": 2547.22120800009,
- "firstDeltaMs": 2021.1848750000354,
+ "roundMs": 2347.7727500000037,
+ "firstDeltaMs": 1991.9787500000093,
"calls": []
}
]
@@ -4609,87 +4240,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 6250.848958999966,
- "execMs": 0.020249999943189323
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 8000.339999999967,
- "execMs": 0.04345900006592274
- },
- {
- "name": "listen_audio",
- "round": 2,
- "atMs": 10447.527541999938,
- "execMs": 0.0043750000186264515
- },
{
"name": "generate_scene",
- "round": 3,
- "atMs": 28689.649124999996,
- "execMs": 0.011458000051788986
+ "round": 0,
+ "atMs": 6097.852833000012,
+ "execMs": 0.009167000011075288
}
],
"timings": {
- "gateMs": 1077.7192090000026,
+ "gateMs": 1165.5827079999726,
"gateSkipped": false,
- "ttftMs": 33704.165833999985,
- "modelMs": 32990.79037599999,
- "toolMs": 0.0795420000795275,
- "totalMs": 34068.58912700007,
+ "ttftMs": 7862.998291000025,
+ "modelMs": 7074.251126000076,
+ "toolMs": 0.009167000011075288,
+ "totalMs": 8239.84300100006,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5172.859417000087,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.020249999943189323
- }
- ]
- },
- {
- "roundMs": 1749.4041669999715,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.04345900006592274
- }
- ]
- },
- {
- "roundMs": 2447.0942499999655,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.0043750000186264515
- }
- ]
- },
- {
- "roundMs": 18242.090959000052,
+ "roundMs": 4931.990834000055,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.011458000051788986
+ "execMs": 0.009167000011075288
}
]
},
{
- "roundMs": 5379.341582999914,
- "firstDeltaMs": 5014.484624999925,
+ "roundMs": 2142.260292000021,
+ "firstDeltaMs": 1765.1068749999977,
"calls": []
}
]
@@ -4697,39 +4281,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 6231.336374999955,
- "execMs": 0.019500000053085387
+ "atMs": 5130.574709000008,
+ "execMs": 0.008749999979045242
}
],
"timings": {
- "gateMs": 1358.28125,
+ "gateMs": 847.5515000000014,
"gateSkipped": false,
- "ttftMs": 7952.655999999959,
- "modelMs": 7069.682875000057,
- "toolMs": 0.019500000053085387,
- "totalMs": 8427.98362500011,
+ "ttftMs": 6715.810125000018,
+ "modelMs": 6230.857875999936,
+ "toolMs": 0.008749999979045242,
+ "totalMs": 7078.418125999917,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4872.868084000074,
+ "roundMs": 4282.869958999974,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.019500000053085387
+ "execMs": 0.008749999979045242
}
]
},
{
- "roundMs": 2196.8147909999825,
- "firstDeltaMs": 1721.2681249999441,
+ "roundMs": 1947.9879169999622,
+ "firstDeltaMs": 1585.2145829999936,
"calls": []
}
]
@@ -4744,27 +4329,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1127,
- "p95": 1789,
- "mean": 1292,
+ "p50": 1405,
+ "p95": 2060,
+ "mean": 1441,
"n": 3
},
"ttftMs": {
- "p50": 8955,
- "p95": 22568,
- "mean": 13337,
+ "p50": 8879,
+ "p95": 10226,
+ "mean": 8954,
"n": 3
},
"totalMs": {
- "p50": 9271,
- "p95": 23037,
- "mean": 13743,
+ "p50": 9125,
+ "p95": 10542,
+ "mean": 9283,
"n": 3
},
"modelMs": {
- "p50": 7964,
- "p95": 21910,
- "mean": 12452,
+ "p50": 7720,
+ "p95": 9682,
+ "mean": 7841,
"n": 3
},
"toolMs": {
@@ -4774,46 +4359,47 @@
"n": 3
},
"rounds": {
- "mean": 3,
- "max": 5
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 7132.29241600004,
- "execMs": 0.4175830000312999
+ "atMs": 7058.804375000007,
+ "execMs": 0.3601659999694675
}
],
"timings": {
- "gateMs": 1789.3681250000373,
+ "gateMs": 1404.968542000046,
"gateSkipped": false,
- "ttftMs": 8954.946291,
- "modelMs": 7480.938123999978,
- "toolMs": 0.4175830000312999,
- "totalMs": 9270.723832000047,
+ "ttftMs": 8879.27004199999,
+ "modelMs": 7719.889625000011,
+ "toolMs": 0.3601659999694675,
+ "totalMs": 9125.218333000026,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5342.749833000009,
+ "roundMs": 5652.8927080000285,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.4175830000312999
+ "execMs": 0.3601659999694675
}
]
},
{
- "roundMs": 2138.1882909999695,
- "firstDeltaMs": 1822.1991249999264,
+ "roundMs": 2066.996916999982,
+ "firstDeltaMs": 1820.0467089999584,
"calls": []
}
]
@@ -4821,87 +4407,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 6011.813833000022,
- "execMs": 0.018249999964609742
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 7751.004582999973,
- "execMs": 0.020875000045634806
- },
- {
- "name": "listen_audio",
- "round": 2,
- "atMs": 10381.920291000046,
- "execMs": 0.022249999921768904
- },
{
"name": "generate_scene",
- "round": 3,
- "atMs": 20481.638957999996,
- "execMs": 0.03625000000465661
+ "round": 0,
+ "atMs": 7883.361250000016,
+ "execMs": 0.02966599998762831
}
],
"timings": {
- "gateMs": 1127.1637920000358,
+ "gateMs": 859.3850000000093,
"gateSkipped": false,
- "ttftMs": 22568.35699999996,
- "modelMs": 21910.028374999878,
- "toolMs": 0.09762499993667006,
- "totalMs": 23037.28979199985,
+ "ttftMs": 10225.966000000015,
+ "modelMs": 9682.103541999939,
+ "toolMs": 0.02966599998762831,
+ "totalMs": 10541.518207999936,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4884.387375000049,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.018249999964609742
- }
- ]
- },
- {
- "roundMs": 1739.1084580000024,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.020875000045634806
- }
- ]
- },
- {
- "roundMs": 2630.758541999967,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.022249999921768904
- }
- ]
- },
- {
- "roundMs": 10099.625207999954,
+ "roundMs": 7023.776041999983,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.03625000000465661
+ "execMs": 0.02966599998762831
}
]
},
{
- "roundMs": 2556.1487919999054,
- "firstDeltaMs": 2086.64829199994,
+ "roundMs": 2658.3274999999558,
+ "firstDeltaMs": 2342.5149999999558,
"calls": []
}
]
@@ -4909,39 +4448,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 6493.074708000058,
- "execMs": 0.030749999918043613
+ "atMs": 5980.798583999975,
+ "execMs": 0.02895800000987947
}
],
"timings": {
- "gateMs": 958.3832920000423,
+ "gateMs": 2059.719458000036,
"gateSkipped": false,
- "ttftMs": 8488.155875000055,
- "modelMs": 7963.98666699999,
- "toolMs": 0.030749999918043613,
- "totalMs": 8922.40070899995,
+ "ttftMs": 7755.575624999998,
+ "modelMs": 6121.51345800003,
+ "toolMs": 0.02895800000987947,
+ "totalMs": 8181.261874000076,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5534.5766669999575,
+ "roundMs": 3920.896250000049,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.030749999918043613
+ "execMs": 0.02895800000987947
}
]
},
{
- "roundMs": 2429.4100000000326,
- "firstDeltaMs": 1995.014208000037,
+ "roundMs": 2200.6172079999815,
+ "firstDeltaMs": 1774.7047080000048,
"calls": []
}
]
@@ -4962,21 +4502,21 @@
"n": 3
},
"ttftMs": {
- "p50": 8413,
- "p95": 9867,
- "mean": 8187,
+ "p50": 7546,
+ "p95": 10273,
+ "mean": 8092,
"n": 3
},
"totalMs": {
- "p50": 8902,
- "p95": 10190,
- "mean": 8582,
+ "p50": 7951,
+ "p95": 10599,
+ "mean": 8485,
"n": 3
},
"modelMs": {
- "p50": 8902,
- "p95": 10190,
- "mean": 8582,
+ "p50": 7951,
+ "p95": 10599,
+ "mean": 8485,
"n": 3
},
"toolMs": {
@@ -4993,39 +4533,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 4559.398874999955,
- "execMs": 0.3616250000195578
+ "atMs": 4815.7670409999555,
+ "execMs": 0.23616700002457947
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6282.474125000066,
- "modelMs": 6654.902833000058,
- "toolMs": 0.3616250000195578,
- "totalMs": 6655.264458000078,
+ "ttftMs": 6456.270416000043,
+ "modelMs": 6904.481874999998,
+ "toolMs": 0.23616700002457947,
+ "totalMs": 6904.718042000022,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4558.882750000106,
+ "roundMs": 4815.239750000008,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.3616250000195578
+ "execMs": 0.23616700002457947
}
]
},
{
- "roundMs": 2096.020082999952,
- "firstDeltaMs": 1722.6777500000317,
+ "roundMs": 2089.2421249999898,
+ "firstDeltaMs": 1640.2082089999458,
"calls": []
}
]
@@ -5033,39 +4574,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 7575.602124999976,
- "execMs": 0.03629199997521937
+ "atMs": 7900.776249999995,
+ "execMs": 0.008874999941326678
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 9866.689250000054,
- "modelMs": 10189.892540999921,
- "toolMs": 0.03629199997521937,
- "totalMs": 10189.928832999896,
+ "ttftMs": 10272.922542000073,
+ "modelMs": 10599.437501000008,
+ "toolMs": 0.008874999941326678,
+ "totalMs": 10599.446375999949,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7575.519290999975,
+ "roundMs": 7900.653166999924,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.03629199997521937
+ "execMs": 0.008874999941326678
}
]
},
{
- "roundMs": 2614.373249999946,
- "firstDeltaMs": 2291.015167000005,
+ "roundMs": 2698.7843340000836,
+ "firstDeltaMs": 2372.1214590000454,
"calls": []
}
]
@@ -5073,39 +4615,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 6410.991959000006,
- "execMs": 0.006917000049725175
+ "atMs": 5653.433792000054,
+ "execMs": 0.031083000008948147
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 8413.130874999915,
- "modelMs": 8902.132915999973,
- "toolMs": 0.006917000049725175,
- "totalMs": 8902.139833000023,
+ "ttftMs": 7545.831292000017,
+ "modelMs": 7950.991458999924,
+ "toolMs": 0.031083000008948147,
+ "totalMs": 7951.022541999933,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6410.9046249999665,
+ "roundMs": 5653.302541999961,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.006917000049725175
+ "execMs": 0.031083000008948147
}
]
},
{
- "roundMs": 2491.2282910000067,
- "firstDeltaMs": 2002.1234160000458,
+ "roundMs": 2297.688916999963,
+ "firstDeltaMs": 1892.3209169999463,
"calls": []
}
]
@@ -5120,27 +4663,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1074,
- "p95": 1096,
- "mean": 1056,
+ "p50": 1347,
+ "p95": 1857,
+ "mean": 1450,
"n": 3
},
"ttftMs": {
- "p50": 6025,
- "p95": 7774,
- "mean": 6506,
+ "p50": 8570,
+ "p95": 9092,
+ "mean": 8488,
"n": 3
},
"totalMs": {
- "p50": 6600,
- "p95": 8658,
- "mean": 7195,
+ "p50": 9027,
+ "p95": 9571,
+ "mean": 9066,
"n": 3
},
"modelMs": {
- "p50": 5525,
- "p95": 7660,
- "mean": 6139,
+ "p50": 7680,
+ "p95": 7715,
+ "mean": 7616,
"n": 3
},
"toolMs": {
@@ -5157,22 +4700,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 997.9349159999983,
+ "gateMs": 1147.1029580000322,
"gateSkipped": false,
- "ttftMs": 7773.699625000008,
- "modelMs": 7659.96420799999,
+ "ttftMs": 7800.428583999979,
+ "modelMs": 7452.815333000035,
"toolMs": 0,
- "totalMs": 8657.899123999989,
+ "totalMs": 8599.918291000067,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7659.96420799999,
- "firstDeltaMs": 6775.277417000034,
+ "roundMs": 7452.815333000035,
+ "firstDeltaMs": 6652.936208000057,
"calls": []
}
]
@@ -5180,22 +4724,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1074.2879169998923,
+ "gateMs": 1346.6199169999454,
"gateSkipped": false,
- "ttftMs": 6024.602667000028,
- "modelMs": 5525.330624999944,
+ "ttftMs": 8570.110209000064,
+ "modelMs": 7680.4369590000715,
"toolMs": 0,
- "totalMs": 6599.618541999836,
+ "totalMs": 9027.056876000017,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5525.330624999944,
- "firstDeltaMs": 4950.074082999956,
+ "roundMs": 7680.4369590000715,
+ "firstDeltaMs": 7223.1557500000345,
"calls": []
}
]
@@ -5203,22 +4748,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1095.7907500000438,
+ "gateMs": 1856.7687920000171,
"gateSkipped": false,
- "ttftMs": 5718.922915999894,
- "modelMs": 5231.582167000044,
+ "ttftMs": 9092.017624999979,
+ "modelMs": 7714.671666000038,
"toolMs": 0,
- "totalMs": 6327.372917000088,
+ "totalMs": 9571.440458000056,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5231.582167000044,
- "firstDeltaMs": 4622.899042000063,
+ "roundMs": 7714.671666000038,
+ "firstDeltaMs": 7235.0339580000145,
"calls": []
}
]
@@ -5229,344 +4775,238 @@
{
"name": "clip-mentions-redo-their-shots",
"bucket": "multi-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 3
},
"ttftMs": {
- "p50": 26602,
- "p95": 35521,
- "mean": 31062,
- "n": 2
+ "p50": 19115,
+ "p95": 26901,
+ "mean": 21170,
+ "n": 3
},
"totalMs": {
- "p50": 27159,
- "p95": 35858,
- "mean": 31508,
- "n": 2
+ "p50": 19820,
+ "p95": 27224,
+ "mean": 21799,
+ "n": 3
},
"modelMs": {
- "p50": 27159,
- "p95": 35857,
- "mean": 31508,
- "n": 2
+ "p50": 19820,
+ "p95": 27224,
+ "mean": 21799,
+ "n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
- "n": 2
+ "n": 3
},
"rounds": {
- "mean": 7.5,
- "max": 9
+ "mean": 4.7,
+ "max": 5
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 4873.609792000032,
- "execMs": 0.5280000000493601
+ "atMs": 3703.8745830000844,
+ "execMs": 0.1215000000083819
},
{
"name": "watch_video",
"round": 1,
- "atMs": 7619.716833999963,
- "execMs": 0.008290999918244779
+ "atMs": 6670.952708000084,
+ "execMs": 0.03249999997206032
},
{
"name": "watch_video",
"round": 1,
- "atMs": 7619.7342499999795,
- "execMs": 0.0006670000730082393
+ "atMs": 6671.021249999991,
+ "execMs": 0.0022499999031424522
},
{
- "name": "list_skills",
+ "name": "regenerate_shot",
"round": 2,
- "atMs": 14415.097124999971,
- "execMs": 0.021542000002227724
- },
- {
- "name": "read_skill",
- "round": 3,
- "atMs": 15988.953458999982,
- "execMs": 0.008917000028304756
- },
- {
- "name": "recut_scene",
- "round": 4,
- "atMs": 23549.67762500001,
- "execMs": 0.02004200010560453
+ "atMs": 18916.16612499999,
+ "execMs": 0.033875000081025064
},
{
- "name": "get_state",
- "round": 5,
- "atMs": 28861.69641700003,
- "execMs": 0.04483300005085766
+ "name": "regenerate_shot",
+ "round": 2,
+ "atMs": 18916.205833000015,
+ "execMs": 0.009042000048793852
},
{
"name": "wait_for_renders",
- "round": 6,
- "atMs": 31089.223499999964,
- "execMs": 0.013959000003524125
- },
- {
- "name": "get_state",
- "round": 7,
- "atMs": 32700.923500000034,
- "execMs": 0.018290999927558005
+ "round": 3,
+ "atMs": 24836.16662500007,
+ "execMs": 0.024167000083252788
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 35521.38591700001,
- "modelMs": 35857.0805009997,
- "toolMs": 0.6645420001586899,
- "totalMs": 35857.74504299986,
+ "ttftMs": 26901.08033300005,
+ "modelMs": 27224.114541999996,
+ "toolMs": 0.22333400009665638,
+ "totalMs": 27224.337876000092,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4873.31479199999,
+ "roundMs": 3702.8952089999802,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.5280000000493601
+ "execMs": 0.1215000000083819
}
]
},
{
- "roundMs": 2745.495416999911,
+ "roundMs": 2966.813458000077,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.008290999918244779
+ "execMs": 0.03249999997206032
},
{
"name": "watch_video",
- "execMs": 0.0006670000730082393
- }
- ]
- },
- {
- "roundMs": 6795.242374999914,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.021542000002227724
- }
- ]
- },
- {
- "roundMs": 1573.795334000024,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.008917000028304756
+ "execMs": 0.0022499999031424522
}
]
},
{
- "roundMs": 7560.687541000079,
+ "roundMs": 12245.090250000008,
"firstDeltaMs": null,
"calls": [
{
- "name": "recut_scene",
- "execMs": 0.02004200010560453
- }
- ]
- },
- {
- "roundMs": 5311.949000000022,
- "firstDeltaMs": null,
- "calls": [
+ "name": "regenerate_shot",
+ "execMs": 0.033875000081025064
+ },
{
- "name": "get_state",
- "execMs": 0.04483300005085766
+ "name": "regenerate_shot",
+ "execMs": 0.009042000048793852
}
]
},
{
- "roundMs": 2227.4263749999227,
+ "roundMs": 5919.880707999924,
"firstDeltaMs": null,
"calls": [
{
"name": "wait_for_renders",
- "execMs": 0.013959000003524125
- }
- ]
- },
- {
- "roundMs": 1611.6412089999067,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.018290999927558005
+ "execMs": 0.024167000083252788
}
]
},
{
- "roundMs": 3157.528457999928,
- "firstDeltaMs": 2820.4183750000084,
+ "roundMs": 2389.434917000006,
+ "firstDeltaMs": 2064.8203330000397,
"calls": []
}
]
}
},
{
- "pass": false,
- "intent": "work",
- "notes": [
- "mutating tools called: subtitles_generate"
- ],
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 4353.067292000051,
- "execMs": 0.06395800004247576
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 8972.265584000037,
- "execMs": 0.04179099993780255
- },
- {
- "name": "watch_video",
- "round": 2,
- "atMs": 15790.264084000024,
- "execMs": 0.0055840000277385116
- },
- {
- "name": "watch_video",
- "round": 2,
- "atMs": 15790.275250000064,
- "execMs": 0.0004999999655410647
- },
- {
- "name": "subtitles_generate",
- "round": 3,
- "atMs": 20983.51129200007,
- "execMs": 0.03370800008997321
+ "atMs": 3631.3538749999134,
+ "execMs": 0.03437499993015081
},
{
"name": "regenerate_shot",
- "round": 4,
- "atMs": 27241.262209000066,
- "execMs": 0.03441600000951439
+ "round": 1,
+ "atMs": 13258.915416999953,
+ "execMs": 0.011624999926425517
},
{
"name": "regenerate_shot",
- "round": 4,
- "atMs": 27241.303709,
- "execMs": 0.00812499993480742
+ "round": 2,
+ "atMs": 15368.927792000002,
+ "execMs": 0.019832999911159277
},
{
"name": "wait_for_renders",
- "round": 5,
- "atMs": 29530.078584000003,
- "execMs": 0.0043750000186264515
+ "round": 3,
+ "atMs": 17457.601874999935,
+ "execMs": 0.004124999977648258
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 32456.772624999983,
- "modelMs": 33101.21412399982,
- "toolMs": 0.19245700002647936,
- "totalMs": 33101.40658099984,
+ "ttftMs": 19115.093249999918,
+ "modelMs": 19819.58349999995,
+ "toolMs": 0.06995799974538386,
+ "totalMs": 19819.653457999695,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4352.853499999968,
+ "roundMs": 3631.0119170000544,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.06395800004247576
- }
- ]
- },
- {
- "roundMs": 4618.959374999977,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.04179099993780255
- }
- ]
- },
- {
- "roundMs": 6817.887332999962,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.0055840000277385116
- },
- {
- "name": "watch_video",
- "execMs": 0.0004999999655410647
+ "execMs": 0.03437499993015081
}
]
},
{
- "roundMs": 5193.193457999965,
+ "roundMs": 9627.457291999948,
"firstDeltaMs": null,
"calls": [
{
- "name": "subtitles_generate",
- "execMs": 0.03370800008997321
+ "name": "regenerate_shot",
+ "execMs": 0.011624999926425517
}
]
},
{
- "roundMs": 6257.665916000027,
+ "roundMs": 2109.957541999989,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.03441600000951439
- },
- {
- "name": "regenerate_shot",
- "execMs": 0.00812499993480742
+ "execMs": 0.019832999911159277
}
]
},
{
- "roundMs": 2288.74524999992,
+ "roundMs": 2088.6204579999903,
"firstDeltaMs": null,
"calls": [
{
"name": "wait_for_renders",
- "execMs": 0.0043750000186264515
+ "execMs": 0.004124999977648258
}
]
},
{
- "roundMs": 3571.9092919999966,
- "firstDeltaMs": 2926.6701250000624,
+ "roundMs": 2362.5362909999676,
+ "firstDeltaMs": 1657.4749580000062,
"calls": []
}
]
@@ -5574,123 +5014,92 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 4892.412250000052,
- "execMs": 0.06125000002793968
+ "atMs": 4389.637083000038,
+ "execMs": 0.04991699999663979
},
{
"name": "watch_video",
"round": 1,
- "atMs": 9880.23345900001,
- "execMs": 0.03754100005608052
+ "atMs": 8277.652165999985,
+ "execMs": 0.010332999983802438
},
{
"name": "watch_video",
"round": 1,
- "atMs": 9880.30700000003,
- "execMs": 0.03762499999720603
+ "atMs": 8277.674583000015,
+ "execMs": 0.0009579999605193734
},
{
"name": "regenerate_shot",
"round": 2,
- "atMs": 17810.912709000055,
- "execMs": 0.05400000000372529
+ "atMs": 15729.088541000034,
+ "execMs": 0.03070800006389618
},
{
"name": "regenerate_shot",
"round": 2,
- "atMs": 17810.97766700003,
- "execMs": 0.010083999950438738
- },
- {
- "name": "wait_for_renders",
- "round": 3,
- "atMs": 22797.392834000057,
- "execMs": 0.004958000034093857
- },
- {
- "name": "get_state",
- "round": 4,
- "atMs": 24480.515459000017,
- "execMs": 0.061750000109896064
+ "atMs": 15729.124915999942,
+ "execMs": 0.00820799998473376
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 26602.13954200002,
- "modelMs": 27158.534665999934,
- "toolMs": 0.2672080001793802,
- "totalMs": 27158.801874000113,
+ "ttftMs": 17492.529457999975,
+ "modelMs": 18354.146666999906,
+ "toolMs": 0.10012399998959154,
+ "totalMs": 18354.246790999896,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4892.027624999988,
+ "roundMs": 4389.195542000001,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.06125000002793968
+ "execMs": 0.04991699999663979
}
]
},
{
- "roundMs": 4987.600957999937,
+ "roundMs": 3887.895041999989,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.03754100005608052
+ "execMs": 0.010332999983802438
},
{
"name": "watch_video",
- "execMs": 0.03762499999720603
+ "execMs": 0.0009579999605193734
}
]
},
{
- "roundMs": 7930.481625000015,
+ "roundMs": 7451.377624999965,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.05400000000372529
+ "execMs": 0.03070800006389618
},
{
"name": "regenerate_shot",
- "execMs": 0.010083999950438738
- }
- ]
- },
- {
- "roundMs": 4986.3432079999475,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "wait_for_renders",
- "execMs": 0.004958000034093857
- }
- ]
- },
- {
- "roundMs": 1683.0593749999534,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.061750000109896064
+ "execMs": 0.00820799998473376
}
]
},
{
- "roundMs": 2679.021875000093,
- "firstDeltaMs": 2121.469041000004,
+ "roundMs": 2625.678457999951,
+ "firstDeltaMs": 1763.3722909999778,
"calls": []
}
]
@@ -5701,75 +5110,75 @@
]
},
{
- "label": "chat=flashLite",
- "chatModel": "gemini-3.1-flash-lite",
+ "label": "all-flash",
+ "chatModel": "gemini-3.5-flash",
"gateModel": "gemini-3.1-flash-lite",
"buckets": {
"chat": {
"cases": 7,
- "passRate": 1,
+ "passRate": 0.95,
"gateMs": {
- "p50": 1110,
- "p95": 1230,
- "mean": 836,
+ "p50": 1217,
+ "p95": 2329,
+ "mean": 1101,
"n": 7
},
"ttftMs": {
- "p50": 2536,
- "p95": 4409,
- "mean": 2602,
+ "p50": 4891,
+ "p95": 6908,
+ "mean": 4666,
"n": 7
},
"totalMs": {
- "p50": 3180,
- "p95": 5089,
- "mean": 3246,
+ "p50": 5478,
+ "p95": 7705,
+ "mean": 5080,
"n": 7
},
"meanRounds": 1
},
"multi-tool": {
"cases": 4,
- "passRate": 0.67,
+ "passRate": 0.92,
"gateMs": {
- "p50": 1170,
- "p95": 1622,
- "mean": 931,
- "n": 3
+ "p50": 0,
+ "p95": 1178,
+ "mean": 584,
+ "n": 4
},
"ttftMs": {
- "p50": 5786,
- "p95": 5969,
- "mean": 5557,
- "n": 3
+ "p50": 14199,
+ "p95": 39798,
+ "mean": 22537,
+ "n": 4
},
"totalMs": {
- "p50": 6088,
- "p95": 6434,
- "mean": 5915,
- "n": 3
+ "p50": 14701,
+ "p95": 40289,
+ "mean": 23066,
+ "n": 4
},
- "meanRounds": 3
+ "meanRounds": 6.5
},
"single-tool": {
"cases": 15,
- "passRate": 0.96,
+ "passRate": 0.98,
"gateMs": {
- "p50": 1170,
- "p95": 1892,
- "mean": 1226,
+ "p50": 1149,
+ "p95": 1950,
+ "mean": 1186,
"n": 15
},
"ttftMs": {
- "p50": 4373,
- "p95": 5038,
- "mean": 4375,
+ "p50": 7591,
+ "p95": 13151,
+ "mean": 7887,
"n": 15
},
"totalMs": {
- "p50": 4661,
- "p95": 5849,
- "mean": 4789,
+ "p50": 7987,
+ "p95": 13527,
+ "mean": 8236,
"n": 15
},
"meanRounds": 2
@@ -5789,21 +5198,21 @@
"n": 3
},
"ttftMs": {
- "p50": 2536,
- "p95": 5880,
- "mean": 3476,
+ "p50": 5992,
+ "p95": 10162,
+ "mean": 6770,
"n": 3
},
"totalMs": {
- "p50": 3214,
- "p95": 6078,
- "mean": 3839,
+ "p50": 6333,
+ "p95": 10332,
+ "mean": 6987,
"n": 3
},
"modelMs": {
- "p50": 3214,
- "p95": 6078,
- "mean": 3839,
+ "p50": 6333,
+ "p95": 10332,
+ "mean": 6987,
"n": 3
},
"toolMs": {
@@ -5820,22 +5229,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5880.14075,
- "modelMs": 6078.341375,
+ "ttftMs": 5991.9733750000005,
+ "modelMs": 6333.405583,
"toolMs": 0,
- "totalMs": 6078.341375,
+ "totalMs": 6333.405583,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6078.341375,
- "firstDeltaMs": 5879.759,
+ "roundMs": 6333.405583,
+ "firstDeltaMs": 5991.59675,
"calls": []
}
]
@@ -5843,22 +5253,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 2011.0292500000005,
- "modelMs": 2225.6733330000006,
+ "ttftMs": 10161.54625,
+ "modelMs": 10331.616584000001,
"toolMs": 0,
- "totalMs": 2225.6733330000006,
+ "totalMs": 10331.616584000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2225.6733330000006,
- "firstDeltaMs": 2010.8525410000002,
+ "roundMs": 10331.616584000001,
+ "firstDeltaMs": 10161.470584,
"calls": []
}
]
@@ -5866,22 +5277,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 2536.1445409999997,
- "modelMs": 3214.2157910000005,
+ "ttftMs": 4156.832166,
+ "modelMs": 4295.417125,
"toolMs": 0,
- "totalMs": 3214.2157910000005,
+ "totalMs": 4295.417125,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3214.2157910000005,
- "firstDeltaMs": 2536.0770410000005,
+ "roundMs": 4295.417125,
+ "firstDeltaMs": 4155.472833,
"calls": []
}
]
@@ -5896,27 +5308,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1226,
- "p95": 1714,
- "mean": 1367,
+ "p50": 1329,
+ "p95": 1889,
+ "mean": 1420,
"n": 3
},
"ttftMs": {
- "p50": 2628,
- "p95": 2907,
- "mean": 2635,
+ "p50": 3028,
+ "p95": 3921,
+ "mean": 3227,
"n": 3
},
"totalMs": {
- "p50": 3419,
- "p95": 3522,
- "mean": 3219,
+ "p50": 3267,
+ "p95": 3957,
+ "mean": 3381,
"n": 3
},
"modelMs": {
- "p50": 1808,
- "p95": 2258,
- "mean": 1852,
+ "p50": 1938,
+ "p95": 2069,
+ "mean": 1961,
"n": 3
},
"toolMs": {
@@ -5934,21 +5346,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1161.3264579999995,
+ "gateMs": 1329.1591249999983,
"gateSkipped": false,
- "ttftMs": 2628.006875000001,
- "modelMs": 2257.8614579999994,
+ "ttftMs": 3027.9948750000003,
+ "modelMs": 1938.2483329999995,
"toolMs": 0,
- "totalMs": 3419.187915999999,
+ "totalMs": 3267.407457999998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2257.8614579999994,
- "firstDeltaMs": 1465.267582999999,
+ "roundMs": 1938.2483329999995,
+ "firstDeltaMs": 1698.4450830000023,
"calls": []
}
]
@@ -5957,21 +5370,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1714.0068330000013,
+ "gateMs": 1042.2432499999995,
"gateSkipped": false,
- "ttftMs": 2906.925792,
- "modelMs": 1807.5048749999987,
+ "ttftMs": 2732.3549160000002,
+ "modelMs": 1875.7684169999993,
"toolMs": 0,
- "totalMs": 3521.511708,
+ "totalMs": 2918.011666999999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1807.5048749999987,
- "firstDeltaMs": 1192.804082999999,
+ "roundMs": 1875.7684169999993,
+ "firstDeltaMs": 1689.9038330000003,
"calls": []
}
]
@@ -5980,21 +5394,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1225.786709,
+ "gateMs": 1888.6148329999996,
"gateSkipped": false,
- "ttftMs": 2368.927917000001,
- "modelMs": 1491.5467500000013,
+ "ttftMs": 3920.9188329999997,
+ "modelMs": 2068.5311660000007,
"toolMs": 0,
- "totalMs": 2717.3334590000013,
+ "totalMs": 3957.1459990000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1491.5467500000013,
- "firstDeltaMs": 1142.9259999999995,
+ "roundMs": 2068.5311660000007,
+ "firstDeltaMs": 2032.035708000003,
"calls": []
}
]
@@ -6009,27 +5424,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1183,
- "p95": 1586,
- "mean": 1273,
+ "p50": 1046,
+ "p95": 2131,
+ "mean": 1386,
"n": 3
},
"ttftMs": {
- "p50": 2634,
- "p95": 2668,
- "mean": 2466,
+ "p50": 2701,
+ "p95": 4269,
+ "mean": 3179,
"n": 3
},
"totalMs": {
- "p50": 2969,
- "p95": 3258,
- "mean": 2910,
+ "p50": 3028,
+ "p95": 4442,
+ "mean": 3394,
"n": 3
},
"modelMs": {
- "p50": 1452,
- "p95": 2076,
- "mean": 1637,
+ "p50": 1982,
+ "p95": 2311,
+ "mean": 2008,
"n": 3
},
"toolMs": {
@@ -6047,21 +5462,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1586.314624999999,
+ "gateMs": 979.9554579999967,
"gateSkipped": false,
- "ttftMs": 2634.1446670000005,
- "modelMs": 1383.0391660000023,
+ "ttftMs": 2566.5399169999982,
+ "modelMs": 1731.659083000006,
"toolMs": 0,
- "totalMs": 2969.3537910000014,
+ "totalMs": 2711.6145410000026,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1383.0391660000023,
- "firstDeltaMs": 1047.4317499999997,
+ "roundMs": 1731.659083000006,
+ "firstDeltaMs": 1586.3014580000017,
"calls": []
}
]
@@ -6070,21 +5486,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1048.8701669999973,
+ "gateMs": 2131.1777500000026,
"gateSkipped": false,
- "ttftMs": 2094.9733750000014,
- "modelMs": 1452.2318330000016,
+ "ttftMs": 4268.614666000001,
+ "modelMs": 2311.1413749999992,
"toolMs": 0,
- "totalMs": 2501.101999999999,
+ "totalMs": 4442.319125000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1452.2318330000016,
- "firstDeltaMs": 1045.9451249999984,
+ "roundMs": 2311.1413749999992,
+ "firstDeltaMs": 2137.2458750000005,
"calls": []
}
]
@@ -6093,21 +5510,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1182.551209000001,
+ "gateMs": 1046.1706669999985,
"gateSkipped": false,
- "ttftMs": 2667.9850420000002,
- "modelMs": 2075.602999999999,
+ "ttftMs": 2700.946707999996,
+ "modelMs": 1981.736499999999,
"toolMs": 0,
- "totalMs": 3258.1542090000003,
+ "totalMs": 3027.9071669999976,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2075.602999999999,
- "firstDeltaMs": 1485.2735840000023,
+ "roundMs": 1981.736499999999,
+ "firstDeltaMs": 1654.603916,
"calls": []
}
]
@@ -6122,86 +5540,87 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1622,
- "p95": 1906,
- "mean": 1610,
+ "p50": 1178,
+ "p95": 1473,
+ "mean": 1224,
"n": 3
},
"ttftMs": {
- "p50": 4915,
- "p95": 6120,
- "mean": 5190,
+ "p50": 8823,
+ "p95": 14377,
+ "mean": 10649,
"n": 3
},
"totalMs": {
- "p50": 5222,
- "p95": 6497,
- "mean": 5486,
+ "p50": 9343,
+ "p95": 14763,
+ "mean": 11105,
"n": 3
},
"modelMs": {
- "p50": 3438,
- "p95": 4875,
- "mean": 3876,
+ "p50": 8188,
+ "p95": 13585,
+ "mean": 9881,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 0,
+ "p95": 1,
"mean": 0,
"n": 3
},
"rounds": {
- "mean": 2.3,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 3099.2119579999962,
- "execMs": 0.2957079999978305
+ "atMs": 7138.9936670000025,
+ "execMs": 1.193041999998968
},
{
"name": "update_cue",
"round": 0,
- "atMs": 3099.517291,
- "execMs": 0.0039590000014868565
+ "atMs": 7140.2172500000015,
+ "execMs": 0.012792000001354609
}
],
"timings": {
- "gateMs": 1300.5645839999997,
+ "gateMs": 1472.701000000001,
"gateSkipped": false,
- "ttftMs": 4535.195833000002,
- "modelMs": 3438.0504999999976,
- "toolMs": 0.29966699999931734,
- "totalMs": 4738.914750999997,
+ "ttftMs": 8748.689874999996,
+ "modelMs": 7869.1473329999935,
+ "toolMs": 1.2058340000003227,
+ "totalMs": 9343.054166999995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1798.0857499999984,
+ "roundMs": 5665.278749999998,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.2957079999978305
+ "execMs": 1.193041999998968
},
{
"name": "update_cue",
- "execMs": 0.0039590000014868565
+ "execMs": 0.012792000001354609
}
]
},
{
- "roundMs": 1639.9647499999992,
- "firstDeltaMs": 1435.646084,
+ "roundMs": 2203.868582999996,
+ "firstDeltaMs": 1608.427499999998,
"calls": []
}
]
@@ -6209,65 +5628,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 3425.902083000001,
- "execMs": 0.01283300000068266
+ "atMs": 7174.046957999999,
+ "execMs": 0.04012499999953434
},
{
"name": "update_cue",
"round": 0,
- "atMs": 3425.9230000000025,
- "execMs": 0.004416999996465165
- },
- {
- "name": "merge_cue",
- "round": 1,
- "atMs": 4970.657082999998,
- "execMs": 0.41566600000078324
+ "atMs": 7174.0951669999995,
+ "execMs": 0.005749999996623956
}
],
"timings": {
- "gateMs": 1621.632458,
+ "gateMs": 1022.4599999999991,
"gateSkipped": false,
- "ttftMs": 6119.872833000001,
- "modelMs": 4875.068582,
- "toolMs": 0.43291599999793107,
- "totalMs": 6497.133955999998,
+ "ttftMs": 8823.202750000004,
+ "modelMs": 8187.908040999995,
+ "toolMs": 0.045874999996158294,
+ "totalMs": 9210.41391599999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1803.7056660000017,
+ "roundMs": 6151.471207999995,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.01283300000068266
+ "execMs": 0.04012499999953434
},
{
"name": "update_cue",
- "execMs": 0.004416999996465165
- }
- ]
- },
- {
- "roundMs": 1544.6617079999996,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "merge_cue",
- "execMs": 0.41566600000078324
+ "execMs": 0.005749999996623956
}
]
},
{
- "roundMs": 1526.7012079999986,
- "firstDeltaMs": 1148.7425409999996,
+ "roundMs": 2036.4368329999998,
+ "firstDeltaMs": 1649.0687079999989,
"calls": []
}
]
@@ -6275,69 +5679,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 3631.3937920000026,
- "execMs": 0.01187499999650754
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 3631.4086670000033,
- "execMs": 0.0018750000017462298
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 3631.4111669999984,
- "execMs": 0.08920800000487361
+ "atMs": 9362.274459000007,
+ "execMs": 0.02425000000221189
},
{
"name": "update_cue",
"round": 0,
- "atMs": 3631.501375,
- "execMs": 0.0014579999988200143
+ "atMs": 9362.306542000006,
+ "execMs": 0.005499999999301508
}
],
"timings": {
- "gateMs": 1906.4029169999994,
+ "gateMs": 1177.6772500000006,
"gateSkipped": false,
- "ttftMs": 4915.123542000001,
- "modelMs": 3315.5366660000072,
- "toolMs": 0.1044160000019474,
- "totalMs": 5222.043999000009,
+ "ttftMs": 14376.592375000007,
+ "modelMs": 13584.920958000002,
+ "toolMs": 0.0297500000015134,
+ "totalMs": 14762.627958000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1724.7902910000048,
+ "roundMs": 8184.408041000002,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.01187499999650754
- },
- {
- "name": "update_cue",
- "execMs": 0.0018750000017462298
- },
- {
- "name": "delete_cue",
- "execMs": 0.08920800000487361
+ "execMs": 0.02425000000221189
},
{
"name": "update_cue",
- "execMs": 0.0014579999988200143
+ "execMs": 0.005499999999301508
}
]
},
{
- "roundMs": 1590.7463750000024,
- "firstDeltaMs": 1283.5993750000052,
+ "roundMs": 5400.512917,
+ "firstDeltaMs": 5014.221917000003,
"calls": []
}
]
@@ -6352,27 +5737,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1110,
- "p95": 2098,
- "mean": 1421,
+ "p50": 1217,
+ "p95": 1248,
+ "mean": 1166,
"n": 3
},
"ttftMs": {
- "p50": 2497,
- "p95": 3405,
- "mean": 2777,
+ "p50": 3622,
+ "p95": 3670,
+ "mean": 3637,
"n": 3
},
"totalMs": {
- "p50": 2933,
- "p95": 3745,
- "mean": 3139,
+ "p50": 3868,
+ "p95": 3937,
+ "mean": 3879,
"n": 3
},
"modelMs": {
- "p50": 1647,
- "p95": 1878,
- "mean": 1718,
+ "p50": 2652,
+ "p95": 2905,
+ "mean": 2713,
"n": 3
},
"toolMs": {
@@ -6389,22 +5774,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2098.163332999997,
+ "gateMs": 1216.5457499999902,
"gateSkipped": false,
- "ttftMs": 3404.9016669999983,
- "modelMs": 1646.5347500000062,
+ "ttftMs": 3617.6995830000087,
+ "modelMs": 2651.863499999992,
"toolMs": 0,
- "totalMs": 3744.698083000003,
+ "totalMs": 3868.4092499999824,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1646.5347500000062,
- "firstDeltaMs": 1306.3443750000006,
+ "roundMs": 2651.863499999992,
+ "firstDeltaMs": 2400.8226659999928,
"calls": []
}
]
@@ -6412,22 +5798,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1110.3014580000017,
+ "gateMs": 1248.091790999999,
"gateSkipped": false,
- "ttftMs": 2427.474875,
- "modelMs": 1629.418500000007,
+ "ttftMs": 3622.337667,
+ "modelMs": 2583.6929999999993,
"toolMs": 0,
- "totalMs": 2739.719958000009,
+ "totalMs": 3831.784790999998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1629.418500000007,
- "firstDeltaMs": 1316.8833330000052,
+ "roundMs": 2583.6929999999993,
+ "firstDeltaMs": 2374.04075,
"calls": []
}
]
@@ -6435,22 +5822,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1054.6625419999982,
+ "gateMs": 1032.0372910000006,
"gateSkipped": false,
- "ttftMs": 2497.4147500000036,
- "modelMs": 1877.9937910000008,
+ "ttftMs": 3670.063458000004,
+ "modelMs": 2904.6831669999956,
"toolMs": 0,
- "totalMs": 2932.656332999999,
+ "totalMs": 3936.720457999996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1877.9937910000008,
- "firstDeltaMs": 1442.6726250000065,
+ "roundMs": 2904.6831669999956,
+ "firstDeltaMs": 2637.7626670000027,
"calls": []
}
]
@@ -6461,405 +5849,305 @@
{
"name": "photos-request-places-clips",
"bucket": "multi-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 14199,
+ "p95": 28865,
+ "mean": 17998,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 14701,
+ "p95": 29379,
+ "mean": 18472,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 14700,
+ "p95": 29379,
+ "mean": 18472,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 4.3,
+ "max": 6
+ }
+ },
"runs": [
{
- "pass": false,
- "intent": "work",
- "notes": [
- "mutating tools called: place_clip, place_clip"
- ],
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 1937.0442910000056,
- "execMs": 0.006458000003476627
+ "atMs": 5276.944791000002,
+ "execMs": 0.004709000000730157
},
{
"name": "add_clip",
"round": 0,
- "atMs": 1937.098291000002,
- "execMs": 0.011417000001529232
+ "atMs": 5276.970165999999,
+ "execMs": 0.0090829999971902
},
{
- "name": "add_clip",
+ "name": "get_state",
"round": 1,
- "atMs": 3678.685000000005,
- "execMs": 0.0025409999943803996
- },
- {
- "name": "place_clip",
- "round": 2,
- "atMs": 5606.275750000001,
- "execMs": 0.02279199999611592
- },
- {
- "name": "place_clip",
- "round": 2,
- "atMs": 5606.301416000002,
- "execMs": 0.008000000001629815
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 10549.785125000002,
- "execMs": 0.00408400000014808
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 10549.80225,
- "execMs": 0.0003330000035930425
+ "atMs": 7586.105874999994,
+ "execMs": 0.18283299999893643
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 11630.630500000007,
- "modelMs": 12087.286416999996,
- "toolMs": 0.055625000000873115,
- "totalMs": 12087.342041999997,
+ "ttftMs": 10930.713166000001,
+ "modelMs": 11336.177124999987,
+ "toolMs": 0.1966249999968568,
+ "totalMs": 11336.373749999984,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1936.362792,
+ "roundMs": 5276.505374999993,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.006458000003476627
+ "execMs": 0.004709000000730157
},
{
"name": "add_clip",
- "execMs": 0.011417000001529232
- }
- ]
- },
- {
- "roundMs": 1741.4594170000055,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0025409999943803996
- }
- ]
- },
- {
- "roundMs": 1927.512499999997,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "place_clip",
- "execMs": 0.02279199999611592
- },
- {
- "name": "place_clip",
- "execMs": 0.008000000001629815
+ "execMs": 0.0090829999971902
}
]
},
{
- "roundMs": 4943.112083,
+ "roundMs": 2309.0523329999996,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.00408400000014808
- },
- {
- "name": "add_clip",
- "execMs": 0.0003330000035930425
+ "name": "get_state",
+ "execMs": 0.18283299999893643
}
]
},
{
- "roundMs": 1538.8396249999932,
- "firstDeltaMs": 1080.7953750000015,
+ "roundMs": 3750.6194169999944,
+ "firstDeltaMs": 3344.3963749999966,
"calls": []
}
]
}
},
{
- "pass": false,
- "intent": "work",
- "notes": [
- "mutating tools called: place_clip"
- ],
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 1537.3374170000025,
- "execMs": 0.002166999998735264
+ "atMs": 6341.121375000002,
+ "execMs": 0.0025000000023283064
},
{
"name": "add_clip",
"round": 0,
- "atMs": 1537.3480839999975,
- "execMs": 0.0001250000059371814
+ "atMs": 6341.135167,
+ "execMs": 0.004708000007667579
},
{
- "name": "add_clip",
+ "name": "get_state",
"round": 1,
- "atMs": 3146.4881250000035,
- "execMs": 0.0027079999999841675
+ "atMs": 8561.080333000005,
+ "execMs": 0.02483300000312738
},
{
- "name": "place_clip",
+ "name": "get_state",
"round": 2,
- "atMs": 5406.725625000006,
- "execMs": 0.007291000001714565
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 6899.150374999997,
- "execMs": 0.0032910000008996576
- },
- {
- "name": "add_clip",
- "round": 4,
- "atMs": 8876.432291999998,
- "execMs": 0.000750000006519258
- },
- {
- "name": "add_clip",
- "round": 5,
- "atMs": 10330.106249999997,
- "execMs": 0.0032499999942956492
- },
- {
- "name": "add_clip",
- "round": 6,
- "atMs": 11879.273291999998,
- "execMs": 0.0032500000088475645
- },
- {
- "name": "add_clip",
- "round": 7,
- "atMs": 14651.747417000006,
- "execMs": 0.003416000006836839
- },
- {
- "name": "add_clip",
- "round": 8,
- "atMs": 16398.580667000002,
- "execMs": 0.002541000008932315
+ "atMs": 11072.12208300001,
+ "execMs": 0.03999999999359716
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 17418.026834000004,
- "modelMs": 17908.517836,
- "toolMs": 0.02878900003270246,
- "totalMs": 17908.54662500003,
+ "ttftMs": 14199.071958,
+ "modelMs": 14700.44879200001,
+ "toolMs": 0.07204100000672042,
+ "totalMs": 14700.520833000017,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1537.1121249999997,
+ "roundMs": 6340.916916999995,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.002166999998735264
+ "execMs": 0.0025000000023283064
},
{
"name": "add_clip",
- "execMs": 0.0001250000059371814
- }
- ]
- },
- {
- "roundMs": 1609.103209000008,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0027079999999841675
+ "execMs": 0.004708000007667579
}
]
},
{
- "roundMs": 2260.194042000003,
+ "roundMs": 2219.825125000003,
"firstDeltaMs": null,
"calls": [
{
- "name": "place_clip",
- "execMs": 0.007291000001714565
+ "name": "get_state",
+ "execMs": 0.02483300000312738
}
]
},
{
- "roundMs": 1492.372875000001,
+ "roundMs": 2510.8880000000063,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0032910000008996576
+ "name": "get_state",
+ "execMs": 0.03999999999359716
}
]
},
{
- "roundMs": 1977.2398339999927,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.000750000006519258
- }
- ]
- },
- {
- "roundMs": 1453.6306249999907,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0032499999942956492
- }
- ]
- },
- {
- "roundMs": 1549.1024170000019,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0032500000088475645
- }
- ]
- },
- {
- "roundMs": 2772.3192500000005,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.003416000006836839
- }
- ]
- },
- {
- "roundMs": 1746.7614590000012,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.002541000008932315
- }
- ]
- },
- {
- "roundMs": 1510.6820000000007,
- "firstDeltaMs": 1019.414042000004,
+ "roundMs": 3628.818750000006,
+ "firstDeltaMs": 3126.8655,
"calls": []
}
]
}
},
{
- "pass": false,
- "intent": "work",
- "notes": [
- "mutating tools called: place_clip"
- ],
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 2167.3225419999944,
- "execMs": 0.0011249999952269718
+ "atMs": 7667.983750000014,
+ "execMs": 0.0015839999978197739
},
{
"name": "add_clip",
"round": 0,
- "atMs": 2167.326457999996,
- "execMs": 8.299999171867967e-05
+ "atMs": 7667.991958000013,
+ "execMs": 0.00012499999138526618
},
{
- "name": "add_clip",
+ "name": "get_state",
"round": 1,
- "atMs": 3628.108374999996,
- "execMs": 0.0021249999990686774
+ "atMs": 9988.64170800001,
+ "execMs": 0.01366700000653509
},
{
- "name": "place_clip",
+ "name": "get_state",
"round": 2,
- "atMs": 5130.456499999986,
- "execMs": 0.00812499999301508
+ "atMs": 12790.767667000007,
+ "execMs": 0.055082999999285676
},
{
- "name": "add_clip",
+ "name": "set_transition",
"round": 3,
- "atMs": 7337.179124999995,
- "execMs": 0.0012080000014975667
+ "atMs": 20660.420750000005,
+ "execMs": 0.023624999972525984
+ },
+ {
+ "name": "get_state",
+ "round": 4,
+ "atMs": 26109.872667000003,
+ "execMs": 0.015625
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 8695.266707999996,
- "modelMs": 9507.452207999988,
- "toolMs": 0.012665999980526976,
- "totalMs": 9507.464873999968,
+ "ttftMs": 28864.750707999992,
+ "modelMs": 29378.664374999993,
+ "toolMs": 0.10970899996755179,
+ "totalMs": 29378.77408399996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2167.108500000002,
+ "roundMs": 7667.629624999987,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.0011249999952269718
+ "execMs": 0.0015839999978197739
},
{
"name": "add_clip",
- "execMs": 8.299999171867967e-05
+ "execMs": 0.00012499999138526618
}
]
},
{
- "roundMs": 1460.7554999999993,
+ "roundMs": 2320.6187500000087,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0021249999990686774
+ "name": "get_state",
+ "execMs": 0.01366700000653509
}
]
},
{
- "roundMs": 1502.274166999996,
+ "roundMs": 2802.051707999999,
"firstDeltaMs": null,
"calls": [
{
- "name": "place_clip",
- "execMs": 0.00812499999301508
+ "name": "get_state",
+ "execMs": 0.055082999999285676
}
]
},
{
- "roundMs": 2206.6704579999932,
+ "roundMs": 7869.523124999992,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0012080000014975667
+ "name": "set_transition",
+ "execMs": 0.023624999972525984
+ }
+ ]
+ },
+ {
+ "roundMs": 5449.359874999995,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.015625
}
]
},
{
- "roundMs": 2170.6435829999973,
- "firstDeltaMs": 1358.0732500000013,
+ "roundMs": 3269.4812920000113,
+ "firstDeltaMs": 2754.814250000025,
"calls": []
}
]
@@ -6880,21 +6168,21 @@
"n": 3
},
"ttftMs": {
- "p50": 1165,
- "p95": 1194,
- "mean": 1171,
+ "p50": 4891,
+ "p95": 6486,
+ "mean": 5393,
"n": 3
},
"totalMs": {
- "p50": 1917,
- "p95": 1982,
- "mean": 1936,
+ "p50": 5478,
+ "p95": 6772,
+ "mean": 5902,
"n": 3
},
"modelMs": {
- "p50": 1917,
- "p95": 1982,
- "mean": 1936,
+ "p50": 5478,
+ "p95": 6772,
+ "mean": 5902,
"n": 3
},
"toolMs": {
@@ -6911,22 +6199,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 1155.4631660000014,
- "modelMs": 1907.7856670000037,
+ "ttftMs": 4890.5401249999995,
+ "modelMs": 5455.5874579999945,
"toolMs": 0,
- "totalMs": 1907.7856670000037,
+ "totalMs": 5455.5874579999945,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1907.7856670000037,
- "firstDeltaMs": 1155.3236670000042,
+ "roundMs": 5455.5874579999945,
+ "firstDeltaMs": 4890.050458000012,
"calls": []
}
]
@@ -6934,22 +6223,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 1165.4279580000002,
- "modelMs": 1982.4587909999973,
+ "ttftMs": 6485.54233299999,
+ "modelMs": 6771.7124579999945,
"toolMs": 0,
- "totalMs": 1982.4587909999973,
+ "totalMs": 6771.7124579999945,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1982.4587909999973,
- "firstDeltaMs": 1165.349166,
+ "roundMs": 6771.7124579999945,
+ "firstDeltaMs": 6485.447665999993,
"calls": []
}
]
@@ -6957,22 +6247,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 1193.5759160000016,
- "modelMs": 1916.9366670000018,
+ "ttftMs": 4802.589250000019,
+ "modelMs": 5478.250042,
"toolMs": 0,
- "totalMs": 1916.9366670000018,
+ "totalMs": 5478.250042,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1916.9366670000018,
- "firstDeltaMs": 1193.1282079999946,
+ "roundMs": 5478.250042,
+ "firstDeltaMs": 4802.407792000013,
"calls": []
}
]
@@ -6987,27 +6278,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1100,
- "p95": 1162,
- "mean": 1098,
+ "p50": 2329,
+ "p95": 2522,
+ "mean": 1955,
"n": 3
},
"ttftMs": {
- "p50": 2343,
- "p95": 2435,
- "mean": 2371,
+ "p50": 5517,
+ "p95": 6500,
+ "mean": 5441,
"n": 3
},
"totalMs": {
- "p50": 3180,
- "p95": 3289,
- "mean": 3087,
+ "p50": 5879,
+ "p95": 6912,
+ "mean": 5860,
"n": 3
},
"modelMs": {
- "p50": 2080,
- "p95": 2257,
- "mean": 1989,
+ "p50": 3775,
+ "p95": 4390,
+ "mean": 3905,
"n": 3
},
"toolMs": {
@@ -7024,22 +6315,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "chat",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1032.8084999999992,
+ "gateMs": 2329.3414589999884,
"gateSkipped": false,
- "ttftMs": 2434.9031669999968,
- "modelMs": 2256.6630000000005,
+ "ttftMs": 5516.768458999984,
+ "modelMs": 3549.7721249999886,
"toolMs": 0,
- "totalMs": 3289.4714999999997,
+ "totalMs": 5879.113583999977,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2256.6630000000005,
- "firstDeltaMs": 1401.9102080000011,
+ "roundMs": 3549.7721249999886,
+ "firstDeltaMs": 3187.2667499999807,
"calls": []
}
]
@@ -7047,22 +6339,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1099.893292000008,
+ "gateMs": 2522.1287500000035,
"gateSkipped": false,
- "ttftMs": 2343.4879169999913,
- "modelMs": 2080.0189170000085,
+ "ttftMs": 6499.913375000004,
+ "modelMs": 4390.280417000002,
"toolMs": 0,
- "totalMs": 3179.9122090000164,
+ "totalMs": 6912.409167000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2080.0189170000085,
- "firstDeltaMs": 1243.4800420000101,
+ "roundMs": 4390.280417000002,
+ "firstDeltaMs": 3977.662209000002,
"calls": []
}
]
@@ -7070,22 +6363,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "chat",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1161.823292000001,
+ "gateMs": 1013.7740829999966,
"gateSkipped": false,
- "ttftMs": 2335.9533749999973,
- "modelMs": 1629.4132499999978,
+ "ttftMs": 4306.009667000006,
+ "modelMs": 3774.7853340000147,
"toolMs": 0,
- "totalMs": 2791.2365419999987,
+ "totalMs": 4788.559417000011,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1629.4132499999978,
- "firstDeltaMs": 1173.977165999997,
+ "roundMs": 3774.7853340000147,
+ "firstDeltaMs": 3291.456292000017,
"calls": []
}
]
@@ -7100,27 +6394,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1449,
- "p95": 1987,
- "mean": 1524,
+ "p50": 1017,
+ "p95": 1057,
+ "mean": 1016,
"n": 3
},
"ttftMs": {
- "p50": 5038,
- "p95": 7188,
- "mean": 5753,
+ "p50": 8714,
+ "p95": 10633,
+ "mean": 8893,
"n": 3
},
"totalMs": {
- "p50": 5849,
- "p95": 8207,
- "mean": 6482,
+ "p50": 9360,
+ "p95": 11159,
+ "mean": 9392,
"n": 3
},
"modelMs": {
- "p50": 4251,
- "p95": 6759,
- "mean": 4957,
+ "p50": 8343,
+ "p95": 10102,
+ "mean": 8376,
"n": 3
},
"toolMs": {
@@ -7137,39 +6431,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 3849.7035000000033,
- "execMs": 0.015249999996740371
+ "atMs": 5586.288125000021,
+ "execMs": 0.0040000000153668225
}
],
"timings": {
- "gateMs": 1136.6855839999916,
+ "gateMs": 974.565499999997,
"gateSkipped": false,
- "ttftMs": 5033.551749999999,
- "modelMs": 4251.434333000012,
- "toolMs": 0.015249999996740371,
- "totalMs": 5388.135167,
+ "ttftMs": 7333.078458000004,
+ "modelMs": 6683.336750000017,
+ "toolMs": 0.0040000000153668225,
+ "totalMs": 7657.906250000029,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2712.3975000000064,
+ "roundMs": 4611.345957999991,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.015249999996740371
+ "execMs": 0.0040000000153668225
}
]
},
{
- "roundMs": 1539.0368330000056,
- "firstDeltaMs": 1183.7791670000006,
+ "roundMs": 2071.990792000026,
+ "firstDeltaMs": 1746.7592500000028,
"calls": []
}
]
@@ -7177,39 +6472,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 4019.459707999995,
- "execMs": 0.06441700000141282
+ "atMs": 6807.225832999975,
+ "execMs": 0.011415999993914738
}
],
"timings": {
- "gateMs": 1987.468417000011,
+ "gateMs": 1016.870957999985,
"gateSkipped": false,
- "ttftMs": 5037.996541,
- "modelMs": 3861.92416699999,
- "toolMs": 0.06441700000141282,
- "totalMs": 5849.4570010000025,
+ "ttftMs": 8714.306540999998,
+ "modelMs": 8343.465082999988,
+ "toolMs": 0.011415999993914738,
+ "totalMs": 9360.347456999967,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2031.6520420000015,
+ "roundMs": 5790.108999999997,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.06441700000141282
+ "execMs": 0.011415999993914738
}
]
},
{
- "roundMs": 1830.2721249999886,
- "firstDeltaMs": 1018.3817079999862,
+ "roundMs": 2553.3560829999915,
+ "firstDeltaMs": 1907.0406249999942,
"calls": []
}
]
@@ -7217,39 +6513,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 5369.4645420000015,
- "execMs": 0.009915999995428137
+ "atMs": 8784.241916999978,
+ "execMs": 0.009292000002460554
}
],
"timings": {
- "gateMs": 1448.6627919999883,
+ "gateMs": 1056.5725419999799,
"gateSkipped": false,
- "ttftMs": 7187.730041999996,
- "modelMs": 6758.524166999996,
- "toolMs": 0.009915999995428137,
- "totalMs": 8207.19687499998,
+ "ttftMs": 10632.893332999985,
+ "modelMs": 10101.949999999983,
+ "toolMs": 0.009292000002460554,
+ "totalMs": 11158.531833999965,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3920.5230420000007,
+ "roundMs": 7727.412749999989,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.009915999995428137
+ "execMs": 0.009292000002460554
}
]
},
{
- "roundMs": 2838.001124999995,
- "firstDeltaMs": 1818.2160409999924,
+ "roundMs": 2374.537249999994,
+ "firstDeltaMs": 1848.598874999996,
"calls": []
}
]
@@ -7264,27 +6561,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1175,
- "p95": 1344,
- "mean": 1205,
+ "p50": 1149,
+ "p95": 1748,
+ "mean": 1320,
"n": 3
},
"ttftMs": {
- "p50": 4456,
- "p95": 4873,
- "mean": 4472,
+ "p50": 8001,
+ "p95": 9507,
+ "mean": 8405,
"n": 3
},
"totalMs": {
- "p50": 4701,
- "p95": 5269,
- "mean": 4816,
+ "p50": 8547,
+ "p95": 9807,
+ "mean": 8775,
"n": 3
},
"modelMs": {
- "p50": 3604,
- "p95": 3925,
- "mean": 3611,
+ "p50": 6821,
+ "p95": 8746,
+ "mean": 7455,
"n": 3
},
"toolMs": {
@@ -7301,39 +6598,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 3560.2812080000003,
- "execMs": 0.10208400001283735
+ "atMs": 7378.335833999998,
+ "execMs": 0.2978749999892898
}
],
"timings": {
- "gateMs": 1344.0571659999987,
+ "gateMs": 1061.4239170000073,
"gateSkipped": false,
- "ttftMs": 4872.838208000001,
- "modelMs": 3924.575332999986,
- "toolMs": 0.10208400001283735,
- "totalMs": 5268.734582999998,
+ "ttftMs": 9506.901249999995,
+ "modelMs": 8745.54020800002,
+ "toolMs": 0.2978749999892898,
+ "totalMs": 9807.262000000017,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2215.775957999984,
+ "roundMs": 6316.168708000012,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.10208400001283735
+ "execMs": 0.2978749999892898
}
]
},
{
- "roundMs": 1708.7993750000023,
- "firstDeltaMs": 1312.440333000006,
+ "roundMs": 2429.3715000000084,
+ "firstDeltaMs": 2128.2252499999886,
"calls": []
}
]
@@ -7341,39 +6639,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 3272.775416999997,
- "execMs": 0.01462499998160638
+ "atMs": 6290.119082999998,
+ "execMs": 0.009209000010741875
}
],
"timings": {
- "gateMs": 1096.5086660000088,
+ "gateMs": 1748.481625000015,
"gateSkipped": false,
- "ttftMs": 4456.354416999995,
- "modelMs": 3604.0923739999707,
- "toolMs": 0.01462499998160638,
- "totalMs": 4700.615664999961,
+ "ttftMs": 8000.507207999995,
+ "modelMs": 6798.743791000015,
+ "toolMs": 0.009209000010741875,
+ "totalMs": 8547.234625000041,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2176.1265829999757,
+ "roundMs": 4541.458416000009,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.01462499998160638
+ "execMs": 0.009209000010741875
}
]
},
{
- "roundMs": 1427.965790999995,
- "firstDeltaMs": 1183.53233300001,
+ "roundMs": 2257.2853750000068,
+ "firstDeltaMs": 1710.3476669999945,
"calls": []
}
]
@@ -7381,39 +6680,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 2780.4420830000017,
- "execMs": 0.027083999972091988
+ "atMs": 6030.996291999996,
+ "execMs": 0.010500000003958121
}
],
"timings": {
- "gateMs": 1175.1687079999829,
+ "gateMs": 1148.6130420000118,
"gateSkipped": false,
- "ttftMs": 4086.6814160000067,
- "modelMs": 3304.642125000013,
- "toolMs": 0.027083999972091988,
- "totalMs": 4479.837916999968,
+ "ttftMs": 7706.307166999992,
+ "modelMs": 6821.260333000013,
+ "toolMs": 0.010500000003958121,
+ "totalMs": 7969.883875000029,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1605.056666999997,
+ "roundMs": 4882.260665999987,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.027083999972091988
+ "execMs": 0.010500000003958121
}
]
},
{
- "roundMs": 1699.585458000016,
- "firstDeltaMs": 1306.1773749999993,
+ "roundMs": 1938.9996670000255,
+ "firstDeltaMs": 1675.2854170000064,
"calls": []
}
]
@@ -7424,38 +6724,38 @@
{
"name": "mute-all-does-not-redo-prior-turn",
"bucket": "single-tool",
- "passRate": 0.3333333333333333,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1377,
- "p95": 1377,
- "mean": 1377,
- "n": 1
+ "p50": 1173,
+ "p95": 1723,
+ "mean": 1350,
+ "n": 3
},
"ttftMs": {
- "p50": 4359,
- "p95": 4359,
- "mean": 4359,
- "n": 1
+ "p50": 8082,
+ "p95": 8171,
+ "mean": 7483,
+ "n": 3
},
"totalMs": {
- "p50": 4536,
- "p95": 4536,
- "mean": 4536,
- "n": 1
+ "p50": 8110,
+ "p95": 8288,
+ "mean": 7600,
+ "n": 3
},
"modelMs": {
- "p50": 3158,
- "p95": 3158,
- "mean": 3158,
- "n": 1
+ "p50": 6565,
+ "p95": 6957,
+ "mean": 6250,
+ "n": 3
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 1
+ "n": 3
},
"rounds": {
"mean": 2,
@@ -7465,183 +6765,142 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 2891.8100829999894,
- "execMs": 0.402708999987226
+ "atMs": 6312.219624999998,
+ "execMs": 0.25616699998499826
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 2892.247457999998,
- "execMs": 0.003792000003159046
+ "atMs": 6312.504667000001,
+ "execMs": 0.0033750000002328306
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 2892.2609159999993,
- "execMs": 0.002082999999402091
+ "atMs": 6312.516166999994,
+ "execMs": 0.001707999996142462
}
],
"timings": {
- "gateMs": 1377.0034589999996,
+ "gateMs": 1722.949042000022,
"gateSkipped": false,
- "ttftMs": 4358.628957999987,
- "modelMs": 3158.351958000014,
- "toolMs": 0.4085839999897871,
- "totalMs": 4535.764001000003,
+ "ttftMs": 8170.9800000000105,
+ "modelMs": 6564.542915999977,
+ "toolMs": 0.26124999998137355,
+ "totalMs": 8287.75320799998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1513.2488330000197,
+ "roundMs": 4588.597582999995,
"firstDeltaMs": null,
"calls": [
{
"name": "set_clip_muted",
- "execMs": 0.402708999987226
+ "execMs": 0.25616699998499826
},
{
"name": "set_clip_muted",
- "execMs": 0.003792000003159046
+ "execMs": 0.0033750000002328306
},
{
"name": "set_clip_muted",
- "execMs": 0.002082999999402091
+ "execMs": 0.001707999996142462
}
]
},
{
- "roundMs": 1645.1031249999942,
- "firstDeltaMs": 1466.335791999998,
+ "roundMs": 1975.9453329999815,
+ "firstDeltaMs": 1858.4344580000034,
"calls": []
}
]
}
},
{
- "pass": false,
- "intent": "work",
- "notes": [
- "reply did not match /mut/i"
- ],
+ "pass": true,
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [],
"trace": [
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 3292.089875000005,
- "execMs": 0.02166699999361299
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 3292.119834000012,
- "execMs": 0.0020000000076834112
- },
- {
- "name": "set_clip_muted",
+ "name": "set_track_muted",
"round": 0,
- "atMs": 3292.123917000019,
- "execMs": 0.001041000010445714
+ "atMs": 5077.221250000002,
+ "execMs": 0.038167000020621344
}
],
"timings": {
- "gateMs": 1934.3151659999858,
+ "gateMs": 1153.36729200001,
"gateSkipped": false,
- "ttftMs": 4500.599917000014,
- "modelMs": 2835.9029580000206,
- "toolMs": 0.024708000011742115,
- "totalMs": 4770.242832000018,
+ "ttftMs": 8081.783291,
+ "modelMs": 6956.816291999974,
+ "toolMs": 0.038167000020621344,
+ "totalMs": 8110.221751000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1357.6407920000202,
+ "roundMs": 3923.67508299998,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.02166699999361299
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0020000000076834112
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.001041000010445714
+ "name": "set_track_muted",
+ "execMs": 0.038167000020621344
}
]
},
{
- "roundMs": 1478.2621660000004,
- "firstDeltaMs": 1208.4541249999893,
+ "roundMs": 3033.141208999994,
+ "firstDeltaMs": 3004.4919590000063,
"calls": []
}
]
}
},
{
- "pass": false,
- "intent": "work",
- "notes": [
- "reply did not match /mut/i"
- ],
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [],
"trace": [
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 3169.725624999992,
- "execMs": 0.0073339999944437295
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 3169.735833999992,
- "execMs": 0.0003339999821037054
- },
- {
- "name": "set_clip_muted",
+ "name": "set_track_muted",
"round": 0,
- "atMs": 3169.7365419999987,
- "execMs": 0.0001660000125411898
+ "atMs": 4401.3262920000125,
+ "execMs": 0.02350000001024455
}
],
"timings": {
- "gateMs": 1908.7791670000006,
+ "gateMs": 1173.1455000000133,
"gateSkipped": false,
- "ttftMs": 4267.150416999997,
- "modelMs": 2548.0939589999907,
- "toolMs": 0.007833999989088625,
- "totalMs": 4456.88095999998,
+ "ttftMs": 6197.502875000006,
+ "modelMs": 5227.473583000014,
+ "toolMs": 0.02350000001024455,
+ "totalMs": 6400.642583000037,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1260.7577089999977,
+ "roundMs": 3228.0440830000152,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.0073339999944437295
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0003339999821037054
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0001660000125411898
+ "name": "set_track_muted",
+ "execMs": 0.02350000001024455
}
]
},
{
- "roundMs": 1287.336249999993,
- "firstDeltaMs": 1097.4005830000096,
+ "roundMs": 1999.4294999999984,
+ "firstDeltaMs": 1796.1121250000142,
"calls": []
}
]
@@ -7656,27 +6915,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1137,
- "p95": 1218,
- "mean": 1085,
+ "p50": 1331,
+ "p95": 1904,
+ "mean": 1455,
"n": 3
},
"ttftMs": {
- "p50": 4748,
- "p95": 4790,
- "mean": 4398,
+ "p50": 9954,
+ "p95": 11384,
+ "mean": 9741,
"n": 3
},
"totalMs": {
- "p50": 5032,
- "p95": 5319,
- "mean": 4774,
+ "p50": 10462,
+ "p95": 11728,
+ "mean": 10190,
"n": 3
},
"modelMs": {
- "p50": 3814,
- "p95": 4182,
- "mean": 3689,
+ "p50": 8558,
+ "p95": 10397,
+ "mean": 8735,
"n": 3
},
"toolMs": {
@@ -7693,39 +6952,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_image",
"round": 0,
- "atMs": 3493.2881669999915,
- "execMs": 0.22716599999694154
+ "atMs": 9578.91958300001,
+ "execMs": 0.11616599996341392
}
],
"timings": {
- "gateMs": 1217.8060839999816,
+ "gateMs": 1330.6315420000174,
"gateSkipped": false,
- "ttftMs": 4747.663334000012,
- "modelMs": 3814.1177509999543,
- "toolMs": 0.22716599999694154,
- "totalMs": 5032.151000999933,
+ "ttftMs": 11384.482000000018,
+ "modelMs": 10397.053875000012,
+ "toolMs": 0.11616599996341392,
+ "totalMs": 11727.801582999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2274.70091699998,
+ "roundMs": 8247.824083999993,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.22716599999694154
+ "execMs": 0.11616599996341392
}
]
},
{
- "roundMs": 1539.4168339999742,
- "firstDeltaMs": 1254.125166999991,
+ "roundMs": 2149.2297910000198,
+ "firstDeltaMs": 1805.4318750000093,
"calls": []
}
]
@@ -7733,39 +6993,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_image",
"round": 0,
- "atMs": 2531.4118749999907,
- "execMs": 0.0498749999969732
+ "atMs": 8062.725124999997,
+ "execMs": 0.040541999973356724
}
],
"timings": {
- "gateMs": 898.8421250000247,
+ "gateMs": 1903.8492500000284,
"gateSkipped": false,
- "ttftMs": 3655.708875000011,
- "modelMs": 3071.427500999969,
- "toolMs": 0.0498749999969732,
- "totalMs": 3970.319500999991,
+ "ttftMs": 9954.027499999967,
+ "modelMs": 8558.107334,
+ "toolMs": 0.040541999973356724,
+ "totalMs": 10461.997126000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1632.3837919999787,
+ "roundMs": 6158.77679199999,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.0498749999969732
+ "execMs": 0.040541999973356724
}
]
},
{
- "roundMs": 1439.0437089999905,
- "firstDeltaMs": 1124.2051250000077,
+ "roundMs": 2399.3305420000106,
+ "firstDeltaMs": 1891.2230839999975,
"calls": []
}
]
@@ -7773,39 +7034,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_image",
"round": 0,
- "atMs": 3569.633749999979,
- "execMs": 0.01837499998509884
+ "atMs": 6439.723583999963,
+ "execMs": 0.060166999988723546
}
],
"timings": {
- "gateMs": 1137.0589170000167,
+ "gateMs": 1129.077958000009,
"gateSkipped": false,
- "ttftMs": 4790.20979199998,
- "modelMs": 4182.271126000007,
- "toolMs": 0.01837499998509884,
- "totalMs": 5319.348418000009,
+ "ttftMs": 7884.625333999982,
+ "modelMs": 7250.096458999964,
+ "toolMs": 0.060166999988723546,
+ "totalMs": 8379.234583999962,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2432.3090840000077,
+ "roundMs": 5310.360875000013,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.01837499998509884
+ "execMs": 0.060166999988723546
}
]
},
{
- "roundMs": 1749.9620419999992,
- "firstDeltaMs": 1220.5286250000063,
+ "roundMs": 1939.735583999951,
+ "firstDeltaMs": 1444.7903339999611,
"calls": []
}
]
@@ -7820,27 +7082,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1878,
- "p95": 2006,
- "mean": 1757,
+ "p50": 1137,
+ "p95": 1137,
+ "mean": 1100,
"n": 3
},
"ttftMs": {
- "p50": 4947,
- "p95": 7455,
- "mean": 5636,
+ "p50": 7732,
+ "p95": 8287,
+ "mean": 7824,
"n": 3
},
"totalMs": {
- "p50": 5638,
- "p95": 7827,
- "mean": 6115,
+ "p50": 7987,
+ "p95": 8649,
+ "mean": 8158,
"n": 3
},
"modelMs": {
- "p50": 3632,
- "p95": 5949,
- "mean": 4358,
+ "p50": 6850,
+ "p95": 7512,
+ "mean": 7059,
"n": 3
},
"toolMs": {
@@ -7857,39 +7119,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 3569.5522920000076,
- "execMs": 0.0301669999898877
+ "atMs": 5601.715749999974,
+ "execMs": 0.02245799999218434
}
],
"timings": {
- "gateMs": 2005.8029169999936,
+ "gateMs": 1024.7750419999938,
"gateSkipped": false,
- "ttftMs": 4947.399208000017,
- "modelMs": 3631.9481249999953,
- "toolMs": 0.0301669999898877,
- "totalMs": 5637.781208999979,
+ "ttftMs": 7452.463624999975,
+ "modelMs": 6814.24637399998,
+ "toolMs": 0.02245799999218434,
+ "totalMs": 7839.043873999966,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1563.3139160000137,
+ "roundMs": 4576.52658299997,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.0301669999898877
+ "execMs": 0.02245799999218434
}
]
},
{
- "roundMs": 2068.6342089999816,
- "firstDeltaMs": 1377.7751669999852,
+ "roundMs": 2237.7197910000104,
+ "firstDeltaMs": 1850.7066249999916,
"calls": []
}
]
@@ -7897,39 +7160,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 3388.087999999989,
- "execMs": 0.004790999984834343
+ "atMs": 5429.4360419999575,
+ "execMs": 0.011541999992914498
}
],
"timings": {
- "gateMs": 1386.282291999989,
+ "gateMs": 1137.0559580000117,
"gateSkipped": false,
- "ttftMs": 4504.1917079999985,
- "modelMs": 3493.8777079999854,
- "toolMs": 0.004790999984834343,
- "totalMs": 4880.164790999959,
+ "ttftMs": 7732.060999999987,
+ "modelMs": 6850.382125000004,
+ "toolMs": 0.011541999992914498,
+ "totalMs": 7987.449625000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2001.5774159999855,
+ "roundMs": 4292.129790999985,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.004790999984834343
+ "execMs": 0.011541999992914498
}
]
},
{
- "roundMs": 1492.300292,
- "firstDeltaMs": 1116.0830419999838,
+ "roundMs": 2558.252334000019,
+ "firstDeltaMs": 2302.5602090000175,
"calls": []
}
]
@@ -7937,39 +7201,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 5786.968334000005,
- "execMs": 0.0027500000142026693
+ "atMs": 6132.081209000025,
+ "execMs": 0.012792000023182482
}
],
"timings": {
- "gateMs": 1878.3122919999878,
+ "gateMs": 1136.6696670000092,
"gateSkipped": false,
- "ttftMs": 7455.391208999994,
- "modelMs": 5948.7000830000325,
- "toolMs": 0.0027500000142026693,
- "totalMs": 7827.0151250000345,
+ "ttftMs": 8287.011334000039,
+ "modelMs": 7512.283667000011,
+ "toolMs": 0.012792000023182482,
+ "totalMs": 8648.966126000043,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3908.405625000014,
+ "roundMs": 4995.133583999996,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.0027500000142026693
+ "execMs": 0.012792000023182482
}
]
},
{
- "roundMs": 2040.2944580000185,
- "firstDeltaMs": 1668.3890830000164,
+ "roundMs": 2517.150083000015,
+ "firstDeltaMs": 2154.8466660000267,
"calls": []
}
]
@@ -7980,38 +7245,38 @@
{
"name": "music-ask-single-generate",
"bucket": "single-tool",
- "passRate": 1,
+ "passRate": 0.6666666666666666,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1083,
- "p95": 1262,
- "mean": 1032,
- "n": 3
+ "p50": 866,
+ "p95": 1813,
+ "mean": 1340,
+ "n": 2
},
"ttftMs": {
- "p50": 4042,
- "p95": 4604,
- "mean": 4221,
- "n": 3
+ "p50": 6877,
+ "p95": 7086,
+ "mean": 6981,
+ "n": 2
},
"totalMs": {
- "p50": 4582,
- "p95": 4819,
- "mean": 4556,
- "n": 3
+ "p50": 7311,
+ "p95": 7378,
+ "mean": 7344,
+ "n": 2
},
"modelMs": {
- "p50": 3499,
- "p95": 4068,
- "mean": 3524,
- "n": 3
+ "p50": 5498,
+ "p95": 6511,
+ "mean": 6005,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -8021,79 +7286,83 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 2609.58033300002,
- "execMs": 0.08508299998356961
+ "atMs": 4611.029916999978,
+ "execMs": 0.28720800002338365
}
],
"timings": {
- "gateMs": 1083.2687499999884,
+ "gateMs": 866.2432499999995,
"gateSkipped": false,
- "ttftMs": 4016.3186250000144,
- "modelMs": 3498.7258750000037,
- "toolMs": 0.08508299998356961,
- "totalMs": 4582.079707999976,
+ "ttftMs": 7085.546583999996,
+ "modelMs": 6511.065917,
+ "toolMs": 0.28720800002338365,
+ "totalMs": 7377.596375000023,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1525.921250000014,
+ "roundMs": 3744.1937920000055,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.08508299998356961
+ "execMs": 0.28720800002338365
}
]
},
{
- "roundMs": 1972.8046249999898,
- "firstDeltaMs": 1406.6436669999966,
+ "roundMs": 2766.8721249999944,
+ "firstDeltaMs": 2474.123666999978,
"calls": []
}
]
}
},
{
- "pass": true,
- "intent": "work",
- "notes": [],
+ "pass": false,
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [
+ "reply did not match /music|track|bed|chill|background/i"
+ ],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 2981.170083999983,
- "execMs": 0.015791999991051853
+ "atMs": 6789.956500000029,
+ "execMs": 0.007041000004392117
}
],
"timings": {
- "gateMs": 1261.9658750000235,
+ "gateMs": 1048.178957999975,
"gateSkipped": false,
- "ttftMs": 4041.8911669999943,
- "modelMs": 3006.7967089999875,
- "toolMs": 0.015791999991051853,
- "totalMs": 4268.778376000002,
+ "ttftMs": 8447.88608299999,
+ "modelMs": 7940.687083999917,
+ "toolMs": 0.007041000004392117,
+ "totalMs": 8988.873082999897,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1718.7033750000119,
+ "roundMs": 5741.57029199996,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.015791999991051853
+ "execMs": 0.007041000004392117
}
]
},
{
- "roundMs": 1288.0933339999756,
- "firstDeltaMs": 1060.6305839999986,
+ "roundMs": 2199.116791999957,
+ "firstDeltaMs": 1657.881166999985,
"calls": []
}
]
@@ -8101,39 +7370,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 3035.3987079999933,
- "execMs": 0.015083999984199181
+ "atMs": 5215.4630829999805,
+ "execMs": 0.015750000020489097
}
],
"timings": {
- "gateMs": 751.0122499999998,
+ "gateMs": 1812.8571249999804,
"gateSkipped": false,
- "ttftMs": 4604.1150830000115,
- "modelMs": 4067.59825000001,
- "toolMs": 0.015083999984199181,
- "totalMs": 4818.625583999994,
+ "ttftMs": 6877.377457999974,
+ "modelMs": 5498.494874999975,
+ "toolMs": 0.015750000020489097,
+ "totalMs": 7311.367749999976,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2283.9672919999866,
+ "roundMs": 3402.3112919999985,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.015083999984199181
+ "execMs": 0.015750000020489097
}
]
},
{
- "roundMs": 1783.6309580000234,
- "firstDeltaMs": 1568.6356249999953,
+ "roundMs": 2096.1835829999764,
+ "firstDeltaMs": 1661.8356669999775,
"calls": []
}
]
@@ -8148,27 +7418,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1357,
- "p95": 3853,
- "mean": 2145,
+ "p50": 1256,
+ "p95": 1306,
+ "mean": 1234,
"n": 3
},
"ttftMs": {
- "p50": 4263,
- "p95": 6277,
- "mean": 4832,
+ "p50": 5593,
+ "p95": 5944,
+ "mean": 5701,
"n": 3
},
"totalMs": {
- "p50": 4516,
- "p95": 6522,
- "mean": 5076,
+ "p50": 5786,
+ "p95": 6200,
+ "mean": 5911,
"n": 3
},
"modelMs": {
- "p50": 2965,
- "p95": 3159,
- "mean": 2931,
+ "p50": 4530,
+ "p95": 5060,
+ "mean": 4677,
"n": 3
},
"toolMs": {
@@ -8185,39 +7455,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 2759.947709,
- "execMs": 0.01641599999857135
+ "atMs": 4216.761125000019,
+ "execMs": 0.005582999961916357
}
],
"timings": {
- "gateMs": 1356.586500000005,
+ "gateMs": 1256.086333000043,
"gateSkipped": false,
- "ttftMs": 4262.625249999983,
- "modelMs": 3159.140250000026,
- "toolMs": 0.01641599999857135,
- "totalMs": 4515.743166000029,
+ "ttftMs": 5592.821834000002,
+ "modelMs": 4529.936667000002,
+ "toolMs": 0.005582999961916357,
+ "totalMs": 5786.028583000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1402.9387080000015,
+ "roundMs": 2960.1880000000237,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.01641599999857135
+ "execMs": 0.005582999961916357
}
]
},
{
- "roundMs": 1756.2015420000243,
- "firstDeltaMs": 1502.616584000003,
+ "roundMs": 1569.748666999978,
+ "firstDeltaMs": 1375.9868750000023,
"calls": []
}
]
@@ -8225,39 +7496,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 5103.48662499999,
- "execMs": 0.039167000009911135
+ "atMs": 4611.807999999961,
+ "execMs": 0.008957999991253018
}
],
"timings": {
- "gateMs": 3852.942624999996,
+ "gateMs": 1140.435416000022,
"gateSkipped": false,
- "ttftMs": 6276.667832999985,
- "modelMs": 2668.8129590000026,
- "toolMs": 0.039167000009911135,
- "totalMs": 6521.794751000009,
+ "ttftMs": 5944.476665999973,
+ "modelMs": 5059.762041000009,
+ "toolMs": 0.008957999991253018,
+ "totalMs": 6200.206415000022,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1250.4026670000167,
+ "roundMs": 3471.0430410000263,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.039167000009911135
+ "execMs": 0.008957999991253018
}
]
},
{
- "roundMs": 1418.410291999986,
- "firstDeltaMs": 1173.1049999999814,
+ "roundMs": 1588.7189999999828,
+ "firstDeltaMs": 1332.624291999964,
"calls": []
}
]
@@ -8265,39 +7537,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 2900.592667000019,
- "execMs": 0.0029159999976400286
+ "atMs": 3651.462666999956,
+ "execMs": 0.003792000003159046
}
],
"timings": {
- "gateMs": 1224.4932910000207,
+ "gateMs": 1306.4265840000007,
"gateSkipped": false,
- "ttftMs": 3956.063959000021,
- "modelMs": 2964.579625000013,
- "toolMs": 0.0029159999976400286,
- "totalMs": 4189.075832000031,
+ "ttftMs": 5564.299499999965,
+ "modelMs": 4439.963792000024,
+ "toolMs": 0.003792000003159046,
+ "totalMs": 5746.394168000028,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1675.8332079999964,
+ "roundMs": 2344.7787920000264,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.0029159999976400286
+ "execMs": 0.003792000003159046
}
]
},
{
- "roundMs": 1288.7464170000167,
- "firstDeltaMs": 1055.432375000004,
+ "roundMs": 2095.1849999999977,
+ "firstDeltaMs": 1912.7869160000118,
"calls": []
}
]
@@ -8312,27 +7585,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1299,
- "p95": 1413,
- "mean": 1257,
+ "p50": 1265,
+ "p95": 1857,
+ "mean": 1338,
"n": 3
},
"ttftMs": {
- "p50": 4990,
- "p95": 5012,
- "mean": 4656,
+ "p50": 6255,
+ "p95": 6998,
+ "mean": 6250,
"n": 3
},
"totalMs": {
- "p50": 5278,
- "p95": 5800,
- "mean": 5275,
+ "p50": 6544,
+ "p95": 7310,
+ "mean": 6592,
"n": 3
},
"modelMs": {
- "p50": 3978,
- "p95": 4388,
- "mean": 4018,
+ "p50": 5030,
+ "p95": 6045,
+ "mean": 5254,
"n": 3
},
"toolMs": {
@@ -8349,39 +7622,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 2441.196541000012,
- "execMs": 0.33954200000152923
+ "atMs": 3902.8577499999665,
+ "execMs": 0.2448749999748543
}
],
"timings": {
- "gateMs": 1058.645292000001,
+ "gateMs": 890.8194159999839,
"gateSkipped": false,
- "ttftMs": 3964.3014580000017,
- "modelMs": 3689.1538750000473,
- "toolMs": 0.33954200000152923,
- "totalMs": 4748.13870900005,
+ "ttftMs": 5497.073332999949,
+ "modelMs": 5030.357083999959,
+ "toolMs": 0.2448749999748543,
+ "totalMs": 5921.421374999918,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1382.084125000023,
+ "roundMs": 3011.801541999972,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.33954200000152923
+ "execMs": 0.2448749999748543
}
]
},
{
- "roundMs": 2307.069750000024,
- "firstDeltaMs": 1522.728167000023,
+ "roundMs": 2018.5555419999873,
+ "firstDeltaMs": 1593.89383299998,
"calls": []
}
]
@@ -8389,39 +7663,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 3734.651582999999,
- "execMs": 0.03912500001024455
+ "atMs": 5611.954582999984,
+ "execMs": 0.017875000019557774
}
],
"timings": {
- "gateMs": 1299.295833000011,
+ "gateMs": 1265.3976249999832,
"gateSkipped": false,
- "ttftMs": 4989.948708000011,
- "modelMs": 3978.444208000001,
- "toolMs": 0.03912500001024455,
- "totalMs": 5277.779166000022,
+ "ttftMs": 6998.137832999986,
+ "modelMs": 6044.536334000004,
+ "toolMs": 0.017875000019557774,
+ "totalMs": 7309.951834000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2434.9237910000084,
+ "roundMs": 4346.152042000031,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.03912500001024455
+ "execMs": 0.017875000019557774
}
]
},
{
- "roundMs": 1543.5204169999924,
- "firstDeltaMs": 1255.1757090000028,
+ "roundMs": 1698.3842919999734,
+ "firstDeltaMs": 1386.103833000001,
"calls": []
}
]
@@ -8429,39 +7704,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 3600.091207999998,
- "execMs": 0.011916999996174127
+ "atMs": 4606.771000000008,
+ "execMs": 0.037708999996539205
}
],
"timings": {
- "gateMs": 1412.6328330000106,
+ "gateMs": 1856.8070420000004,
"gateSkipped": false,
- "ttftMs": 5012.297749999998,
- "modelMs": 4387.545416000037,
- "toolMs": 0.011916999996174127,
- "totalMs": 5800.190166000044,
+ "ttftMs": 6255.188332999998,
+ "modelMs": 4686.867582999985,
+ "toolMs": 0.037708999996539205,
+ "totalMs": 6543.712333999982,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2187.1944580000127,
+ "roundMs": 2749.6446250000154,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.011916999996174127
+ "execMs": 0.037708999996539205
}
]
},
{
- "roundMs": 2200.3509580000245,
- "firstDeltaMs": 1412.1720830000122,
+ "roundMs": 1937.2229579999694,
+ "firstDeltaMs": 1648.335541999957,
"calls": []
}
]
@@ -8476,27 +7752,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1137,
- "p95": 1228,
- "mean": 1082,
+ "p50": 1729,
+ "p95": 2019,
+ "mean": 1730,
"n": 3
},
"ttftMs": {
- "p50": 4373,
- "p95": 6756,
- "mean": 5026,
+ "p50": 7524,
+ "p95": 8857,
+ "mean": 7805,
"n": 3
},
"totalMs": {
- "p50": 4620,
- "p95": 7009,
- "mean": 5250,
+ "p50": 7878,
+ "p95": 9302,
+ "mean": 8160,
"n": 3
},
"modelMs": {
- "p50": 3391,
- "p95": 6129,
- "mean": 4168,
+ "p50": 5860,
+ "p95": 7859,
+ "mean": 6430,
"n": 3
},
"toolMs": {
@@ -8513,39 +7789,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 2951.4414169999945,
- "execMs": 0.41412500001024455
+ "atMs": 4982.480250000022,
+ "execMs": 0.19274999998742715
}
],
"timings": {
- "gateMs": 1228.4434170000022,
+ "gateMs": 1729.317834000045,
"gateSkipped": false,
- "ttftMs": 4373.0563339999935,
- "modelMs": 3391.4044170000125,
- "toolMs": 0.41412500001024455,
- "totalMs": 4620.261959000025,
+ "ttftMs": 7033.669750000001,
+ "modelMs": 5570.3408340000315,
+ "toolMs": 0.19274999998742715,
+ "totalMs": 7299.851418000064,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1722.7462499999965,
+ "roundMs": 3252.5617090000305,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.41412500001024455
+ "execMs": 0.19274999998742715
}
]
},
{
- "roundMs": 1668.658167000016,
- "firstDeltaMs": 1421.1499580000236,
+ "roundMs": 2317.779125000001,
+ "firstDeltaMs": 2050.9708329999703,
"calls": []
}
]
@@ -8553,39 +7830,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 2671.151165999996,
- "execMs": 0.05154200000106357
+ "atMs": 7521.289749999996,
+ "execMs": 0.00979199999710545
}
],
"timings": {
- "gateMs": 1136.740124999982,
+ "gateMs": 1442.8819169999915,
"gateSkipped": false,
- "ttftMs": 3949.2807500000054,
- "modelMs": 2982.7199579999724,
- "toolMs": 0.05154200000106357,
- "totalMs": 4119.5116249999555,
+ "ttftMs": 8857.49433299998,
+ "modelMs": 7859.268375000043,
+ "toolMs": 0.00979199999710545,
+ "totalMs": 9302.160084000032,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1534.1699999999837,
+ "roundMs": 6078.317917000037,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.05154200000106357
+ "execMs": 0.00979199999710545
}
]
},
{
- "roundMs": 1448.5499579999887,
- "firstDeltaMs": 1278.031290999992,
+ "roundMs": 1780.9504580000066,
+ "firstDeltaMs": 1336.1836250000051,
"calls": []
}
]
@@ -8593,39 +7871,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 5600.647874999995,
- "execMs": 0.01779099999112077
+ "atMs": 5600.396750000014,
+ "execMs": 0.013124999997671694
}
],
"timings": {
- "gateMs": 880.1609999999928,
+ "gateMs": 2018.6497910000035,
"gateSkipped": false,
- "ttftMs": 6755.517582999979,
- "modelMs": 6129.214583000023,
- "toolMs": 0.01779099999112077,
- "totalMs": 7009.393374000007,
+ "ttftMs": 7524.318957999989,
+ "modelMs": 5859.5447919999715,
+ "toolMs": 0.013124999997671694,
+ "totalMs": 7878.207707999973,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4720.2537500000035,
+ "roundMs": 3581.496541999979,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.01779099999112077
+ "execMs": 0.013124999997671694
}
]
},
{
- "roundMs": 1408.9608330000192,
- "firstDeltaMs": 1154.8112499999988,
+ "roundMs": 2278.0482499999925,
+ "firstDeltaMs": 1923.8742079999647,
"calls": []
}
]
@@ -8640,27 +7919,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1170,
- "p95": 1250,
- "mean": 1119,
+ "p50": 1733,
+ "p95": 1735,
+ "mean": 1550,
"n": 3
},
"ttftMs": {
- "p50": 4056,
- "p95": 4418,
- "mean": 4124,
+ "p50": 6898,
+ "p95": 7165,
+ "mean": 6844,
"n": 3
},
"totalMs": {
- "p50": 4379,
- "p95": 4869,
- "mean": 4502,
+ "p50": 7345,
+ "p95": 7518,
+ "mean": 7267,
"n": 3
},
"modelMs": {
- "p50": 3440,
- "p95": 3619,
- "mean": 3383,
+ "p50": 5759,
+ "p95": 5784,
+ "mean": 5717,
"n": 3
},
"toolMs": {
@@ -8677,39 +7956,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 2471.428749999992,
- "execMs": 0.3545410000078846
+ "atMs": 5352.9307499999995,
+ "execMs": 0.24004099995363504
}
],
"timings": {
- "gateMs": 938.6916250000068,
+ "gateMs": 1733.3685830000322,
"gateSkipped": false,
- "ttftMs": 4055.778041999991,
- "modelMs": 3439.9353739999933,
- "toolMs": 0.3545410000078846,
- "totalMs": 4378.981540000008,
+ "ttftMs": 7165.039374999993,
+ "modelMs": 5784.079458000022,
+ "toolMs": 0.24004099995363504,
+ "totalMs": 7517.688082000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1531.9294579999987,
+ "roundMs": 3619.05037500005,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.3545410000078846
+ "execMs": 0.24004099995363504
}
]
},
{
- "roundMs": 1908.0059159999946,
- "firstDeltaMs": 1583.9580410000053,
+ "roundMs": 2165.029082999972,
+ "firstDeltaMs": 1811.8146660000202,
"calls": []
}
]
@@ -8717,39 +7997,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 2631.1161250000005,
- "execMs": 0.024625000020023435
+ "atMs": 4623.090415999992,
+ "execMs": 0.019625000015366822
}
],
"timings": {
- "gateMs": 1169.5677499999874,
+ "gateMs": 1181.2353749999893,
"gateSkipped": false,
- "ttftMs": 3898.331124999997,
- "modelMs": 3089.0599159999983,
- "toolMs": 0.024625000020023435,
- "totalMs": 4258.652291000006,
+ "ttftMs": 6468.31341599999,
+ "modelMs": 5758.925667000003,
+ "toolMs": 0.019625000015366822,
+ "totalMs": 6940.180667000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1461.2959580000024,
+ "roundMs": 3441.6897500000196,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.024625000020023435
+ "execMs": 0.019625000015366822
}
]
},
{
- "roundMs": 1627.763957999996,
- "firstDeltaMs": 1267.1504579999892,
+ "roundMs": 2317.2359169999836,
+ "firstDeltaMs": 1845.1149999999907,
"calls": []
}
]
@@ -8757,39 +8038,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 3151.091207999998,
- "execMs": 0.018207999994046986
+ "atMs": 5501.34166700003,
+ "execMs": 0.014083000016398728
}
],
"timings": {
- "gateMs": 1249.919041999994,
+ "gateMs": 1735.4468329999945,
"gateSkipped": false,
- "ttftMs": 4417.703917000006,
- "modelMs": 3619.34466599999,
- "toolMs": 0.018207999994046986,
- "totalMs": 4869.281915999978,
+ "ttftMs": 6897.61050000001,
+ "modelMs": 5609.050833999936,
+ "toolMs": 0.014083000016398728,
+ "totalMs": 7344.511749999947,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1900.9975410000188,
+ "roundMs": 3765.733208999969,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.018207999994046986
+ "execMs": 0.014083000016398728
}
]
},
{
- "roundMs": 1718.3471249999711,
- "firstDeltaMs": 1266.5643339999951,
+ "roundMs": 1843.317624999967,
+ "firstDeltaMs": 1396.2085840000072,
"calls": []
}
]
@@ -8804,27 +8086,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1892,
- "p95": 1990,
- "mean": 1675,
+ "p50": 966,
+ "p95": 1325,
+ "mean": 1083,
"n": 3
},
"ttftMs": {
- "p50": 4890,
- "p95": 5485,
- "mean": 4995,
+ "p50": 9343,
+ "p95": 10221,
+ "mean": 9529,
"n": 3
},
"totalMs": {
- "p50": 5572,
- "p95": 6305,
- "mean": 5590,
+ "p50": 9505,
+ "p95": 10506,
+ "mean": 9785,
"n": 3
},
"modelMs": {
- "p50": 3749,
- "p95": 4315,
- "mean": 3915,
+ "p50": 8384,
+ "p95": 9540,
+ "mean": 8701,
"n": 3
},
"toolMs": {
@@ -8841,49 +8123,50 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 3502.7356250000303,
- "execMs": 0.4294169999775477
+ "atMs": 7204.605833000038,
+ "execMs": 0.1987080000108108
},
{
"name": "set_animation",
"round": 0,
- "atMs": 3503.2135410000337,
- "execMs": 0.013832999975420535
+ "atMs": 7204.815125000023,
+ "execMs": 0.004083000007085502
}
],
"timings": {
- "gateMs": 1142.21875,
+ "gateMs": 1325.3524580000085,
"gateSkipped": false,
- "ttftMs": 4609.562291000038,
- "modelMs": 3749.1882910000277,
- "toolMs": 0.4432499999529682,
- "totalMs": 4891.850290999981,
+ "ttftMs": 9022.974583000003,
+ "modelMs": 8179.511999999988,
+ "toolMs": 0.2027910000178963,
+ "totalMs": 9505.067249000014,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2360.067625000025,
+ "roundMs": 5878.623708999949,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.4294169999775477
+ "execMs": 0.1987080000108108
},
{
"name": "set_animation",
- "execMs": 0.013832999975420535
+ "execMs": 0.004083000007085502
}
]
},
{
- "roundMs": 1389.1206660000025,
- "firstDeltaMs": 1106.2861660000053,
+ "roundMs": 2300.8882910000393,
+ "firstDeltaMs": 1818.1320410000044,
"calls": []
}
]
@@ -8891,49 +8174,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 4119.214042000007,
- "execMs": 0.010833000007551163
+ "atMs": 8320.231082999962,
+ "execMs": 0.017791999969631433
},
{
"name": "set_animation",
"round": 0,
- "atMs": 4119.266291999957,
- "execMs": 0.008874999999534339
+ "atMs": 8320.264666999981,
+ "execMs": 0.004415999981574714
}
],
"timings": {
- "gateMs": 1990.3166250000359,
+ "gateMs": 966.3863339999807,
"gateSkipped": false,
- "ttftMs": 5485.105166999972,
- "modelMs": 4315.029126000009,
- "toolMs": 0.019708000007085502,
- "totalMs": 6305.3654590000515,
+ "ttftMs": 10220.509124999982,
+ "modelMs": 9540.03125,
+ "toolMs": 0.022207999951206148,
+ "totalMs": 10506.439791999932,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2128.3894590000273,
+ "roundMs": 7353.48199999996,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.010833000007551163
+ "execMs": 0.017791999969631433
},
{
"name": "set_animation",
- "execMs": 0.008874999999534339
+ "execMs": 0.004415999981574714
}
]
},
{
- "roundMs": 2186.6396669999813,
- "firstDeltaMs": 1365.7561250000144,
+ "roundMs": 2186.54925000004,
+ "firstDeltaMs": 1900.2129999999888,
"calls": []
}
]
@@ -8941,49 +8225,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 3403.306082999974,
- "execMs": 0.016957999963779002
+ "atMs": 4141.057583000045,
+ "execMs": 0.007041000004392117
},
{
"name": "set_animation",
"round": 0,
- "atMs": 3403.338541999983,
- "execMs": 0.008291999984066933
+ "atMs": 4141.068958000047,
+ "execMs": 0.001999999978579581
}
],
"timings": {
- "gateMs": 1892.0746250000084,
+ "gateMs": 958.4116250000079,
"gateSkipped": false,
- "ttftMs": 4890.418917000003,
- "modelMs": 3680.231625000015,
- "toolMs": 0.025249999947845936,
- "totalMs": 5572.331499999971,
+ "ttftMs": 9342.519958000048,
+ "modelMs": 8384.298415999976,
+ "toolMs": 0.009040999982971698,
+ "totalMs": 9342.719081999967,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1510.9511670000502,
+ "roundMs": 3182.521458000003,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.016957999963779002
+ "execMs": 0.007041000004392117
},
{
"name": "set_animation",
- "execMs": 0.008291999984066933
+ "execMs": 0.001999999978579581
}
]
},
{
- "roundMs": 2169.2804579999647,
- "firstDeltaMs": 1487.022082999989,
+ "roundMs": 5201.776957999973,
+ "firstDeltaMs": 5201.435165999981,
"calls": []
}
]
@@ -8994,346 +8279,469 @@
{
"name": "filler-words-cut-with-editor",
"bucket": "multi-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1170,
- "p95": 1308,
- "mean": 1239,
- "n": 2
+ "p50": 1158,
+ "p95": 1940,
+ "mean": 1416,
+ "n": 3
},
"ttftMs": {
- "p50": 5786,
- "p95": 12080,
- "mean": 8933,
- "n": 2
+ "p50": 39798,
+ "p95": 76661,
+ "mean": 51988,
+ "n": 3
},
"totalMs": {
- "p50": 6088,
- "p95": 12702,
- "mean": 9395,
- "n": 2
+ "p50": 40289,
+ "p95": 77116,
+ "mean": 52498,
+ "n": 3
},
"modelMs": {
- "p50": 4916,
- "p95": 11394,
- "mean": 8155,
- "n": 2
+ "p50": 38936,
+ "p95": 75954,
+ "mean": 51079,
+ "n": 3
},
"toolMs": {
"p50": 1,
- "p95": 1,
- "mean": 1,
- "n": 2
+ "p95": 4,
+ "mean": 2,
+ "n": 3
},
"rounds": {
- "mean": 3,
- "max": 4
+ "mean": 14,
+ "max": 20
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
- "name": "trim_clip",
+ "name": "list_skills",
"round": 0,
- "atMs": 4250.074416999996,
- "execMs": 0.9586250000284053
+ "atMs": 5022.804292000015,
+ "execMs": 0.23929200001293793
},
{
- "name": "update_cue",
- "round": 0,
- "atMs": 4251.049584000022,
- "execMs": 0.03095799998845905
+ "name": "read_skill",
+ "round": 1,
+ "atMs": 6547.173124999972,
+ "execMs": 0.007832999981474131
},
{
- "name": "update_cue",
- "round": 0,
- "atMs": 4251.082959000021,
- "execMs": 0.017166999983601272
+ "name": "detect_silence",
+ "round": 2,
+ "atMs": 9616.514291999978,
+ "execMs": 0.014665999973658472
},
{
- "name": "delete_cue",
- "round": 0,
- "atMs": 4251.102167000005,
- "execMs": 0.03583300003083423
+ "name": "listen_audio",
+ "round": 3,
+ "atMs": 12756.814083000005,
+ "execMs": 0.01554200000828132
},
{
- "name": "update_cue",
- "round": 0,
- "atMs": 4251.140874999983,
- "execMs": 0.006957999954465777
+ "name": "split_at",
+ "round": 4,
+ "atMs": 22150.790374999982,
+ "execMs": 1.484375
},
{
- "name": "update_cue",
- "round": 0,
- "atMs": 4251.150583999988,
- "execMs": 0.00574999995296821
- }
- ],
- "timings": {
- "gateMs": 1170.2128340000054,
- "gateSkipped": false,
- "ttftMs": 5785.889542000019,
- "modelMs": 4916.3042499999865,
- "toolMs": 1.0552909999387339,
- "totalMs": 6087.572374999931,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3079.238167000003,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "trim_clip",
- "execMs": 0.9586250000284053
- },
- {
- "name": "update_cue",
- "execMs": 0.03095799998845905
- },
- {
- "name": "update_cue",
- "execMs": 0.017166999983601272
- },
- {
- "name": "delete_cue",
- "execMs": 0.03583300003083423
- },
- {
- "name": "update_cue",
- "execMs": 0.006957999954465777
- },
- {
- "name": "update_cue",
- "execMs": 0.00574999995296821
- }
- ]
- },
- {
- "roundMs": 1837.0660829999833,
- "firstDeltaMs": 1534.6825409999583,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": false,
- "intent": "work",
- "notes": [
- "mutating tools called: remove_gap, remove_gap"
- ],
- "trace": [
+ "name": "split_at",
+ "round": 4,
+ "atMs": 22152.288624999986,
+ "execMs": 0.045082999975420535
+ },
{
"name": "split_at",
- "round": 0,
- "atMs": 3247.5335830000113,
- "execMs": 0.3268339999485761
+ "round": 4,
+ "atMs": 22152.338332999963,
+ "execMs": 0.017875000019557774
+ },
+ {
+ "name": "undo",
+ "round": 5,
+ "atMs": 31146.19750000001,
+ "execMs": 0.015458000008948147
+ },
+ {
+ "name": "undo",
+ "round": 5,
+ "atMs": 31146.21554200002,
+ "execMs": 0.003542000020388514
+ },
+ {
+ "name": "undo",
+ "round": 5,
+ "atMs": 31146.219500000007,
+ "execMs": 0.0009999999892897904
+ },
+ {
+ "name": "select",
+ "round": 6,
+ "atMs": 33756.8615,
+ "execMs": 0.020791000046301633
},
{
"name": "split_at",
- "round": 0,
- "atMs": 3247.871041000006,
- "execMs": 0.008707999950274825
+ "round": 6,
+ "atMs": 33756.88583300001,
+ "execMs": 0.07325000001583248
+ },
+ {
+ "name": "select",
+ "round": 7,
+ "atMs": 35835.835374999966,
+ "execMs": 0.006832999992184341
},
{
"name": "split_at",
- "round": 0,
- "atMs": 3247.885707999987,
- "execMs": 0.004832999955397099
+ "round": 7,
+ "atMs": 35835.844958,
+ "execMs": 0.033041999966371804
+ },
+ {
+ "name": "select",
+ "round": 8,
+ "atMs": 38079.69208299997,
+ "execMs": 0.023332999960985035
},
{
"name": "split_at",
- "round": 0,
- "atMs": 3247.891249999986,
- "execMs": 0.005000000004656613
+ "round": 8,
+ "atMs": 38079.72445799998,
+ "execMs": 0.07087500003399327
},
{
- "name": "delete_item",
- "round": 0,
- "atMs": 3247.897875000024,
- "execMs": 0.014750000031199306
+ "name": "undo",
+ "round": 9,
+ "atMs": 45000.175583000004,
+ "execMs": 0.054583000019192696
},
{
- "name": "delete_item",
- "round": 0,
- "atMs": 3247.913707999978,
- "execMs": 0.0020829999702982605
+ "name": "undo",
+ "round": 9,
+ "atMs": 45000.239833,
+ "execMs": 0.005124999966938049
},
{
- "name": "remove_gap",
- "round": 0,
- "atMs": 3247.9162909999723,
- "execMs": 0.005625000048894435
+ "name": "select",
+ "round": 10,
+ "atMs": 47236.570582999964,
+ "execMs": 0.006875000020954758
},
{
- "name": "remove_gap",
- "round": 0,
- "atMs": 3247.9222500000033,
- "execMs": 0.003958999994210899
+ "name": "split_at",
+ "round": 10,
+ "atMs": 47236.57937499997,
+ "execMs": 0.019583000044804066
},
{
- "name": "get_state",
- "round": 1,
- "atMs": 4999.557582999987,
- "execMs": 0.8057499999995343
+ "name": "update_audio",
+ "round": 11,
+ "atMs": 49808.219125,
+ "execMs": 0.18670799996471033
},
{
- "name": "delete_item",
- "round": 2,
- "atMs": 6680.561416000011,
- "execMs": 0.29029100004117936
+ "name": "select",
+ "round": 12,
+ "atMs": 51946.40633299999,
+ "execMs": 0.006624999979976565
},
{
- "name": "delete_item",
- "round": 2,
- "atMs": 6680.870249999978,
- "execMs": 0.030958999996073544
+ "name": "split_at",
+ "round": 12,
+ "atMs": 51946.4155,
+ "execMs": 0.0490410000202246
},
{
"name": "delete_item",
- "round": 2,
- "atMs": 6680.904250000021,
- "execMs": 0.011667000013403594
+ "round": 13,
+ "atMs": 55467.96799999999,
+ "execMs": 0.3750410000211559
},
{
"name": "delete_item",
- "round": 3,
- "atMs": 8786.732207999972,
- "execMs": 0.12120899994624779
+ "round": 14,
+ "atMs": 62527.78391699999,
+ "execMs": 0.06154099997365847
},
{
- "name": "delete_item",
- "round": 3,
- "atMs": 8786.857040999981,
- "execMs": 0.0054579999996349216
+ "name": "update_audio",
+ "round": 15,
+ "atMs": 64900.144457999966,
+ "execMs": 0.06174999999348074
},
{
- "name": "delete_cue",
- "round": 4,
- "atMs": 11277.009083000012,
- "execMs": 0.02254099998390302
+ "name": "refine_speech_cuts",
+ "round": 16,
+ "atMs": 67827.75724999997,
+ "execMs": 0.3430419999640435
},
{
- "name": "delete_cue",
- "round": 4,
- "atMs": 11277.034832999983,
- "execMs": 0.0020829999702982605
+ "name": "get_state",
+ "round": 17,
+ "atMs": 69766.05212499999,
+ "execMs": 0.26408299995819107
+ },
+ {
+ "name": "subtitles_generate",
+ "round": 18,
+ "atMs": 72880.570917,
+ "execMs": 0.04245800001081079
}
],
"timings": {
- "gateMs": 930.9135410000454,
+ "gateMs": 1158.1053750000428,
"gateSkipped": false,
- "ttftMs": 12766.206540999992,
- "modelMs": 12683.429292999965,
- "toolMs": 1.6617499998537824,
- "totalMs": 13616.004583999864,
+ "ttftMs": 76661.166792,
+ "modelMs": 75953.92462600017,
+ "toolMs": 3.5492449999437667,
+ "totalMs": 77115.57924600015,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2316.0317499999655,
+ "roundMs": 3864.4538750000065,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "list_skills",
+ "execMs": 0.23929200001293793
+ }
+ ]
+ },
+ {
+ "roundMs": 1524.0737080000108,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "read_skill",
+ "execMs": 0.007832999981474131
+ }
+ ]
+ },
+ {
+ "roundMs": 3069.314958999981,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "detect_silence",
+ "execMs": 0.014665999973658472
+ }
+ ]
+ },
+ {
+ "roundMs": 3140.263042000006,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.01554200000828132
+ }
+ ]
+ },
+ {
+ "roundMs": 9393.908916999993,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.3268339999485761
+ "execMs": 1.484375
},
{
"name": "split_at",
- "execMs": 0.008707999950274825
+ "execMs": 0.045082999975420535
},
{
"name": "split_at",
- "execMs": 0.004832999955397099
+ "execMs": 0.017875000019557774
+ }
+ ]
+ },
+ {
+ "roundMs": 8993.79812500003,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "undo",
+ "execMs": 0.015458000008948147
},
{
- "name": "split_at",
- "execMs": 0.005000000004656613
+ "name": "undo",
+ "execMs": 0.003542000020388514
},
{
- "name": "delete_item",
- "execMs": 0.014750000031199306
+ "name": "undo",
+ "execMs": 0.0009999999892897904
+ }
+ ]
+ },
+ {
+ "roundMs": 2610.6235419999575,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "select",
+ "execMs": 0.020791000046301633
},
{
- "name": "delete_item",
- "execMs": 0.0020829999702982605
+ "name": "split_at",
+ "execMs": 0.07325000001583248
+ }
+ ]
+ },
+ {
+ "roundMs": 2078.8490409999504,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "select",
+ "execMs": 0.006832999992184341
},
{
- "name": "remove_gap",
- "execMs": 0.005625000048894435
+ "name": "split_at",
+ "execMs": 0.033041999966371804
+ }
+ ]
+ },
+ {
+ "roundMs": 2243.7742500000168,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "select",
+ "execMs": 0.023332999960985035
},
{
- "name": "remove_gap",
- "execMs": 0.003958999994210899
+ "name": "split_at",
+ "execMs": 0.07087500003399327
}
]
},
{
- "roundMs": 1751.4909169999883,
+ "roundMs": 6920.324042000051,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.8057499999995343
+ "name": "undo",
+ "execMs": 0.054583000019192696
+ },
+ {
+ "name": "undo",
+ "execMs": 0.005124999966938049
}
]
},
{
- "roundMs": 1680.0798749999958,
+ "roundMs": 2236.2985830000252,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.29029100004117936
+ "name": "select",
+ "execMs": 0.006875000020954758
},
{
- "name": "delete_item",
- "execMs": 0.030958999996073544
+ "name": "split_at",
+ "execMs": 0.019583000044804066
+ }
+ ]
+ },
+ {
+ "roundMs": 2571.5945420000353,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_audio",
+ "execMs": 0.18670799996471033
+ }
+ ]
+ },
+ {
+ "roundMs": 2137.973541000043,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "select",
+ "execMs": 0.006624999979976565
},
{
- "name": "delete_item",
- "execMs": 0.011667000013403594
+ "name": "split_at",
+ "execMs": 0.0490410000202246
}
]
},
{
- "roundMs": 2105.7529170000344,
+ "roundMs": 3521.4734999999637,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.12120899994624779
- },
+ "execMs": 0.3750410000211559
+ }
+ ]
+ },
+ {
+ "roundMs": 7059.387959000014,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "delete_item",
- "execMs": 0.0054579999996349216
+ "execMs": 0.06154099997365847
}
]
},
{
- "roundMs": 2490.1198339999537,
+ "roundMs": 2372.242666999984,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_cue",
- "execMs": 0.02254099998390302
- },
+ "name": "update_audio",
+ "execMs": 0.06174999999348074
+ }
+ ]
+ },
+ {
+ "roundMs": 2927.506000000052,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "delete_cue",
- "execMs": 0.0020829999702982605
+ "name": "refine_speech_cuts",
+ "execMs": 0.3430419999640435
+ }
+ ]
+ },
+ {
+ "roundMs": 1937.9183750000084,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.26408299995819107
+ }
+ ]
+ },
+ {
+ "roundMs": 3114.2263750000275,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "subtitles_generate",
+ "execMs": 0.04245800001081079
}
]
},
{
- "roundMs": 2339.954000000027,
- "firstDeltaMs": 1489.1373749999912,
+ "roundMs": 4235.91958300001,
+ "firstDeltaMs": 3780.528875000018,
"calls": []
}
]
@@ -9341,141 +8749,399 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
- "name": "split_at",
+ "name": "list_skills",
"round": 0,
- "atMs": 3650.1817080000183,
- "execMs": 0.08837499999208376
+ "atMs": 5710.850083000027,
+ "execMs": 0.09324999991804361
},
{
- "name": "split_at",
- "round": 0,
- "atMs": 3650.2767079999903,
- "execMs": 0.011541999992914498
+ "name": "read_skill",
+ "round": 1,
+ "atMs": 7404.054832999944,
+ "execMs": 0.029959000064991415
+ },
+ {
+ "name": "detect_silence",
+ "round": 2,
+ "atMs": 9999.54700000002,
+ "execMs": 0.015165999997407198
+ },
+ {
+ "name": "listen_audio",
+ "round": 3,
+ "atMs": 13149.960707999999,
+ "execMs": 0.02441700000781566
},
{
"name": "split_at",
- "round": 0,
- "atMs": 3650.289208000002,
- "execMs": 0.012457999982871115
+ "round": 4,
+ "atMs": 21021.866540999967,
+ "execMs": 0.5075420000357553
},
{
"name": "split_at",
- "round": 0,
- "atMs": 3650.302291000029,
- "execMs": 0.018041000002995133
+ "round": 5,
+ "atMs": 22892.135583000025,
+ "execMs": 0.2069589999737218
},
{
"name": "split_at",
- "round": 0,
- "atMs": 3650.3209999999963,
- "execMs": 0.017333000025246292
+ "round": 6,
+ "atMs": 24762.00745799998,
+ "execMs": 0.07499999995343387
},
{
"name": "delete_item",
- "round": 0,
- "atMs": 3650.338916000037,
- "execMs": 0.006583000009413809
+ "round": 7,
+ "atMs": 27747.58924999996,
+ "execMs": 0.13137499999720603
},
{
"name": "delete_item",
- "round": 0,
- "atMs": 3650.34612500004,
- "execMs": 0.002791999955661595
+ "round": 8,
+ "atMs": 29552.192374999984,
+ "execMs": 0.10241699998732656
},
{
- "name": "get_state",
- "round": 1,
- "atMs": 5224.696540999983,
- "execMs": 0.21766699996078387
+ "name": "refine_speech_cuts",
+ "round": 9,
+ "atMs": 31833.656666000024,
+ "execMs": 0.04749999998603016
},
{
- "name": "delete_item",
- "round": 2,
- "atMs": 10464.005416000029,
- "execMs": 0.08929099998204038
+ "name": "get_state",
+ "round": 10,
+ "atMs": 33912.53645799996,
+ "execMs": 0.16574999992735684
},
{
- "name": "delete_item",
- "round": 2,
- "atMs": 10464.106415999995,
- "execMs": 0.06999999994877726
+ "name": "subtitles_generate",
+ "round": 11,
+ "atMs": 36498.906582999974,
+ "execMs": 0.005708000040613115
}
],
"timings": {
- "gateMs": 1307.5696250000037,
+ "gateMs": 1150.7756250000675,
"gateSkipped": false,
- "ttftMs": 12080.270583000034,
- "modelMs": 11394.39500099991,
- "toolMs": 0.5340819998527877,
- "totalMs": 12702.498707999766,
+ "ttftMs": 39505.454875000054,
+ "modelMs": 38936.40983099991,
+ "toolMs": 1.4050429998897016,
+ "totalMs": 40088.59049899987,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2342.4413339999737,
+ "roundMs": 4559.847624999937,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.08837499999208376
- },
- {
- "name": "split_at",
- "execMs": 0.011541999992914498
- },
+ "name": "list_skills",
+ "execMs": 0.09324999991804361
+ }
+ ]
+ },
+ {
+ "roundMs": 1692.9922910000896,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "read_skill",
+ "execMs": 0.029959000064991415
+ }
+ ]
+ },
+ {
+ "roundMs": 2595.389749999973,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "detect_silence",
+ "execMs": 0.015165999997407198
+ }
+ ]
+ },
+ {
+ "roundMs": 3150.3675829999847,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.02441700000781566
+ }
+ ]
+ },
+ {
+ "roundMs": 7871.8288750000065,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "split_at",
- "execMs": 0.012457999982871115
- },
+ "execMs": 0.5075420000357553
+ }
+ ]
+ },
+ {
+ "roundMs": 1869.6390829999,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "split_at",
- "execMs": 0.018041000002995133
- },
+ "execMs": 0.2069589999737218
+ }
+ ]
+ },
+ {
+ "roundMs": 1869.6137909999816,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "split_at",
- "execMs": 0.017333000025246292
- },
+ "execMs": 0.07499999995343387
+ }
+ ]
+ },
+ {
+ "roundMs": 2985.42745800002,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "delete_item",
- "execMs": 0.006583000009413809
- },
+ "execMs": 0.13137499999720603
+ }
+ ]
+ },
+ {
+ "roundMs": 1804.361166000017,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "delete_item",
- "execMs": 0.002791999955661595
+ "execMs": 0.10241699998732656
+ }
+ ]
+ },
+ {
+ "roundMs": 2281.280541999964,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "refine_speech_cuts",
+ "execMs": 0.04749999998603016
}
]
},
{
- "roundMs": 1574.2900839999784,
+ "roundMs": 2078.7908750000643,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.21766699996078387
+ "execMs": 0.16574999992735684
+ }
+ ]
+ },
+ {
+ "roundMs": 2586.1594589999877,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "subtitles_generate",
+ "execMs": 0.005708000040613115
+ }
+ ]
+ },
+ {
+ "roundMs": 3590.7113329999847,
+ "firstDeltaMs": 3006.522875000024,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [],
+ "trace": [
+ {
+ "name": "listen_audio",
+ "round": 0,
+ "atMs": 10263.734167000046,
+ "execMs": 0.1294580000685528
+ },
+ {
+ "name": "detect_silence",
+ "round": 1,
+ "atMs": 14484.453332999954,
+ "execMs": 0.05062500003259629
+ },
+ {
+ "name": "split_at",
+ "round": 2,
+ "atMs": 19703.18400000001,
+ "execMs": 0.26604099990800023
+ },
+ {
+ "name": "split_at",
+ "round": 2,
+ "atMs": 19703.47645800002,
+ "execMs": 0.042625000001862645
+ },
+ {
+ "name": "split_at",
+ "round": 2,
+ "atMs": 19703.521833000006,
+ "execMs": 0.04787499993108213
+ },
+ {
+ "name": "split_at",
+ "round": 3,
+ "atMs": 22462.433042000048,
+ "execMs": 0.14362500002607703
+ },
+ {
+ "name": "delete_item",
+ "round": 4,
+ "atMs": 27190.866541999974,
+ "execMs": 0.07991700002457947
+ },
+ {
+ "name": "delete_item",
+ "round": 4,
+ "atMs": 27190.952374999993,
+ "execMs": 0.038625000044703484
+ },
+ {
+ "name": "refine_speech_cuts",
+ "round": 5,
+ "atMs": 30273.204125000047,
+ "execMs": 0.12666700000409037
+ },
+ {
+ "name": "get_state",
+ "round": 6,
+ "atMs": 32823.889417,
+ "execMs": 0.1640419999603182
+ },
+ {
+ "name": "subtitles_generate",
+ "round": 7,
+ "atMs": 37103.029582999996,
+ "execMs": 0.031667000032030046
+ }
+ ],
+ "timings": {
+ "gateMs": 1940.262999999919,
+ "gateSkipped": false,
+ "ttftMs": 39798.11025000003,
+ "modelMs": 38347.4569560003,
+ "toolMs": 1.1211670000338927,
+ "totalMs": 40288.84112300025,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 8323.12725000002,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.1294580000685528
+ }
+ ]
+ },
+ {
+ "roundMs": 4220.395458000014,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "detect_silence",
+ "execMs": 0.05062500003259629
+ }
+ ]
+ },
+ {
+ "roundMs": 5218.540458000032,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "split_at",
+ "execMs": 0.26604099990800023
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.042625000001862645
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.04787499993108213
}
]
},
{
- "roundMs": 5238.996249999967,
+ "roundMs": 2758.803958000033,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "split_at",
+ "execMs": 0.14362500002607703
+ }
+ ]
+ },
+ {
+ "roundMs": 4728.239249999984,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.08929099998204038
+ "execMs": 0.07991700002457947
},
{
"name": "delete_item",
- "execMs": 0.06999999994877726
+ "execMs": 0.038625000044703484
+ }
+ ]
+ },
+ {
+ "roundMs": 3082.1709580000024,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "refine_speech_cuts",
+ "execMs": 0.12666700000409037
+ }
+ ]
+ },
+ {
+ "roundMs": 2550.471750000026,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.1640419999603182
+ }
+ ]
+ },
+ {
+ "roundMs": 4278.882166000083,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "subtitles_generate",
+ "execMs": 0.031667000032030046
}
]
},
{
- "roundMs": 2238.6673329999903,
- "firstDeltaMs": 1616.059540999995,
+ "roundMs": 3186.825708000106,
+ "firstDeltaMs": 2694.9817500000354,
"calls": []
}
]
@@ -9490,27 +9156,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1144,
- "p95": 1219,
- "mean": 1110,
+ "p50": 1079,
+ "p95": 1292,
+ "mean": 1103,
"n": 3
},
"ttftMs": {
- "p50": 4738,
- "p95": 5413,
- "mean": 4655,
+ "p50": 6080,
+ "p95": 6594,
+ "mean": 6222,
"n": 3
},
"totalMs": {
- "p50": 5073,
- "p95": 5839,
- "mean": 4988,
+ "p50": 6360,
+ "p95": 6843,
+ "mean": 6475,
"n": 3
},
"modelMs": {
- "p50": 3929,
- "p95": 4620,
- "mean": 3878,
+ "p50": 5282,
+ "p95": 5903,
+ "mean": 5371,
"n": 3
},
"toolMs": {
@@ -9527,39 +9193,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 2498.907332999981,
- "execMs": 0.017457999987527728
+ "atMs": 3980.090792000061,
+ "execMs": 0.001832999987527728
}
],
"timings": {
- "gateMs": 966.8014579999726,
+ "gateMs": 1078.5002919998951,
"gateSkipped": false,
- "ttftMs": 3814.564458000008,
- "modelMs": 3085.2306250000256,
- "toolMs": 0.017457999987527728,
- "totalMs": 4052.0495409999858,
+ "ttftMs": 6080.213667000062,
+ "modelMs": 5281.528499000007,
+ "toolMs": 0.001832999987527728,
+ "totalMs": 6360.03062399989,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1531.5070410000044,
+ "roundMs": 2901.3965409999946,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.017457999987527728
+ "execMs": 0.001832999987527728
}
]
},
{
- "roundMs": 1553.7235840000212,
- "firstDeltaMs": 1315.5369170000195,
+ "roundMs": 2380.1319580000127,
+ "firstDeltaMs": 2100.1054170000134,
"calls": []
}
]
@@ -9567,39 +9234,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 3474.2385420000064,
- "execMs": 0.02404200000455603
+ "atMs": 4000.6652079999913,
+ "execMs": 0.010374999954365194
}
],
"timings": {
- "gateMs": 1144.1720830000122,
+ "gateMs": 939.8322919999482,
"gateSkipped": false,
- "ttftMs": 4737.89850000001,
- "modelMs": 3928.5681249999907,
- "toolMs": 0.02404200000455603,
- "totalMs": 5072.764250000007,
+ "ttftMs": 6594.344417000073,
+ "modelMs": 5903.099791999906,
+ "toolMs": 0.010374999954365194,
+ "totalMs": 6842.942458999809,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2329.6522920000134,
+ "roundMs": 3060.5287499999395,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.02404200000455603
+ "execMs": 0.010374999954365194
}
]
},
{
- "roundMs": 1598.9158329999773,
- "firstDeltaMs": 1263.5810409999685,
+ "roundMs": 2842.571041999967,
+ "firstDeltaMs": 2593.599292000057,
"calls": []
}
]
@@ -9607,39 +9275,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 4191.5419160000165,
- "execMs": 0.011624999984633178
+ "atMs": 4217.401208000025,
+ "execMs": 0.027834000065922737
}
],
"timings": {
- "gateMs": 1218.7544160000398,
+ "gateMs": 1291.7622919999994,
"gateSkipped": false,
- "ttftMs": 5413.118666000024,
- "modelMs": 4620.437749999983,
- "toolMs": 0.011624999984633178,
- "totalMs": 5839.203791000007,
+ "ttftMs": 5990.942082999973,
+ "modelMs": 4929.091833000071,
+ "toolMs": 0.027834000065922737,
+ "totalMs": 6220.881959000137,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2972.5316250000033,
+ "roundMs": 2925.295792000019,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.011624999984633178
+ "execMs": 0.027834000065922737
}
]
},
{
- "roundMs": 1647.9061249999795,
- "firstDeltaMs": 1221.487833000021,
+ "roundMs": 2003.7960410000524,
+ "firstDeltaMs": 1773.4431249999907,
"calls": []
}
]
@@ -9654,27 +9323,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1126,
- "p95": 1174,
- "mean": 1062,
+ "p50": 1950,
+ "p95": 1962,
+ "mean": 1661,
"n": 3
},
"ttftMs": {
- "p50": 3903,
- "p95": 4430,
- "mean": 3902,
+ "p50": 13151,
+ "p95": 13757,
+ "mean": 11478,
"n": 3
},
"totalMs": {
- "p50": 4207,
- "p95": 5161,
- "mean": 4400,
+ "p50": 13527,
+ "p95": 14084,
+ "mean": 11856,
"n": 3
},
"modelMs": {
- "p50": 3081,
- "p95": 3988,
- "mean": 3337,
+ "p50": 11577,
+ "p95": 12122,
+ "mean": 10195,
"n": 3
},
"toolMs": {
@@ -9684,46 +9353,47 @@
"n": 3
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 2.3,
+ "max": 3
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 2821.5138329999754,
- "execMs": 0.39145799999823794
+ "atMs": 8149.161874999991,
+ "execMs": 0.16341699997428805
}
],
"timings": {
- "gateMs": 1173.5396670000046,
+ "gateMs": 1961.6897919999901,
"gateSkipped": false,
- "ttftMs": 4429.524582999991,
- "modelMs": 3987.525917000021,
- "toolMs": 0.39145799999823794,
- "totalMs": 5161.457042000024,
+ "ttftMs": 13757.130000000005,
+ "modelMs": 12121.821250000037,
+ "toolMs": 0.16341699997428805,
+ "totalMs": 14083.674459000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1647.5600420000264,
+ "roundMs": 6186.7967499999795,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.39145799999823794
+ "execMs": 0.16341699997428805
}
]
},
{
- "roundMs": 2339.9658749999944,
- "firstDeltaMs": 1607.573625000019,
+ "roundMs": 5935.024500000058,
+ "firstDeltaMs": 5607.778708000085,
"calls": []
}
]
@@ -9731,39 +9401,56 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
- "name": "generate_scene",
+ "name": "listen_audio",
"round": 0,
- "atMs": 2830.8571250000386,
- "execMs": 0.005249999987427145
+ "atMs": 8106.679832999944,
+ "execMs": 0.024541000020690262
+ },
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 11400.040249999962,
+ "execMs": 0.020500000100582838
}
],
"timings": {
- "gateMs": 1126.0706670000218,
+ "gateMs": 1950.1460829999996,
"gateSkipped": false,
- "ttftMs": 3903.0114170000306,
- "modelMs": 3081.2015000000247,
- "toolMs": 0.005249999987427145,
- "totalMs": 4207.277417000034,
+ "ttftMs": 13150.988332999987,
+ "modelMs": 11576.519168000086,
+ "toolMs": 0.0450410001212731,
+ "totalMs": 13526.710292000207,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1704.23766600003,
+ "roundMs": 6156.05620900006,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.024541000020690262
+ }
+ ]
+ },
+ {
+ "roundMs": 3293.2117500000168,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.005249999987427145
+ "execMs": 0.020500000100582838
}
]
},
{
- "roundMs": 1376.9638339999947,
- "firstDeltaMs": 1072.1294169999892,
+ "roundMs": 2127.251209000009,
+ "firstDeltaMs": 1750.8358749999898,
"calls": []
}
]
@@ -9771,39 +9458,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 2264.615208000003,
- "execMs": 0.04879200004506856
+ "atMs": 5534.1724170000525,
+ "execMs": 0.034875000012107193
}
],
"timings": {
- "gateMs": 887.7749169999734,
+ "gateMs": 1072.5601660000393,
"gateSkipped": false,
- "ttftMs": 3372.8143329999875,
- "modelMs": 2941.9537919999566,
- "toolMs": 0.04879200004506856,
- "totalMs": 3829.777500999975,
+ "ttftMs": 7527.230083000031,
+ "modelMs": 6885.1882920000935,
+ "toolMs": 0.034875000012107193,
+ "totalMs": 7957.783333000145,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1376.3440419999533,
+ "roundMs": 4461.4968339999905,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.04879200004506856
+ "execMs": 0.034875000012107193
}
]
},
{
- "roundMs": 1565.6097500000033,
- "firstDeltaMs": 1108.0701250000275,
+ "roundMs": 2423.691458000103,
+ "firstDeltaMs": 1992.9785000000848,
"calls": []
}
]
@@ -9818,27 +9506,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1159,
- "p95": 1227,
- "mean": 1158,
+ "p50": 1139,
+ "p95": 1418,
+ "mean": 1173,
"n": 3
},
"ttftMs": {
- "p50": 3996,
- "p95": 4207,
- "mean": 4063,
+ "p50": 7591,
+ "p95": 10326,
+ "mean": 8501,
"n": 3
},
"totalMs": {
- "p50": 4661,
- "p95": 4668,
- "mean": 4626,
+ "p50": 8111,
+ "p95": 10608,
+ "mean": 8860,
"n": 3
},
"modelMs": {
- "p50": 3441,
- "p95": 3573,
- "mean": 3468,
+ "p50": 7149,
+ "p95": 9190,
+ "mean": 7687,
"n": 3
},
"toolMs": {
@@ -9855,39 +9543,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 2619.140458000009,
- "execMs": 0.12174999999115244
+ "atMs": 8292.812957999995,
+ "execMs": 0.3355000000447035
}
],
"timings": {
- "gateMs": 1159.372665999981,
+ "gateMs": 1417.7554170000367,
"gateSkipped": false,
- "ttftMs": 3985.4543330000015,
- "modelMs": 3389.534791999962,
- "toolMs": 0.12174999999115244,
- "totalMs": 4549.029207999934,
+ "ttftMs": 10325.98987499997,
+ "modelMs": 9190.293041999917,
+ "toolMs": 0.3355000000447035,
+ "totalMs": 10608.383958999999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1459.5266669999692,
+ "roundMs": 6874.512332999962,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.12174999999115244
+ "execMs": 0.3355000000447035
}
]
},
{
- "roundMs": 1930.008124999993,
- "firstDeltaMs": 1366.1647499999963,
+ "roundMs": 2315.780708999955,
+ "firstDeltaMs": 2032.7514589999337,
"calls": []
}
]
@@ -9895,39 +9584,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 2626.4082499999786,
- "execMs": 0.013000000035390258
+ "atMs": 5553.688291999977,
+ "execMs": 0.05033300002105534
}
],
"timings": {
- "gateMs": 1227.2937920000404,
+ "gateMs": 1138.9273749999702,
"gateSkipped": false,
- "ttftMs": 3995.807541999966,
- "modelMs": 3440.5166249999893,
- "toolMs": 0.013000000035390258,
- "totalMs": 4667.823417000065,
+ "ttftMs": 7586.98520799994,
+ "modelMs": 6720.836958000087,
+ "toolMs": 0.05033300002105534,
+ "totalMs": 7859.814666000078,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1398.8292500000098,
+ "roundMs": 4414.536708,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.013000000035390258
+ "execMs": 0.05033300002105534
}
]
},
{
- "roundMs": 2041.6873749999795,
- "firstDeltaMs": 1369.3764589999919,
+ "roundMs": 2306.3002500000875,
+ "firstDeltaMs": 2033.1560840000166,
"calls": []
}
]
@@ -9935,39 +9625,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 2930.236333000008,
- "execMs": 0.023291999998036772
+ "atMs": 5713.895332999993,
+ "execMs": 0.026540999999269843
}
],
"timings": {
- "gateMs": 1088.3035420000087,
+ "gateMs": 962.205958000035,
"gateSkipped": false,
- "ttftMs": 4207.4956660000025,
- "modelMs": 3572.977917000011,
- "toolMs": 0.023291999998036772,
- "totalMs": 4661.304751000018,
+ "ttftMs": 7591.21091699996,
+ "modelMs": 7148.809501000098,
+ "toolMs": 0.026540999999269843,
+ "totalMs": 8111.042000000132,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1841.7007079999894,
+ "roundMs": 4751.460667000036,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.023291999998036772
+ "execMs": 0.026540999999269843
}
]
},
{
- "roundMs": 1731.2772090000217,
- "firstDeltaMs": 1277.1960839999956,
+ "roundMs": 2397.348834000062,
+ "firstDeltaMs": 1877.2196669999976,
"calls": []
}
]
@@ -9988,21 +9679,21 @@
"n": 3
},
"ttftMs": {
- "p50": 2823,
- "p95": 3180,
- "mean": 2926,
+ "p50": 6508,
+ "p95": 10705,
+ "mean": 7551,
"n": 3
},
"totalMs": {
- "p50": 3184,
- "p95": 3426,
- "mean": 3252,
+ "p50": 6708,
+ "p95": 11250,
+ "mean": 7920,
"n": 3
},
"modelMs": {
- "p50": 3184,
- "p95": 3425,
- "mean": 3252,
+ "p50": 6707,
+ "p95": 11249,
+ "mean": 7920,
"n": 3
},
"toolMs": {
@@ -10019,39 +9710,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 1478.3543749999953,
- "execMs": 0.2778340000077151
+ "atMs": 4732.549500000081,
+ "execMs": 0.2459159999853
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 2774.45541699999,
- "modelMs": 3147.6037920000963,
- "toolMs": 0.2778340000077151,
- "totalMs": 3147.881626000104,
+ "ttftMs": 6508.412416000036,
+ "modelMs": 6707.380000000005,
+ "toolMs": 0.2459159999853,
+ "totalMs": 6707.62591599999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1477.7467500000494,
+ "roundMs": 4732.424875000026,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.2778340000077151
+ "execMs": 0.2459159999853
}
]
},
{
- "roundMs": 1669.857042000047,
- "firstDeltaMs": 1295.7922499999986,
+ "roundMs": 1974.9551249999786,
+ "firstDeltaMs": 1775.5867499999003,
"calls": []
}
]
@@ -10059,39 +9751,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 1840.7548749999842,
- "execMs": 0.03862499998649582
+ "atMs": 9153.968208000064,
+ "execMs": 0.11404100002255291
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3180.3607089999714,
- "modelMs": 3425.4919579999987,
- "toolMs": 0.03862499998649582,
- "totalMs": 3425.530582999985,
+ "ttftMs": 10704.952416999964,
+ "modelMs": 11249.387666000053,
+ "toolMs": 0.11404100002255291,
+ "totalMs": 11249.501707000076,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1840.4343329999829,
+ "roundMs": 9153.629915999947,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.03862499998649582
+ "execMs": 0.11404100002255291
}
]
},
{
- "roundMs": 1585.0576250000158,
- "firstDeltaMs": 1339.5372080000234,
+ "roundMs": 2095.757750000106,
+ "firstDeltaMs": 1550.800375000108,
"calls": []
}
]
@@ -10099,39 +9792,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 1565.6481669999775,
- "execMs": 0.028667000005953014
+ "atMs": 3563.3613749999786,
+ "execMs": 0.03170799999497831
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 2822.831291999959,
- "modelMs": 3183.674665999948,
- "toolMs": 0.028667000005953014,
- "totalMs": 3183.703332999954,
+ "ttftMs": 5438.194541999954,
+ "modelMs": 5803.058915999951,
+ "toolMs": 0.03170799999497831,
+ "totalMs": 5803.090623999946,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1565.4197079999722,
+ "roundMs": 3563.2020000000484,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.028667000005953014
+ "execMs": 0.03170799999497831
}
]
},
{
- "roundMs": 1618.2549579999759,
- "firstDeltaMs": 1257.1191249999683,
+ "roundMs": 2239.8569159999024,
+ "firstDeltaMs": 1874.7358749998966,
"calls": []
}
]
@@ -10142,38 +9836,38 @@
{
"name": "cartoon-question-stays-in-chat",
"bucket": "chat",
- "passRate": 1,
+ "passRate": 0.6666666666666666,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1230,
- "p95": 2078,
- "mean": 1493,
- "n": 3
+ "p50": 1787,
+ "p95": 1797,
+ "mean": 1792,
+ "n": 2
},
"ttftMs": {
- "p50": 4409,
- "p95": 5275,
- "mean": 4329,
- "n": 3
+ "p50": 6908,
+ "p95": 8890,
+ "mean": 7899,
+ "n": 2
},
"totalMs": {
- "p50": 5089,
- "p95": 6300,
- "mean": 5178,
- "n": 3
+ "p50": 7705,
+ "p95": 9391,
+ "mean": 8548,
+ "n": 2
},
"modelMs": {
- "p50": 3918,
- "p95": 5070,
- "mean": 3685,
- "n": 3
+ "p50": 5918,
+ "p95": 7594,
+ "mean": 6756,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 1,
@@ -10183,45 +9877,66 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1229.6249170000083,
+ "gateMs": 1787.1156659999397,
"gateSkipped": false,
- "ttftMs": 5274.793707999983,
- "modelMs": 5070.290959000005,
+ "ttftMs": 6908.266458000056,
+ "modelMs": 5918.2662080000155,
"toolMs": 0,
- "totalMs": 6299.915876000014,
+ "totalMs": 7705.381873999955,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5070.290959000005,
- "firstDeltaMs": 4044.663459000003,
+ "roundMs": 5918.2662080000155,
+ "firstDeltaMs": 5120.705707999994,
"calls": []
}
]
}
},
{
- "pass": true,
- "intent": "work",
- "notes": [],
- "trace": [],
+ "pass": false,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [
+ "mutating tools called: generate_scene"
+ ],
+ "trace": [
+ {
+ "name": "generate_scene",
+ "round": 0,
+ "atMs": 10189.281707999995,
+ "execMs": 0.009833000018261373
+ }
+ ],
"timings": {
- "gateMs": 2077.9729170000064,
+ "gateMs": 1307.4187499999534,
"gateSkipped": false,
- "ttftMs": 3301.988791999989,
- "modelMs": 2068.511708999984,
- "toolMs": 0,
- "totalMs": 4146.48462599999,
+ "ttftMs": 13751.320041000028,
+ "modelMs": 13012.41729100002,
+ "toolMs": 0.009833000018261373,
+ "totalMs": 14319.845873999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2068.511708999984,
- "firstDeltaMs": 1223.4812499999534,
+ "roundMs": 8881.456000000006,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "generate_scene",
+ "execMs": 0.009833000018261373
+ }
+ ]
+ },
+ {
+ "roundMs": 4130.961291000014,
+ "firstDeltaMs": 3561.982874999987,
"calls": []
}
]
@@ -10229,22 +9944,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1170.9177910000435,
+ "gateMs": 1796.6629170000087,
"gateSkipped": false,
- "ttftMs": 4409.353249999986,
- "modelMs": 3917.6903750000056,
+ "ttftMs": 8890.090624999953,
+ "modelMs": 7594.128499999992,
"toolMs": 0,
- "totalMs": 5088.608166000049,
+ "totalMs": 9390.791417,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3917.6903750000056,
- "firstDeltaMs": 3238.3192500000005,
+ "roundMs": 7594.128499999992,
+ "firstDeltaMs": 7093.202499999898,
"calls": []
}
]
@@ -10255,224 +9971,288 @@
{
"name": "clip-mentions-redo-their-shots",
"bucket": "multi-tool",
- "passRate": 1,
+ "passRate": 0.6666666666666666,
"budgetBreaches": [],
"latency": {
"gateMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"ttftMs": {
- "p50": 5969,
- "p95": 6642,
- "mean": 5936,
- "n": 3
+ "p50": 27326,
+ "p95": 29754,
+ "mean": 28540,
+ "n": 2
},
"totalMs": {
- "p50": 6434,
- "p95": 7076,
- "mean": 6363,
- "n": 3
+ "p50": 27930,
+ "p95": 30346,
+ "mean": 29138,
+ "n": 2
},
"modelMs": {
- "p50": 6433,
- "p95": 7075,
- "mean": 6363,
- "n": 3
+ "p50": 27929,
+ "p95": 30345,
+ "mean": 29137,
+ "n": 2
},
"toolMs": {
"p50": 0,
- "p95": 2,
- "mean": 1,
- "n": 3
+ "p95": 1,
+ "mean": 0,
+ "n": 2
},
"rounds": {
- "mean": 3.7,
- "max": 4
+ "mean": 5.5,
+ "max": 6
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 1364.6457500000251,
- "execMs": 1.4009160000132397
+ "atMs": 3787.586250000051,
+ "execMs": 0.5012080000014976
},
{
"name": "watch_video",
"round": 1,
- "atMs": 2763.104541000037,
- "execMs": 0.0413340000086464
+ "atMs": 10398.684833999956,
+ "execMs": 0.028667000005953014
},
{
"name": "watch_video",
"round": 1,
- "atMs": 2763.2028750000172,
- "execMs": 0.0039160000160336494
+ "atMs": 10398.74991699995,
+ "execMs": 0.0023330000694841146
},
{
"name": "regenerate_shot",
"round": 2,
- "atMs": 4663.254750000022,
- "execMs": 0.06912499997997656
+ "atMs": 21957.498499999987,
+ "execMs": 0.07220799999777228
},
{
"name": "regenerate_shot",
"round": 2,
- "atMs": 4663.335458000016,
- "execMs": 0.011750000005122274
+ "atMs": 21957.582124999957,
+ "execMs": 0.01462500006891787
+ },
+ {
+ "name": "wait_for_renders",
+ "round": 3,
+ "atMs": 27425.195042000036,
+ "execMs": 0.0055840000277385116
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5969.1795000000275,
- "modelMs": 6432.828166999971,
- "toolMs": 1.5270410000230186,
- "totalMs": 6434.355207999994,
+ "ttftMs": 29753.92749999999,
+ "modelMs": 30345.497624000185,
+ "toolMs": 0.6246250001713634,
+ "totalMs": 30346.122249000357,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1364.3447910000104,
+ "roundMs": 3786.759291000082,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 1.4009160000132397
+ "execMs": 0.5012080000014976
}
]
},
{
- "roundMs": 1396.8829169999808,
+ "roundMs": 6610.479625000036,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.0413340000086464
+ "execMs": 0.028667000005953014
},
{
"name": "watch_video",
- "execMs": 0.0039160000160336494
+ "execMs": 0.0023330000694841146
}
]
},
{
- "roundMs": 1899.9379590000026,
+ "roundMs": 11558.681749999989,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.06912499997997656
+ "execMs": 0.07220799999777228
},
{
"name": "regenerate_shot",
- "execMs": 0.011750000005122274
+ "execMs": 0.01462500006891787
+ }
+ ]
+ },
+ {
+ "roundMs": 5467.5480830000015,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "wait_for_renders",
+ "execMs": 0.0055840000277385116
}
]
},
{
- "roundMs": 1771.6624999999767,
- "firstDeltaMs": 1305.7965419999673,
+ "roundMs": 2922.0288750000764,
+ "firstDeltaMs": 2328.7157080000034,
"calls": []
}
]
}
},
{
- "pass": true,
- "intent": "work",
- "notes": [],
+ "pass": false,
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
+ "notes": [
+ "mutating tools called: subtitles_generate"
+ ],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 1624.0761249999632,
- "execMs": 0.005000000004656613
+ "atMs": 3499.740125000011,
+ "execMs": 0.016291999956592917
},
{
- "name": "watch_video",
+ "name": "subtitles_generate",
"round": 1,
- "atMs": 3666.1938329999684,
- "execMs": 0.01945800002431497
+ "atMs": 7860.107541000005,
+ "execMs": 0.0510830000275746
},
{
"name": "watch_video",
- "round": 1,
- "atMs": 3666.222832999949,
- "execMs": 0.000708000035956502
+ "round": 2,
+ "atMs": 9926.091040999978,
+ "execMs": 0.0195839999942109
+ },
+ {
+ "name": "get_state",
+ "round": 3,
+ "atMs": 14921.034083000035,
+ "execMs": 0.0332500000949949
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 5514.493749999965,
- "execMs": 0.006167000043205917
+ "round": 4,
+ "atMs": 23348.933958000038,
+ "execMs": 0.026166999945417047
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 5514.50195799995,
- "execMs": 0.0008749999688006938
+ "round": 5,
+ "atMs": 25308.621916000033,
+ "execMs": 0.020874999929219484
+ },
+ {
+ "name": "wait_for_renders",
+ "round": 6,
+ "atMs": 27194.039541000035,
+ "execMs": 0.007665999932214618
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6642.16733299999,
- "modelMs": 7075.494498999964,
- "toolMs": 0.032208000076934695,
- "totalMs": 7075.526707000041,
+ "ttftMs": 29894.101665999973,
+ "modelMs": 30586.517375999945,
+ "toolMs": 0.17491699988022447,
+ "totalMs": 30586.692292999825,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1623.9063329999917,
+ "roundMs": 3499.4688340000575,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.005000000004656613
+ "execMs": 0.016291999956592917
}
]
},
{
- "roundMs": 2042.071707999974,
+ "roundMs": 4360.297874999931,
"firstDeltaMs": null,
"calls": [
{
- "name": "watch_video",
- "execMs": 0.01945800002431497
- },
+ "name": "subtitles_generate",
+ "execMs": 0.0510830000275746
+ }
+ ]
+ },
+ {
+ "roundMs": 2065.854874999961,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "watch_video",
- "execMs": 0.000708000035956502
+ "execMs": 0.0195839999942109
+ }
+ ]
+ },
+ {
+ "roundMs": 4994.8588750000345,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.0332500000949949
}
]
},
{
- "roundMs": 1848.2327499999665,
+ "roundMs": 8427.802125000046,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.006167000043205917
- },
+ "execMs": 0.026166999945417047
+ }
+ ]
+ },
+ {
+ "roundMs": 1959.6031659999862,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "regenerate_shot",
- "execMs": 0.0008749999688006938
+ "execMs": 0.020874999929219484
+ }
+ ]
+ },
+ {
+ "roundMs": 1885.357166999951,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "wait_for_renders",
+ "execMs": 0.007665999932214618
}
]
},
{
- "roundMs": 1561.2837080000318,
- "firstDeltaMs": 1127.6557500000345,
+ "roundMs": 3393.2744589999784,
+ "firstDeltaMs": 2700.023959000013,
"calls": []
}
]
@@ -10480,85 +10260,124 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.5-flash",
"notes": [],
"trace": [
{
- "name": "watch_video",
+ "name": "listen_audio",
"round": 0,
- "atMs": 2180.776708999998,
- "execMs": 0.01945800002431497
+ "atMs": 3482.735209000064,
+ "execMs": 0.021499999915249646
},
{
"name": "watch_video",
- "round": 0,
- "atMs": 2180.8076669999864,
- "execMs": 0.0010419999598525465
+ "round": 1,
+ "atMs": 7850.065792000038,
+ "execMs": 0.012749999994412065
},
{
- "name": "listen_audio",
- "round": 0,
- "atMs": 2180.809666999965,
- "execMs": 0.0006249999860301614
+ "name": "watch_video",
+ "round": 1,
+ "atMs": 7850.083209000062,
+ "execMs": 0.0005840000230818987
+ },
+ {
+ "name": "get_state",
+ "round": 2,
+ "atMs": 10232.509291999973,
+ "execMs": 0.0074159998912364244
},
{
"name": "regenerate_shot",
- "round": 1,
- "atMs": 3930.787249999994,
- "execMs": 0.02808400004869327
+ "round": 3,
+ "atMs": 20381.120875000022,
+ "execMs": 0.027541000046767294
},
{
"name": "regenerate_shot",
- "round": 1,
- "atMs": 3930.824666999979,
- "execMs": 0.0021249999990686774
+ "round": 3,
+ "atMs": 20381.16758400004,
+ "execMs": 0.006249999976716936
+ },
+ {
+ "name": "wait_for_renders",
+ "round": 4,
+ "atMs": 22489.941709000035,
+ "execMs": 0.0051250000251457095
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5196.850541999971,
- "modelMs": 5579.476333000057,
- "toolMs": 0.051334000017959625,
- "totalMs": 5579.527667000075,
+ "ttftMs": 27325.926124999998,
+ "modelMs": 27929.492667999933,
+ "toolMs": 0.08116599987260997,
+ "totalMs": 27929.573833999806,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2180.579458000022,
+ "roundMs": 3482.6494589999784,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.021499999915249646
+ }
+ ]
+ },
+ {
+ "roundMs": 4367.256792000029,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.01945800002431497
+ "execMs": 0.012749999994412065
},
{
"name": "watch_video",
- "execMs": 0.0010419999598525465
- },
+ "execMs": 0.0005840000230818987
+ }
+ ]
+ },
+ {
+ "roundMs": 2382.4038329999894,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "listen_audio",
- "execMs": 0.0006249999860301614
+ "name": "get_state",
+ "execMs": 0.0074159998912364244
}
]
},
{
- "roundMs": 1749.915083000029,
+ "roundMs": 10148.507291999995,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.02808400004869327
+ "execMs": 0.027541000046767294
},
{
"name": "regenerate_shot",
- "execMs": 0.0021249999990686774
+ "execMs": 0.006249999976716936
+ }
+ ]
+ },
+ {
+ "roundMs": 2108.7085840000072,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "wait_for_renders",
+ "execMs": 0.0051250000251457095
}
]
},
{
- "roundMs": 1648.981792000006,
- "firstDeltaMs": 1265.984375,
+ "roundMs": 5439.966707999934,
+ "firstDeltaMs": 4835.957457999932,
"calls": []
}
]
@@ -10569,78 +10388,78 @@
]
},
{
- "label": "gate=flash",
- "chatModel": "gemini-3.5-flash",
- "gateModel": "gemini-3.5-flash",
+ "label": "all-flashLite",
+ "chatModel": "gemini-3.1-flash-lite",
+ "gateModel": "gemini-3.1-flash-lite",
"buckets": {
"chat": {
"cases": 7,
"passRate": 1,
"gateMs": {
- "p50": 1636,
- "p95": 3037,
- "mean": 1381,
+ "p50": 1058,
+ "p95": 1783,
+ "mean": 891,
"n": 7
},
"ttftMs": {
- "p50": 6494,
- "p95": 8336,
- "mean": 5636,
+ "p50": 2106,
+ "p95": 2946,
+ "mean": 2091,
"n": 7
},
"totalMs": {
- "p50": 6881,
- "p95": 9017,
- "mean": 6012,
+ "p50": 2569,
+ "p95": 3650,
+ "mean": 2595,
"n": 7
},
- "meanRounds": 1.1
+ "meanRounds": 1
},
"multi-tool": {
"cases": 4,
- "passRate": 0.92,
+ "passRate": 0.75,
"gateMs": {
"p50": 0,
- "p95": 1816,
- "mean": 865,
+ "p95": 1183,
+ "mean": 568,
"n": 4
},
"ttftMs": {
- "p50": 25170,
- "p95": 90706,
- "mean": 39430,
+ "p50": 4619,
+ "p95": 5880,
+ "mean": 4725,
"n": 4
},
"totalMs": {
- "p50": 25461,
- "p95": 91406,
- "mean": 39900,
+ "p50": 5037,
+ "p95": 6156,
+ "mean": 5101,
"n": 4
},
- "meanRounds": 8.3
+ "meanRounds": 2.8
},
"single-tool": {
"cases": 15,
- "passRate": 1,
+ "passRate": 0.96,
"gateMs": {
- "p50": 1752,
- "p95": 1997,
- "mean": 1665,
+ "p50": 1175,
+ "p95": 1645,
+ "mean": 1111,
"n": 15
},
"ttftMs": {
- "p50": 8036,
- "p95": 24656,
- "mean": 9851,
+ "p50": 4165,
+ "p95": 4499,
+ "mean": 4071,
"n": 15
},
"totalMs": {
- "p50": 8132,
- "p95": 17296,
- "mean": 9071,
+ "p50": 4630,
+ "p95": 5155,
+ "mean": 4582,
"n": 15
},
- "meanRounds": 2.2
+ "meanRounds": 2
}
},
"cases": [
@@ -10657,21 +10476,21 @@
"n": 3
},
"ttftMs": {
- "p50": 6554,
- "p95": 6561,
- "mean": 5639,
+ "p50": 1696,
+ "p95": 5233,
+ "mean": 2811,
"n": 3
},
"totalMs": {
- "p50": 6882,
- "p95": 7004,
- "mean": 5931,
+ "p50": 1895,
+ "p95": 5427,
+ "mean": 3011,
"n": 3
},
"modelMs": {
- "p50": 6882,
- "p95": 7004,
- "mean": 5931,
+ "p50": 1895,
+ "p95": 5427,
+ "mean": 3011,
"n": 3
},
"toolMs": {
@@ -10681,29 +10500,30 @@
"n": 3
},
"rounds": {
- "mean": 1.3,
- "max": 2
+ "mean": 1,
+ "max": 1
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6553.993459,
- "modelMs": 6881.569875,
+ "ttftMs": 5233.0578749999995,
+ "modelMs": 5426.661417,
"toolMs": 0,
- "totalMs": 6881.569875,
+ "totalMs": 5426.661417,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6881.569875,
- "firstDeltaMs": 6553.588457999999,
+ "roundMs": 5426.661417,
+ "firstDeltaMs": 5232.658292,
"calls": []
}
]
@@ -10711,22 +10531,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3803.4759999999997,
- "modelMs": 3908.5944580000005,
+ "ttftMs": 1695.573375,
+ "modelMs": 1894.9740000000002,
"toolMs": 0,
- "totalMs": 3908.5944580000005,
+ "totalMs": 1894.9740000000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3908.5944580000005,
- "firstDeltaMs": 3803.301375,
+ "roundMs": 1894.9740000000002,
+ "firstDeltaMs": 1695.3623340000004,
"calls": []
}
]
@@ -10734,39 +10555,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 4033.1577909999996,
- "execMs": 0.1134169999986625
- }
- ],
+ "trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6560.917083,
- "modelMs": 7003.768001,
- "toolMs": 0.1134169999986625,
- "totalMs": 7003.881417999999,
+ "ttftMs": 1503.856040999999,
+ "modelMs": 1712.5314169999992,
+ "toolMs": 0,
+ "totalMs": 1712.5314169999992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4031.3700420000005,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.1134169999986625
- }
- ]
- },
- {
- "roundMs": 2972.397959,
- "firstDeltaMs": 2526.513334000001,
+ "roundMs": 1712.5314169999992,
+ "firstDeltaMs": 1503.7868339999995,
"calls": []
}
]
@@ -10781,27 +10586,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1450,
- "p95": 1518,
- "mean": 1449,
+ "p50": 1278,
+ "p95": 1591,
+ "mean": 1314,
"n": 3
},
"ttftMs": {
- "p50": 3180,
- "p95": 3294,
- "mean": 3157,
+ "p50": 2074,
+ "p95": 2481,
+ "mean": 2157,
"n": 3
},
"totalMs": {
- "p50": 3212,
- "p95": 3313,
- "mean": 3225,
+ "p50": 2223,
+ "p95": 2539,
+ "mean": 2274,
"n": 3
},
"modelMs": {
- "p50": 1762,
- "p95": 1935,
- "mean": 1776,
+ "p50": 949,
+ "p95": 986,
+ "mean": 960,
"n": 3
},
"toolMs": {
@@ -10819,21 +10624,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1450.1936670000032,
+ "gateMs": 1278.3665419999998,
"gateSkipped": false,
- "ttftMs": 3179.778957999999,
- "modelMs": 1761.9570830000011,
+ "ttftMs": 2074.461416,
+ "modelMs": 944.3923750000013,
"toolMs": 0,
- "totalMs": 3212.1507500000043,
+ "totalMs": 2222.758917000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1761.9570830000011,
- "firstDeltaMs": 1729.1647909999992,
+ "roundMs": 944.3923750000013,
+ "firstDeltaMs": 795.0627080000013,
"calls": []
}
]
@@ -10842,21 +10648,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1518.317665999999,
+ "gateMs": 1074.3056670000005,
"gateSkipped": false,
- "ttftMs": 2995.629417,
- "modelMs": 1631.0322919999999,
+ "ttftMs": 1915.8132910000004,
+ "modelMs": 986.028459000001,
"toolMs": 0,
- "totalMs": 3149.349957999999,
+ "totalMs": 2060.3341260000016,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1631.0322919999999,
- "firstDeltaMs": 1477.0789590000022,
+ "roundMs": 986.028459000001,
+ "firstDeltaMs": 841.4192500000008,
"calls": []
}
]
@@ -10865,21 +10672,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1377.120500000001,
+ "gateMs": 1590.5742920000012,
"gateSkipped": false,
- "ttftMs": 3294.1238749999975,
- "modelMs": 1935.4827910000022,
+ "ttftMs": 2480.8764169999995,
+ "modelMs": 948.8515409999982,
"toolMs": 0,
- "totalMs": 3312.603291000003,
+ "totalMs": 2539.4258329999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1935.4827910000022,
- "firstDeltaMs": 1916.6697910000003,
+ "roundMs": 948.8515409999982,
+ "firstDeltaMs": 890.1923330000009,
"calls": []
}
]
@@ -10894,27 +10702,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1684,
- "p95": 1721,
- "mean": 1660,
+ "p50": 1058,
+ "p95": 1156,
+ "mean": 1088,
"n": 3
},
"ttftMs": {
- "p50": 3478,
- "p95": 3830,
- "mean": 3595,
+ "p50": 2157,
+ "p95": 2563,
+ "mean": 2215,
"n": 3
},
"totalMs": {
- "p50": 3782,
- "p95": 3962,
- "mean": 3765,
+ "p50": 2689,
+ "p95": 3259,
+ "mean": 2797,
"n": 3
},
"modelMs": {
- "p50": 2205,
- "p95": 2279,
- "mean": 2104,
+ "p50": 1638,
+ "p95": 2103,
+ "mean": 1709,
"n": 3
},
"toolMs": {
@@ -10932,21 +10740,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1683.5990840000013,
+ "gateMs": 1156.0989160000026,
"gateSkipped": false,
- "ttftMs": 3829.7804160000014,
- "modelMs": 2278.622374999999,
+ "ttftMs": 2562.5304170000018,
+ "modelMs": 2102.8871670000008,
"toolMs": 0,
- "totalMs": 3962.2214590000003,
+ "totalMs": 3258.9860830000034,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2278.622374999999,
- "firstDeltaMs": 2145.6878749999996,
+ "roundMs": 2102.8871670000008,
+ "firstDeltaMs": 1406.158459000002,
"calls": []
}
]
@@ -10955,21 +10764,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1576.3630410000005,
+ "gateMs": 1057.8520419999986,
"gateSkipped": false,
- "ttftMs": 3477.6224999999977,
- "modelMs": 2205.2803330000024,
+ "ttftMs": 1926.8231669999986,
+ "modelMs": 1385.068374999999,
"toolMs": 0,
- "totalMs": 3781.643374000003,
+ "totalMs": 2442.9204169999975,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2205.2803330000024,
- "firstDeltaMs": 1901.174291000003,
+ "roundMs": 1385.068374999999,
+ "firstDeltaMs": 868.893082999999,
"calls": []
}
]
@@ -10978,21 +10788,22 @@
{
"pass": true,
"intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1720.8931250000023,
+ "gateMs": 1050.3534580000014,
"gateSkipped": false,
- "ttftMs": 3477.750874999998,
- "modelMs": 1829.3851670000004,
+ "ttftMs": 2156.516125000002,
+ "modelMs": 1638.2501249999987,
"toolMs": 0,
- "totalMs": 3550.2782920000027,
+ "totalMs": 2688.603583,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1829.3851670000004,
- "firstDeltaMs": 1756.6495419999992,
+ "roundMs": 1638.2501249999987,
+ "firstDeltaMs": 1105.935207999999,
"calls": []
}
]
@@ -11003,38 +10814,38 @@
{
"name": "follow-up-do-it-works",
"bucket": "multi-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1816,
- "p95": 2667,
- "mean": 2242,
- "n": 2
+ "p50": 1090,
+ "p95": 1834,
+ "mean": 1327,
+ "n": 3
},
"ttftMs": {
- "p50": 10776,
- "p95": 13181,
- "mean": 11979,
- "n": 2
+ "p50": 4619,
+ "p95": 4873,
+ "mean": 4684,
+ "n": 3
},
"totalMs": {
- "p50": 11234,
- "p95": 13576,
- "mean": 12405,
- "n": 2
+ "p50": 5037,
+ "p95": 5277,
+ "mean": 5055,
+ "n": 3
},
"modelMs": {
- "p50": 8566,
- "p95": 11760,
- "mean": 10163,
- "n": 2
+ "p50": 3947,
+ "p95": 4219,
+ "mean": 3728,
+ "n": 3
},
"toolMs": {
"p50": 0,
"p95": 2,
"mean": 1,
- "n": 2
+ "n": 3
},
"rounds": {
"mean": 2,
@@ -11044,107 +10855,212 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 9040.397917000002,
- "execMs": 1.4233750000057626
+ "atMs": 3360.024375000001,
+ "execMs": 1.3826250000020082
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 3361.4427079999987,
+ "execMs": 0.01320800000030431
},
{
"name": "delete_cue",
"round": 0,
- "atMs": 9041.857000000004,
- "execMs": 0.10725000000093132
+ "atMs": 3361.4623329999995,
+ "execMs": 0.16870799999742303
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 3361.636208,
+ "execMs": 0.013083000001643086
}
],
"timings": {
- "gateMs": 2667.2535840000055,
+ "gateMs": 1057.0134999999973,
"gateSkipped": false,
- "ttftMs": 10776.346250000002,
- "modelMs": 8565.634916000003,
- "toolMs": 1.5306250000066939,
- "totalMs": 11234.419125000015,
+ "ttftMs": 4873.178958,
+ "modelMs": 4218.586542000001,
+ "toolMs": 1.5776240000013786,
+ "totalMs": 5277.177666,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6372.156708000002,
+ "roundMs": 2301.7727080000004,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 1.4233750000057626
+ "execMs": 1.3826250000020082
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.01320800000030431
},
{
"name": "delete_cue",
- "execMs": 0.10725000000093132
+ "execMs": 0.16870799999742303
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.013083000001643086
}
]
},
{
- "roundMs": 2193.4782080000004,
- "firstDeltaMs": 1734.3452499999985,
+ "roundMs": 1916.8138340000005,
+ "firstDeltaMs": 1511.4897090000013,
"calls": []
}
]
}
},
{
- "pass": false,
- "intent": "work",
+ "pass": true,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "trace": [],
- "timings": null,
- "error": "got status: RESOURCE_EXHAUSTED. {\"error\":{\"code\":429,\"message\":\"Resource exhausted. Please try again later. Please refer to https://cloud.google.com/vertex-ai/generative-ai/docs/error-code-429 for more details.\",\"status\":\"RESOURCE_EXHAUSTED\"}}"
+ "trace": [
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 3389.5024999999987,
+ "execMs": 0.029249999999592546
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 3389.537542000002,
+ "execMs": 0.007915999995020684
+ },
+ {
+ "name": "delete_cue",
+ "round": 0,
+ "atMs": 3389.547416999998,
+ "execMs": 0.04645800000434974
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 3389.5963749999974,
+ "execMs": 0.005834000003233086
+ }
+ ],
+ "timings": {
+ "gateMs": 1089.7129160000004,
+ "gateSkipped": false,
+ "ttftMs": 4618.9595420000005,
+ "modelMs": 3947.093375000004,
+ "toolMs": 0.08945800000219606,
+ "totalMs": 5036.895749000007,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 2299.528542,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_cue",
+ "execMs": 0.029249999999592546
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.007915999995020684
+ },
+ {
+ "name": "delete_cue",
+ "execMs": 0.04645800000434974
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.005834000003233086
+ }
+ ]
+ },
+ {
+ "roundMs": 1647.564833000004,
+ "firstDeltaMs": 1229.3279170000023,
+ "calls": []
+ }
+ ]
+ }
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 11168.724874999993,
- "execMs": 0.1386250000068685
+ "atMs": 3433.824042,
+ "execMs": 0.037167000002227724
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 3433.8698749999967,
+ "execMs": 0.014874999993480742
+ },
+ {
+ "name": "delete_cue",
+ "round": 0,
+ "atMs": 3433.887999999999,
+ "execMs": 0.02295800000138115
},
{
"name": "update_cue",
"round": 0,
- "atMs": 11168.894624999994,
- "execMs": 0.01504199999908451
+ "atMs": 3433.913542000002,
+ "execMs": 0.006499999995867256
}
],
"timings": {
- "gateMs": 1816.4701249999998,
+ "gateMs": 1833.999291,
"gateSkipped": false,
- "ttftMs": 13181.092834000003,
- "modelMs": 11759.816373999987,
- "toolMs": 0.15366700000595301,
- "totalMs": 13576.440165999993,
+ "ttftMs": 4560.632207999995,
+ "modelMs": 3017.7608340000006,
+ "toolMs": 0.08149999999295687,
+ "totalMs": 4851.841624999994,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 9351.626957999993,
+ "roundMs": 1599.6392090000008,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.1386250000068685
+ "execMs": 0.037167000002227724
},
{
"name": "update_cue",
- "execMs": 0.01504199999908451
+ "execMs": 0.014874999993480742
+ },
+ {
+ "name": "delete_cue",
+ "execMs": 0.02295800000138115
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.006499999995867256
}
]
},
{
- "roundMs": 2408.189415999994,
- "firstDeltaMs": 2012.133415999997,
+ "roundMs": 1418.1216249999998,
+ "firstDeltaMs": 1126.6768750000047,
"calls": []
}
]
@@ -11159,27 +11075,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1636,
- "p95": 1963,
- "mean": 1733,
+ "p50": 1051,
+ "p95": 1157,
+ "mean": 1030,
"n": 3
},
"ttftMs": {
- "p50": 3856,
- "p95": 4138,
- "mean": 3935,
+ "p50": 2158,
+ "p95": 2323,
+ "mean": 2209,
"n": 3
},
"totalMs": {
- "p50": 4267,
- "p95": 4267,
- "mean": 4172,
+ "p50": 2390,
+ "p95": 2588,
+ "mean": 2456,
"n": 3
},
"modelMs": {
- "p50": 2381,
- "p95": 2631,
- "mean": 2439,
+ "p50": 1430,
+ "p95": 1510,
+ "mean": 1426,
"n": 3
},
"toolMs": {
@@ -11196,22 +11112,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1636.1342910000094,
+ "gateMs": 1157.4879170000058,
"gateSkipped": false,
- "ttftMs": 3855.526417000001,
- "modelMs": 2631.021959000005,
+ "ttftMs": 2323.203708000001,
+ "modelMs": 1430.1306660000046,
"toolMs": 0,
- "totalMs": 4267.156250000015,
+ "totalMs": 2587.6185830000104,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2631.021959000005,
- "firstDeltaMs": 2218.798416999998,
+ "roundMs": 1430.1306660000046,
+ "firstDeltaMs": 1165.3310409999976,
"calls": []
}
]
@@ -11219,22 +11136,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1962.840790999995,
+ "gateMs": 879.8641250000001,
"gateSkipped": false,
- "ttftMs": 4137.775666999994,
- "modelMs": 2303.754666000008,
+ "ttftMs": 2145.2822499999966,
+ "modelMs": 1509.6476669999975,
"toolMs": 0,
- "totalMs": 4266.595457000003,
+ "totalMs": 2389.5117919999975,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2303.754666000008,
- "firstDeltaMs": 2174.5667079999985,
+ "roundMs": 1509.6476669999975,
+ "firstDeltaMs": 1265.1558749999967,
"calls": []
}
]
@@ -11242,22 +11160,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1600.5205419999984,
+ "gateMs": 1051.2344589999993,
"gateSkipped": false,
- "ttftMs": 3811.5333329999994,
- "modelMs": 2381.0375829999975,
+ "ttftMs": 2158.274375000001,
+ "modelMs": 1338.4919169999994,
"toolMs": 0,
- "totalMs": 3981.558124999996,
+ "totalMs": 2389.7263759999987,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2381.0375829999975,
- "firstDeltaMs": 2210.835249999989,
+ "roundMs": 1338.4919169999994,
+ "firstDeltaMs": 1106.8992079999953,
"calls": []
}
]
@@ -11268,392 +11187,301 @@
{
"name": "photos-request-places-clips",
"bucket": "multi-tool",
- "passRate": 1,
+ "passRate": 0.3333333333333333,
"budgetBreaches": [],
"latency": {
"gateMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 1
},
"ttftMs": {
- "p50": 31066,
- "p95": 60554,
- "mean": 37145,
- "n": 3
+ "p50": 3135,
+ "p95": 3135,
+ "mean": 3135,
+ "n": 1
},
"totalMs": {
- "p50": 31498,
- "p95": 60895,
- "mean": 37611,
- "n": 3
+ "p50": 3660,
+ "p95": 3660,
+ "mean": 3660,
+ "n": 1
},
"modelMs": {
- "p50": 31498,
- "p95": 60895,
- "mean": 37611,
- "n": 3
+ "p50": 3660,
+ "p95": 3660,
+ "mean": 3660,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 1
},
"rounds": {
- "mean": 6,
- "max": 8
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 4970.850375000009,
- "execMs": 0.00416699999186676
+ "atMs": 2078.2273339999956,
+ "execMs": 0.002458999995724298
},
{
"name": "add_clip",
"round": 0,
- "atMs": 4970.880375000008,
- "execMs": 0.007750000004307367
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 6771.416750000004,
- "execMs": 0.10012499999720603
- },
- {
- "name": "get_state",
- "round": 2,
- "atMs": 10971.187584,
- "execMs": 0.046208999992813915
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 15031.858124999999,
- "execMs": 0.00900000000547152
- },
- {
- "name": "get_state",
- "round": 4,
- "atMs": 21490.511792000005,
- "execMs": 0.06787499999336433
- },
- {
- "name": "add_clip",
- "round": 5,
- "atMs": 27530.899292000002,
- "execMs": 0.027041999986977316
+ "atMs": 2078.242333999995,
+ "execMs": 0.004041999993205536
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 31065.657709000006,
- "modelMs": 31497.855417,
- "toolMs": 0.26216799997200724,
- "totalMs": 31498.11758499997,
+ "ttftMs": 3135.497083999995,
+ "modelMs": 3660.467499999999,
+ "toolMs": 0.006500999988929834,
+ "totalMs": 3660.4740009999878,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4969.851750000002,
+ "roundMs": 2077.888875000004,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.00416699999186676
+ "execMs": 0.002458999995724298
},
{
"name": "add_clip",
- "execMs": 0.007750000004307367
- }
- ]
- },
- {
- "roundMs": 1800.367459000001,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.10012499999720603
- }
- ]
- },
- {
- "roundMs": 4199.562791999997,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.046208999992813915
- }
- ]
- },
- {
- "roundMs": 4060.3923330000107,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.00900000000547152
- }
- ]
- },
- {
- "roundMs": 6458.46149999999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.06787499999336433
- }
- ]
- },
- {
- "roundMs": 6040.210458000001,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.027041999986977316
+ "execMs": 0.004041999993205536
}
]
},
{
- "roundMs": 3969.009124999997,
- "firstDeltaMs": 3534.6786250000005,
+ "roundMs": 1582.5786249999946,
+ "firstDeltaMs": 1057.237957999998,
"calls": []
}
]
}
},
{
- "pass": true,
- "intent": "work",
- "notes": [],
+ "pass": false,
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [
+ "mutating tools called: trim_clip"
+ ],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 6620.947125000006,
- "execMs": 0.009208000003127381
+ "atMs": 1308.9524579999998,
+ "execMs": 0.0017080000034184195
},
{
"name": "add_clip",
"round": 0,
- "atMs": 6621.012499999997,
- "execMs": 0.0186249999969732
+ "atMs": 1308.9627499999988,
+ "execMs": 0.00016599999798927456
},
{
- "name": "get_state",
+ "name": "add_clip",
"round": 1,
- "atMs": 21638.826375000004,
- "execMs": 0.041250000009313226
+ "atMs": 3456.088082999995,
+ "execMs": 0.0009579999968991615
},
{
"name": "add_clip",
- "round": 2,
- "atMs": 31520.657540999993,
- "execMs": 0.004333999997470528
+ "round": 1,
+ "atMs": 3456.091790999999,
+ "execMs": 0
},
{
- "name": "add_clip",
+ "name": "trim_clip",
"round": 2,
- "atMs": 31520.671875,
- "execMs": 0.0002919999824371189
- },
- {
- "name": "get_state",
- "round": 3,
- "atMs": 38239.557415999996,
- "execMs": 0.038957999990088865
+ "atMs": 4696.507291000002,
+ "execMs": 0.0062499999985448085
},
{
"name": "add_clip",
- "round": 4,
- "atMs": 43779.74645800001,
- "execMs": 0.0037919999740552157
+ "round": 2,
+ "atMs": 4696.515790999998,
+ "execMs": 0.000375000003259629
},
{
"name": "add_clip",
- "round": 5,
- "atMs": 45789.261541000014,
- "execMs": 0.0032500000088475645
+ "round": 3,
+ "atMs": 6492.464457999995,
+ "execMs": 0.0032920000012381934
},
{
- "name": "get_state",
- "round": 6,
- "atMs": 49257.09570800001,
- "execMs": 0.013291000010212883
+ "name": "add_clip",
+ "round": 3,
+ "atMs": 6492.476582999996,
+ "execMs": 8.400000660913065e-05
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 60553.840333,
- "modelMs": 60895.13366600002,
- "toolMs": 0.13299999997252598,
- "totalMs": 60895.266665999996,
+ "ttftMs": 7703.1548330000005,
+ "modelMs": 8066.906790999987,
+ "toolMs": 0.012833000007958617,
+ "totalMs": 8066.919623999995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6620.468792,
+ "roundMs": 1308.6607079999958,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.009208000003127381
+ "execMs": 0.0017080000034184195
},
{
"name": "add_clip",
- "execMs": 0.0186249999969732
- }
- ]
- },
- {
- "roundMs": 15017.656249999985,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.041250000009313226
+ "execMs": 0.00016599999798927456
}
]
},
{
- "roundMs": 9881.699791000021,
+ "roundMs": 2147.0864169999986,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.004333999997470528
+ "execMs": 0.0009579999968991615
},
{
"name": "add_clip",
- "execMs": 0.0002919999824371189
+ "execMs": 0
}
]
},
{
- "roundMs": 6718.786457999988,
+ "roundMs": 1240.3925830000007,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.038957999990088865
- }
- ]
- },
- {
- "roundMs": 5540.06904100001,
- "firstDeltaMs": null,
- "calls": [
+ "name": "trim_clip",
+ "execMs": 0.0062499999985448085
+ },
{
"name": "add_clip",
- "execMs": 0.0037919999740552157
+ "execMs": 0.000375000003259629
}
]
},
{
- "roundMs": 2009.4302090000128,
+ "roundMs": 1795.9077499999985,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.0032500000088475645
- }
- ]
- },
- {
- "roundMs": 3467.7318749999977,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.0032920000012381934
+ },
{
- "name": "get_state",
- "execMs": 0.013291000010212883
+ "name": "add_clip",
+ "execMs": 8.400000660913065e-05
}
]
},
{
- "roundMs": 11639.29125000001,
- "firstDeltaMs": 11296.689417000016,
+ "roundMs": 1574.859332999993,
+ "firstDeltaMs": 1210.6317499999932,
"calls": []
}
]
}
},
{
- "pass": true,
- "intent": "work",
- "notes": [],
+ "pass": false,
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [
+ "mutating tools called: place_clip, place_clip"
+ ],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 4571.313040999987,
- "execMs": 0.005084000003989786
+ "atMs": 2043.1788329999981,
+ "execMs": 0.0030000000042491592
},
{
"name": "add_clip",
"round": 0,
- "atMs": 4571.335374999995,
- "execMs": 0.008082999993348494
+ "atMs": 2043.190042000002,
+ "execMs": 0.00016699999832781032
},
{
- "name": "get_state",
+ "name": "place_clip",
+ "round": 1,
+ "atMs": 6843.212375000003,
+ "execMs": 0.06037500000093132
+ },
+ {
+ "name": "place_clip",
"round": 1,
- "atMs": 6650.7935,
- "execMs": 0.04099999999743886
+ "atMs": 6843.283208000008,
+ "execMs": 0.005000000004656613
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 19815.25433299999,
- "modelMs": 20439.472457999975,
- "toolMs": 0.05416699999477714,
- "totalMs": 20439.52662499997,
+ "ttftMs": 7877.5503330000065,
+ "modelMs": 8188.838833000002,
+ "toolMs": 0.0685420000081649,
+ "totalMs": 8188.90737500001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4571.169457999989,
+ "roundMs": 2043.0927910000028,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.005084000003989786
+ "execMs": 0.0030000000042491592
},
{
"name": "add_clip",
- "execMs": 0.008082999993348494
+ "execMs": 0.00016699999832781032
}
]
},
{
- "roundMs": 2079.382332999987,
+ "roundMs": 4799.960416999995,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.04099999999743886
+ "name": "place_clip",
+ "execMs": 0.06037500000093132
+ },
+ {
+ "name": "place_clip",
+ "execMs": 0.005000000004656613
}
]
},
{
- "roundMs": 13788.920666999999,
- "firstDeltaMs": 13164.37741700001,
+ "roundMs": 1345.785625000004,
+ "firstDeltaMs": 1034.2303749999992,
"calls": []
}
]
@@ -11674,21 +11502,21 @@
"n": 3
},
"ttftMs": {
- "p50": 6494,
- "p95": 13936,
- "mean": 8171,
+ "p50": 1499,
+ "p95": 1526,
+ "mean": 1383,
"n": 3
},
"totalMs": {
- "p50": 6881,
- "p95": 15231,
- "mean": 8963,
+ "p50": 2569,
+ "p95": 3162,
+ "mean": 2553,
"n": 3
},
"modelMs": {
- "p50": 6881,
- "p95": 15231,
- "mean": 8963,
+ "p50": 2569,
+ "p95": 3162,
+ "mean": 2553,
"n": 3
},
"toolMs": {
@@ -11705,22 +11533,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6493.960208999983,
- "modelMs": 6880.53012499999,
+ "ttftMs": 1525.8710419999989,
+ "modelMs": 3162.3151249999937,
"toolMs": 0,
- "totalMs": 6880.53012499999,
+ "totalMs": 3162.3151249999937,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6880.53012499999,
- "firstDeltaMs": 6493.854292000004,
+ "roundMs": 3162.3151249999937,
+ "firstDeltaMs": 1525.7886670000007,
"calls": []
}
]
@@ -11728,22 +11557,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 13935.77008300001,
- "modelMs": 15230.621458000009,
+ "ttftMs": 1499.2625419999968,
+ "modelMs": 2568.514875000008,
"toolMs": 0,
- "totalMs": 15230.621458000009,
+ "totalMs": 2568.514875000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 15230.621458000009,
- "firstDeltaMs": 13935.67975000001,
+ "roundMs": 2568.514875000008,
+ "firstDeltaMs": 1499.0327090000064,
"calls": []
}
]
@@ -11751,22 +11581,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4084.7394170000043,
- "modelMs": 4777.928291000018,
+ "ttftMs": 1123.5997500000085,
+ "modelMs": 1928.023749999993,
"toolMs": 0,
- "totalMs": 4777.928291000018,
+ "totalMs": 1928.023749999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4777.928291000018,
- "firstDeltaMs": 4084.4794160000165,
+ "roundMs": 1928.023749999993,
+ "firstDeltaMs": 1123.3492499999993,
"calls": []
}
]
@@ -11781,27 +11612,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 3037,
- "p95": 4141,
- "mean": 3079,
+ "p50": 1065,
+ "p95": 1740,
+ "mean": 1275,
"n": 3
},
"ttftMs": {
- "p50": 7554,
- "p95": 8352,
- "mean": 7147,
+ "p50": 2106,
+ "p95": 2455,
+ "mean": 2160,
"n": 3
},
"totalMs": {
- "p50": 8040,
- "p95": 8969,
- "mean": 7799,
+ "p50": 2748,
+ "p95": 2971,
+ "mean": 2713,
"n": 3
},
"modelMs": {
- "p50": 4828,
- "p95": 5003,
- "mean": 4720,
+ "p50": 1401,
+ "p95": 1683,
+ "mean": 1438,
"n": 3
},
"toolMs": {
@@ -11818,22 +11649,23 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2058.411542000016,
+ "gateMs": 1739.6398340000014,
"gateSkipped": false,
- "ttftMs": 5535.561707999994,
- "modelMs": 4329.836459000013,
+ "ttftMs": 2454.5032499999943,
+ "modelMs": 1230.9142500000016,
"toolMs": 0,
- "totalMs": 6388.248001000029,
+ "totalMs": 2970.554084000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4329.836459000013,
- "firstDeltaMs": 3475.448375000007,
+ "roundMs": 1230.9142500000016,
+ "firstDeltaMs": 714.6299169999984,
"calls": []
}
]
@@ -11841,22 +11673,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 3036.940042000002,
+ "gateMs": 1020.6809170000051,
"gateSkipped": false,
- "ttftMs": 7554.012458000012,
- "modelMs": 5002.6289169999945,
+ "ttftMs": 1920.2747079999972,
+ "modelMs": 1400.587290999989,
"toolMs": 0,
- "totalMs": 8039.5689589999965,
+ "totalMs": 2421.268207999994,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5002.6289169999945,
- "firstDeltaMs": 4516.811709000001,
+ "roundMs": 1400.587290999989,
+ "firstDeltaMs": 899.4720410000009,
"calls": []
}
]
@@ -11864,22 +11697,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "chat",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 4141.091959000012,
+ "gateMs": 1065.320875000005,
"gateSkipped": false,
- "ttftMs": 8351.846124999982,
- "modelMs": 4827.7582500000135,
+ "ttftMs": 2106.048582999996,
+ "modelMs": 1682.7050000000017,
"toolMs": 0,
- "totalMs": 8968.850209000026,
+ "totalMs": 2748.0258750000066,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4827.7582500000135,
- "firstDeltaMs": 4210.412249999994,
+ "roundMs": 1682.7050000000017,
+ "firstDeltaMs": 1040.5430420000048,
"calls": []
}
]
@@ -11891,32 +11725,30 @@
"name": "video-ask-single-generate",
"bucket": "single-tool",
"passRate": 1,
- "budgetBreaches": [
- "totalMs p50 17296 > budget 14000"
- ],
+ "budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1997,
- "p95": 7914,
- "mean": 3789,
+ "p50": 910,
+ "p95": 1855,
+ "mean": 1200,
"n": 3
},
"ttftMs": {
- "p50": 16693,
- "p95": 17748,
- "mean": 14499,
+ "p50": 4499,
+ "p95": 4524,
+ "mean": 4475,
"n": 3
},
"totalMs": {
- "p50": 17296,
- "p95": 18101,
- "mean": 14949,
+ "p50": 4840,
+ "p95": 5179,
+ "mean": 4920,
"n": 3
},
"modelMs": {
- "p50": 9382,
- "p95": 16644,
- "mean": 11159,
+ "p50": 3832,
+ "p95": 4006,
+ "mean": 3721,
"n": 3
},
"toolMs": {
@@ -11933,39 +11765,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 15079.543624999991,
- "execMs": 0.03379200003109872
+ "atMs": 3291.438458000004,
+ "execMs": 0.02841699999407865
}
],
"timings": {
- "gateMs": 7913.903500000015,
+ "gateMs": 833.7955409999995,
"gateSkipped": false,
- "ttftMs": 16692.50816700002,
- "modelMs": 9382.425125000009,
- "toolMs": 0.03379200003109872,
- "totalMs": 17296.362417000055,
+ "ttftMs": 4499.444541999997,
+ "modelMs": 4006.2011670000065,
+ "toolMs": 0.02841699999407865,
+ "totalMs": 4840.025125,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7165.328666999994,
+ "roundMs": 2457.412292000008,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.03379200003109872
+ "execMs": 0.02841699999407865
}
]
},
{
- "roundMs": 2217.0964580000145,
- "firstDeltaMs": 1612.8977500000037,
+ "roundMs": 1548.7888749999984,
+ "firstDeltaMs": 1207.9414580000011,
"calls": []
}
]
@@ -11973,39 +11806,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 7980.800208000001,
- "execMs": 0.008042000001296401
+ "atMs": 3217.008209000007,
+ "execMs": 0.0026249999937135726
}
],
"timings": {
- "gateMs": 1457.2664999999688,
+ "gateMs": 1855.302874999994,
"gateSkipped": false,
- "ttftMs": 17747.96749999997,
- "modelMs": 16644.09391699999,
- "toolMs": 0.008042000001296401,
- "totalMs": 18101.36845899996,
+ "ttftMs": 4523.856459000002,
+ "modelMs": 3323.7346249999973,
+ "toolMs": 0.0026249999937135726,
+ "totalMs": 5179.040124999985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6523.419374999998,
+ "roundMs": 1361.4744999999966,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.008042000001296401
+ "execMs": 0.0026249999937135726
}
]
},
{
- "roundMs": 10120.674541999993,
- "firstDeltaMs": 9767.120292000007,
+ "roundMs": 1962.2601250000007,
+ "firstDeltaMs": 1306.8319579999952,
"calls": []
}
]
@@ -12013,39 +11847,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 7482.108540999994,
- "execMs": 0.012667000002693385
+ "atMs": 3211.7446659999987,
+ "execMs": 0.00941699999384582
}
],
"timings": {
- "gateMs": 1997.3182919999817,
+ "gateMs": 909.7765420000069,
"gateSkipped": false,
- "ttftMs": 9055.489207999955,
- "modelMs": 7451.421415999997,
- "toolMs": 0.012667000002693385,
- "totalMs": 9448.752374999982,
+ "ttftMs": 4400.650708000001,
+ "modelMs": 3831.6939580000035,
+ "toolMs": 0.00941699999384582,
+ "totalMs": 4741.479917000004,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5484.387457999983,
+ "roundMs": 2301.831541000007,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.012667000002693385
+ "execMs": 0.00941699999384582
}
]
},
{
- "roundMs": 1967.0339580000145,
- "firstDeltaMs": 1573.2924999999814,
+ "roundMs": 1529.8624169999966,
+ "firstDeltaMs": 1188.860416999989,
"calls": []
}
]
@@ -12060,27 +11895,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1736,
- "p95": 2281,
- "mean": 1846,
+ "p50": 1186,
+ "p95": 1968,
+ "mean": 1429,
"n": 3
},
"ttftMs": {
- "p50": 11452,
- "p95": 13264,
- "mean": 11926,
+ "p50": 4165,
+ "p95": 4300,
+ "mean": 4139,
"n": 3
},
"totalMs": {
- "p50": 12020,
- "p95": 13588,
- "mean": 12332,
+ "p50": 4692,
+ "p95": 4846,
+ "mean": 4716,
"n": 3
},
"modelMs": {
- "p50": 10500,
- "p95": 11307,
- "mean": 10486,
+ "p50": 3476,
+ "p95": 3660,
+ "mean": 3287,
"n": 3
},
"toolMs": {
@@ -12090,62 +11925,47 @@
"n": 3
},
"rounds": {
- "mean": 2.3,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
- {
- "name": "stock_search",
- "round": 0,
- "atMs": 8777.187875000003,
- "execMs": 0.10083399998256937
- },
{
"name": "generate_video",
- "round": 1,
- "atMs": 11420.228958000022,
- "execMs": 0.036874999990686774
+ "round": 0,
+ "atMs": 2797.8774589999957,
+ "execMs": 0.22508299999753945
}
],
"timings": {
- "gateMs": 2281.214500000002,
+ "gateMs": 1186.3750419999997,
"gateSkipped": false,
- "ttftMs": 13264.204458000022,
- "modelMs": 11306.539541999984,
- "toolMs": 0.13770899997325614,
- "totalMs": 13587.89175099996,
+ "ttftMs": 4164.566959000003,
+ "modelMs": 3659.836457000012,
+ "toolMs": 0.22508299999753945,
+ "totalMs": 4846.436582000009,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6495.3183749999735,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "stock_search",
- "execMs": 0.10083399998256937
- }
- ]
- },
- {
- "roundMs": 2642.884791999997,
+ "roundMs": 1610.8559160000004,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.036874999990686774
+ "execMs": 0.22508299999753945
}
]
},
{
- "roundMs": 2168.3363750000135,
- "firstDeltaMs": 1843.8831670000218,
+ "roundMs": 2048.9805410000117,
+ "firstDeltaMs": 1366.4287500000064,
"calls": []
}
]
@@ -12153,39 +11973,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 8600.30920899997,
- "execMs": 0.018625000026077032
+ "atMs": 2610.543875000003,
+ "execMs": 0.013666999991983175
}
],
"timings": {
- "gateMs": 1735.6253330000327,
+ "gateMs": 1132.6067909999983,
"gateSkipped": false,
- "ttftMs": 11062.16949999996,
- "modelMs": 9651.85500000004,
- "toolMs": 0.018625000026077032,
- "totalMs": 11387.498958000098,
+ "ttftMs": 3950.9111249999987,
+ "modelMs": 3475.9445410000044,
+ "toolMs": 0.013666999991983175,
+ "totalMs": 4608.564998999995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6864.421875,
+ "roundMs": 1477.7147079999995,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.018625000026077032
+ "execMs": 0.013666999991983175
}
]
},
{
- "roundMs": 2787.4331250000396,
- "firstDeltaMs": 2461.8080840000184,
+ "roundMs": 1998.2298330000049,
+ "firstDeltaMs": 1340.3294170000008,
"calls": []
}
]
@@ -12193,39 +12014,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 9822.61841699999,
- "execMs": 0.012916999985463917
+ "atMs": 3283.111791000003,
+ "execMs": 0.023832999999285676
}
],
"timings": {
- "gateMs": 1520.2101249999832,
+ "gateMs": 1968.4477909999987,
"gateSkipped": false,
- "ttftMs": 11451.524582999991,
- "modelMs": 10500.009459000023,
- "toolMs": 0.012916999985463917,
- "totalMs": 12020.232500999991,
+ "ttftMs": 4300.198707999996,
+ "modelMs": 2723.863540999999,
+ "toolMs": 0.023832999999285676,
+ "totalMs": 4692.335164999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8302.206459000008,
+ "roundMs": 1314.5063329999975,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.012916999985463917
+ "execMs": 0.023832999999285676
}
]
},
{
- "roundMs": 2197.8030000000144,
- "firstDeltaMs": 1628.856832999969,
+ "roundMs": 1409.3572080000013,
+ "firstDeltaMs": 1017.0311249999941,
"calls": []
}
]
@@ -12236,38 +12058,38 @@
{
"name": "mute-all-does-not-redo-prior-turn",
"bucket": "single-tool",
- "passRate": 1,
+ "passRate": 0.3333333333333333,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1575,
- "p95": 1599,
- "mean": 1535,
- "n": 3
+ "p50": 1202,
+ "p95": 1202,
+ "mean": 1202,
+ "n": 1
},
"ttftMs": {
- "p50": 5893,
- "p95": 7077,
- "mean": 6256,
- "n": 3
+ "p50": 3992,
+ "p95": 3992,
+ "mean": 3992,
+ "n": 1
},
"totalMs": {
- "p50": 6234,
- "p95": 7163,
- "mean": 6481,
- "n": 3
+ "p50": 4765,
+ "p95": 4765,
+ "mean": 4765,
+ "n": 1
},
"modelMs": {
- "p50": 4803,
- "p95": 5564,
- "mean": 4946,
- "n": 3
+ "p50": 3563,
+ "p95": 3563,
+ "mean": 3563,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -12276,40 +12098,63 @@
},
"runs": [
{
- "pass": true,
- "intent": "work",
- "notes": [],
+ "pass": false,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [
+ "reply did not match /mut/i"
+ ],
"trace": [
{
- "name": "set_track_muted",
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 2469.124457999991,
+ "execMs": 0.2545000000100117
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 2469.392832999991,
+ "execMs": 0.002082999999402091
+ },
+ {
+ "name": "set_clip_muted",
"round": 0,
- "atMs": 5233.247042000003,
- "execMs": 0.1461669999989681
+ "atMs": 2469.3961249999993,
+ "execMs": 0.000750000006519258
}
],
"timings": {
- "gateMs": 1599.1186250000028,
+ "gateMs": 991.5635830000101,
"gateSkipped": false,
- "ttftMs": 7077.417957999976,
- "modelMs": 5563.559542000003,
- "toolMs": 0.1461669999989681,
- "totalMs": 7162.8243340000045,
+ "ttftMs": 3626.333249999996,
+ "modelMs": 2851.6182919999846,
+ "toolMs": 0.25733300001593307,
+ "totalMs": 3843.4392080000107,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3633.2849590000114,
+ "roundMs": 1475.74549999999,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_track_muted",
- "execMs": 0.1461669999989681
+ "name": "set_clip_muted",
+ "execMs": 0.2545000000100117
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.002082999999402091
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.000750000006519258
}
]
},
{
- "roundMs": 1930.2745829999913,
- "firstDeltaMs": 1844.0019169999869,
+ "roundMs": 1375.8727919999947,
+ "firstDeltaMs": 1156.9040830000013,
"calls": []
}
]
@@ -12317,79 +12162,123 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
- "name": "set_track_muted",
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 2493.2206670000014,
+ "execMs": 0.014083000001846813
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 2493.2427499999903,
+ "execMs": 0.0005410000012489036
+ },
+ {
+ "name": "set_clip_muted",
"round": 0,
- "atMs": 4315.894207999983,
- "execMs": 0.008374999975785613
+ "atMs": 2493.2440829999978,
+ "execMs": 0.0003330000035930425
}
],
"timings": {
- "gateMs": 1574.9483750000363,
+ "gateMs": 1202.0403340000048,
"gateSkipped": false,
- "ttftMs": 5797.1797079999815,
- "modelMs": 4471.643458000035,
- "toolMs": 0.008374999975785613,
- "totalMs": 6046.600208000047,
+ "ttftMs": 3992.077416999993,
+ "modelMs": 3563.0451659999962,
+ "toolMs": 0.014957000006688759,
+ "totalMs": 4765.100457000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2740.512083000038,
+ "roundMs": 1290.940665999995,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_track_muted",
- "execMs": 0.008374999975785613
+ "name": "set_clip_muted",
+ "execMs": 0.014083000001846813
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0005410000012489036
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0003330000035930425
}
]
},
{
- "roundMs": 1731.1313749999972,
- "firstDeltaMs": 1481.262584000011,
+ "roundMs": 2272.1045000000013,
+ "firstDeltaMs": 1498.7875000000058,
"calls": []
}
]
}
},
{
- "pass": true,
- "intent": "work",
- "notes": [],
+ "pass": false,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [
+ "reply did not match /mut/i"
+ ],
"trace": [
{
- "name": "set_track_muted",
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 5449.9354160000075,
+ "execMs": 0.01745899999514222
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 5449.962790999998,
+ "execMs": 0.001041000010445714
+ },
+ {
+ "name": "set_clip_muted",
"round": 0,
- "atMs": 4341.245458999998,
- "execMs": 0.02183300000615418
+ "atMs": 5449.964749999999,
+ "execMs": 0.0005419999943114817
}
],
"timings": {
- "gateMs": 1431.5874579999945,
+ "gateMs": 936.8374579999945,
"gateSkipped": false,
- "ttftMs": 5892.593791999971,
- "modelMs": 4802.649584000057,
- "toolMs": 0.02183300000615418,
- "totalMs": 6234.258875000058,
+ "ttftMs": 6955.200540999998,
+ "modelMs": 6742.061791999993,
+ "toolMs": 0.019041999999899417,
+ "totalMs": 7678.918291999988,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2909.3462090000394,
+ "roundMs": 4512.70537499999,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_track_muted",
- "execMs": 0.02183300000615418
+ "name": "set_clip_muted",
+ "execMs": 0.01745899999514222
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.001041000010445714
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0005419999943114817
}
]
},
{
- "roundMs": 1893.3033750000177,
- "firstDeltaMs": 1551.2840000000433,
+ "roundMs": 2229.3564170000027,
+ "firstDeltaMs": 1505.1956250000076,
"calls": []
}
]
@@ -12404,27 +12293,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1567,
- "p95": 1603,
- "mean": 1495,
+ "p50": 1200,
+ "p95": 1897,
+ "mean": 1370,
"n": 3
},
"ttftMs": {
- "p50": 8590,
- "p95": 8923,
- "mean": 8337,
+ "p50": 4353,
+ "p95": 5599,
+ "mean": 4537,
"n": 3
},
"totalMs": {
- "p50": 8804,
- "p95": 9149,
- "mean": 8629,
+ "p50": 4630,
+ "p95": 6178,
+ "mean": 4921,
"n": 3
},
"modelMs": {
- "p50": 7238,
- "p95": 7833,
- "mean": 7133,
+ "p50": 3429,
+ "p95": 4281,
+ "mean": 3551,
"n": 3
},
"toolMs": {
@@ -12441,39 +12330,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_image",
"round": 0,
- "atMs": 5785.273207999999,
- "execMs": 0.23287499998696148
+ "atMs": 2602.3965829999943,
+ "execMs": 0.18316699998104014
}
],
"timings": {
- "gateMs": 1603.4916250000242,
+ "gateMs": 1012.1205000000045,
"gateSkipped": false,
- "ttftMs": 7500.0566659999895,
- "modelMs": 6328.871875000012,
- "toolMs": 0.23287499998696148,
- "totalMs": 7932.596375000023,
+ "ttftMs": 3658.8847499999974,
+ "modelMs": 2942.1026670000138,
+ "toolMs": 0.18316699998104014,
+ "totalMs": 3954.4063339999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4181.136582999956,
+ "roundMs": 1589.3782090000022,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.23287499998696148
+ "execMs": 0.18316699998104014
}
]
},
{
- "roundMs": 2147.735292000056,
- "firstDeltaMs": 1714.5316250000033,
+ "roundMs": 1352.7244580000115,
+ "firstDeltaMs": 1056.2735000000102,
"calls": []
}
]
@@ -12481,39 +12371,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_image",
"round": 0,
- "atMs": 7062.45004200004,
- "execMs": 0.042625000001862645
+ "atMs": 4187.458084000013,
+ "execMs": 0.018666999996639788
}
],
"timings": {
- "gateMs": 1316.0909579999861,
+ "gateMs": 1897.2694999999949,
"gateSkipped": false,
- "ttftMs": 8922.602417000046,
- "modelMs": 7833.170209000062,
- "toolMs": 0.042625000001862645,
- "totalMs": 9149.30379200005,
+ "ttftMs": 5599.341125000006,
+ "modelMs": 4280.8706250000105,
+ "toolMs": 0.018666999996639788,
+ "totalMs": 6178.158792000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5746.180084000051,
+ "roundMs": 2289.9490410000144,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.042625000001862645
+ "execMs": 0.018666999996639788
}
]
},
{
- "roundMs": 2086.990125000011,
- "firstDeltaMs": 1860.0742090000422,
+ "roundMs": 1990.921583999996,
+ "firstDeltaMs": 1411.8238750000019,
"calls": []
}
]
@@ -12521,39 +12412,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_image",
"round": 0,
- "atMs": 6811.071082999988,
- "execMs": 0.01866700005484745
+ "atMs": 3247.8937089999963,
+ "execMs": 0.012291999999433756
}
],
"timings": {
- "gateMs": 1566.5023749999818,
+ "gateMs": 1200.3345419999969,
"gateSkipped": false,
- "ttftMs": 8589.74325,
- "modelMs": 7237.665126000007,
- "toolMs": 0.01866700005484745,
- "totalMs": 8804.186168000044,
+ "ttftMs": 4352.935167000018,
+ "modelMs": 3429.3286249999946,
+ "toolMs": 0.012291999999433756,
+ "totalMs": 4629.675458999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5244.3718339999905,
+ "roundMs": 2047.3996669999906,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.01866700005484745
+ "execMs": 0.012291999999433756
}
]
},
{
- "roundMs": 1993.2932920000167,
- "firstDeltaMs": 1778.6270420000073,
+ "roundMs": 1381.928958000004,
+ "firstDeltaMs": 1105.0031250000175,
"calls": []
}
]
@@ -12568,27 +12460,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1807,
- "p95": 1808,
- "mean": 1779,
+ "p50": 1645,
+ "p95": 2000,
+ "mean": 1600,
"n": 3
},
"ttftMs": {
- "p50": 7724,
- "p95": 10809,
- "mean": 8730,
+ "p50": 4321,
+ "p95": 5073,
+ "mean": 4309,
"n": 3
},
"totalMs": {
- "p50": 8009,
- "p95": 11165,
- "mean": 9042,
+ "p50": 5104,
+ "p95": 5373,
+ "mean": 4735,
"n": 3
},
"modelMs": {
- "p50": 6287,
- "p95": 9358,
- "mean": 7262,
+ "p50": 3373,
+ "p95": 3459,
+ "mean": 3134,
"n": 3
},
"toolMs": {
@@ -12605,39 +12497,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 6042.902624999988,
- "execMs": 0.011500000022351742
+ "atMs": 2367.1293750000186,
+ "execMs": 0.006083000014768913
}
],
"timings": {
- "gateMs": 1807.0632910000277,
+ "gateMs": 1156.0266249999986,
"gateSkipped": false,
- "ttftMs": 10809.33712500002,
- "modelMs": 9358.317125000001,
- "toolMs": 0.011500000022351742,
- "totalMs": 11165.391916000051,
+ "ttftMs": 3532.4629160000186,
+ "modelMs": 2571.1672499999986,
+ "toolMs": 0.006083000014768913,
+ "totalMs": 3727.199958000012,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4235.366875000007,
+ "roundMs": 1210.3715000000084,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.011500000022351742
+ "execMs": 0.006083000014768913
}
]
},
{
- "roundMs": 5122.950249999994,
- "firstDeltaMs": 4766.379707999993,
+ "roundMs": 1360.7957499999902,
+ "firstDeltaMs": 1165.2747499999823,
"calls": []
}
]
@@ -12645,39 +12538,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 5810.050416000013,
- "execMs": 0.01854199997615069
+ "atMs": 4021.699499999988,
+ "execMs": 0.03958400001283735
}
],
"timings": {
- "gateMs": 1807.7508749999688,
+ "gateMs": 2000.1446249999863,
"gateSkipped": false,
- "ttftMs": 7655.730333000014,
- "modelMs": 6142.217000000004,
- "toolMs": 0.01854199997615069,
- "totalMs": 7949.986416999949,
+ "ttftMs": 5073.328624999995,
+ "modelMs": 3373.02125000002,
+ "toolMs": 0.03958400001283735,
+ "totalMs": 5373.205459000019,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4002.0142500000075,
+ "roundMs": 2021.2889580000192,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.01854199997615069
+ "execMs": 0.03958400001283735
}
]
},
{
- "roundMs": 2140.2027499999967,
- "firstDeltaMs": 1845.6290000000154,
+ "roundMs": 1351.7322920000006,
+ "firstDeltaMs": 1051.5500830000092,
"calls": []
}
]
@@ -12685,39 +12579,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 6183.302207999979,
- "execMs": 0.01554200000828132
+ "atMs": 2717.6621250000026,
+ "execMs": 0.009625000006053597
}
],
"timings": {
- "gateMs": 1722.646875000035,
+ "gateMs": 1644.9369169999845,
"gateSkipped": false,
- "ttftMs": 7724.34600000002,
- "modelMs": 6286.811207999999,
- "toolMs": 0.01554200000828132,
- "totalMs": 8009.473625000042,
+ "ttftMs": 4321.151042000012,
+ "modelMs": 3459.062750000012,
+ "toolMs": 0.009625000006053597,
+ "totalMs": 5104.0092920000025,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4460.329041999998,
+ "roundMs": 1072.3950000000186,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.01554200000828132
+ "execMs": 0.009625000006053597
}
]
},
{
- "roundMs": 1826.4821660000016,
- "firstDeltaMs": 1540.9807080000173,
+ "roundMs": 2386.6677499999932,
+ "firstDeltaMs": 1603.4005839999882,
"calls": []
}
]
@@ -12732,27 +12627,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1983,
- "p95": 2345,
- "mean": 2079,
+ "p50": 1260,
+ "p95": 1989,
+ "mean": 1413,
"n": 3
},
"ttftMs": {
- "p50": 7587,
- "p95": 7951,
- "mean": 7617,
+ "p50": 4319,
+ "p95": 4686,
+ "mean": 4274,
"n": 3
},
"totalMs": {
- "p50": 8132,
- "p95": 8424,
- "mean": 8115,
+ "p50": 4916,
+ "p95": 5150,
+ "mean": 4693,
"n": 3
},
"modelMs": {
- "p50": 6079,
- "p95": 6148,
- "mean": 6035,
+ "p50": 3161,
+ "p95": 3656,
+ "mean": 3280,
"n": 3
},
"toolMs": {
@@ -12769,39 +12664,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 6430.777708000038,
- "execMs": 0.16499999997904524
+ "atMs": 2579.882459000015,
+ "execMs": 0.1275420000019949
}
],
"timings": {
- "gateMs": 2344.8572919999715,
+ "gateMs": 989.1708339999896,
"gateSkipped": false,
- "ttftMs": 7950.8784160000505,
- "modelMs": 6079.020042000047,
- "toolMs": 0.16499999997904524,
- "totalMs": 8424.042333999998,
+ "ttftMs": 3815.4959589999926,
+ "modelMs": 3022.6715830000176,
+ "toolMs": 0.1275420000019949,
+ "totalMs": 4011.969959000009,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4084.9560420000344,
+ "roundMs": 1590.3109169999952,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.16499999997904524
+ "execMs": 0.1275420000019949
}
]
},
{
- "roundMs": 1994.064000000013,
- "firstDeltaMs": 1519.917208000028,
+ "roundMs": 1432.3606660000223,
+ "firstDeltaMs": 1235.421250000014,
"calls": []
}
]
@@ -12809,39 +12705,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 5982.250041999971,
- "execMs": 0.010917000006884336
+ "atMs": 2800.384334000002,
+ "execMs": 0.01854199997615069
}
],
"timings": {
- "gateMs": 1983.32904099999,
+ "gateMs": 1259.9949579999957,
"gateSkipped": false,
- "ttftMs": 7587.286124999984,
- "modelMs": 6148.408249999979,
- "toolMs": 0.010917000006884336,
- "totalMs": 8131.748207999975,
+ "ttftMs": 4319.396042000008,
+ "modelMs": 3656.0903760000074,
+ "toolMs": 0.01854199997615069,
+ "totalMs": 4916.103875999979,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3998.675874999957,
+ "roundMs": 1539.9935840000107,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.010917000006884336
+ "execMs": 0.01854199997615069
}
]
},
{
- "roundMs": 2149.7323750000214,
- "firstDeltaMs": 1604.9844160000212,
+ "roundMs": 2116.0967919999966,
+ "firstDeltaMs": 1518.9050829999906,
"calls": []
}
]
@@ -12849,39 +12746,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 5549.231625000015,
- "execMs": 0.02954200003296137
+ "atMs": 3433.7099160000216,
+ "execMs": 0.019041999999899417
}
],
"timings": {
- "gateMs": 1910.1484169999603,
+ "gateMs": 1988.8493749999907,
"gateSkipped": false,
- "ttftMs": 7313.0014580000425,
- "modelMs": 5877.851458999969,
- "toolMs": 0.02954200003296137,
- "totalMs": 7788.029417999962,
+ "ttftMs": 4685.677666000003,
+ "modelMs": 3161.2570000000123,
+ "toolMs": 0.019041999999899417,
+ "totalMs": 5150.125417000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3638.6097500000033,
+ "roundMs": 1444.763666999992,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.02954200003296137
+ "execMs": 0.019041999999899417
}
]
},
{
- "roundMs": 2239.2417089999653,
- "firstDeltaMs": 1763.6805419999873,
+ "roundMs": 1716.4933330000204,
+ "firstDeltaMs": 1251.9139580000192,
"calls": []
}
]
@@ -12896,27 +12794,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1599,
- "p95": 1791,
- "mean": 1660,
+ "p50": 929,
+ "p95": 969,
+ "mean": 942,
"n": 3
},
"ttftMs": {
- "p50": 6427,
- "p95": 9016,
- "mean": 7246,
+ "p50": 3274,
+ "p95": 3461,
+ "mean": 3327,
"n": 3
},
"totalMs": {
- "p50": 6699,
- "p95": 9458,
- "mean": 7552,
+ "p50": 3481,
+ "p95": 3661,
+ "mean": 3524,
"n": 3
},
"modelMs": {
- "p50": 5110,
- "p95": 7667,
- "mean": 5892,
+ "p50": 2551,
+ "p95": 2732,
+ "mean": 2582,
"n": 3
},
"toolMs": {
@@ -12933,39 +12831,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 4999.312542000029,
- "execMs": 0.025834000029135495
+ "atMs": 2111.554415999999,
+ "execMs": 0.013500000000931323
}
],
"timings": {
- "gateMs": 1588.9105829999899,
+ "gateMs": 929.1623750000144,
"gateSkipped": false,
- "ttftMs": 6427.11712499999,
- "modelMs": 5109.944707999995,
- "toolMs": 0.025834000029135495,
- "totalMs": 6698.881125000014,
+ "ttftMs": 3273.649875000003,
+ "modelMs": 2551.3536670000176,
+ "toolMs": 0.013500000000931323,
+ "totalMs": 3480.529542000033,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3409.9829579999787,
+ "roundMs": 1182.1783750000177,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.025834000029135495
+ "execMs": 0.013500000000931323
}
]
},
{
- "roundMs": 1699.9617500000168,
- "firstDeltaMs": 1427.713291000051,
+ "roundMs": 1369.175292,
+ "firstDeltaMs": 1162.0414170000004,
"calls": []
}
]
@@ -12973,39 +12872,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 4458.446042000025,
- "execMs": 0.015665999962948263
+ "atMs": 2271.778957999981,
+ "execMs": 0.05741700000362471
}
],
"timings": {
- "gateMs": 1599.0555829999503,
+ "gateMs": 928.5401249999995,
"gateSkipped": false,
- "ttftMs": 6293.67312500003,
- "modelMs": 4900.230707999959,
- "toolMs": 0.015665999962948263,
- "totalMs": 6499.301956999872,
+ "ttftMs": 3460.755832999974,
+ "modelMs": 2732.03916700001,
+ "toolMs": 0.05741700000362471,
+ "totalMs": 3660.636709000013,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2859.1905829999596,
+ "roundMs": 1343.076457999996,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.015665999962948263
+ "execMs": 0.05741700000362471
}
]
},
{
- "roundMs": 2041.0401249999995,
- "firstDeltaMs": 1835.1848340000142,
+ "roundMs": 1388.962709000014,
+ "firstDeltaMs": 1188.8762920000008,
"calls": []
}
]
@@ -13013,39 +12913,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "trim_clip",
"round": 0,
- "atMs": 7448.687417000008,
- "execMs": 0.002957999997306615
+ "atMs": 2131.2555839999986,
+ "execMs": 0.0032500000088475645
}
],
"timings": {
- "gateMs": 1791.333666999999,
+ "gateMs": 968.6253750000033,
"gateSkipped": false,
- "ttftMs": 9016.483917000005,
- "modelMs": 7667.035417000006,
- "toolMs": 0.002957999997306615,
- "totalMs": 9458.372042000003,
+ "ttftMs": 3247.705042000016,
+ "modelMs": 2461.4264580000017,
+ "toolMs": 0.0032500000088475645,
+ "totalMs": 3430.055083000014,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5657.150541999959,
+ "roundMs": 1162.4330420000188,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.002957999997306615
+ "execMs": 0.0032500000088475645
}
]
},
{
- "roundMs": 2009.884875000047,
- "firstDeltaMs": 1567.7612910000025,
+ "roundMs": 1298.993415999983,
+ "firstDeltaMs": 1116.4057080000057,
"calls": []
}
]
@@ -13060,27 +12961,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1752,
- "p95": 1986,
- "mean": 1830,
+ "p50": 1557,
+ "p95": 2076,
+ "mean": 1596,
"n": 3
},
"ttftMs": {
- "p50": 7087,
- "p95": 7095,
- "mean": 7042,
+ "p50": 4482,
+ "p95": 7900,
+ "mean": 5597,
"n": 3
},
"totalMs": {
- "p50": 7289,
- "p95": 7628,
- "mean": 7380,
+ "p50": 5155,
+ "p95": 8229,
+ "mean": 6179,
"n": 3
},
"modelMs": {
- "p50": 5471,
- "p95": 5876,
- "mean": 5550,
+ "p50": 3998,
+ "p95": 6153,
+ "mean": 4583,
"n": 3
},
"toolMs": {
@@ -13097,39 +12998,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 5331.942582999996,
- "execMs": 0.3509999999660067
+ "atMs": 6748.411499999987,
+ "execMs": 0.2836670000106096
}
],
"timings": {
- "gateMs": 1752.4414999999572,
+ "gateMs": 2075.8542920000036,
"gateSkipped": false,
- "ttftMs": 7087.306333000015,
- "modelMs": 5875.683499999985,
- "toolMs": 0.3509999999660067,
- "totalMs": 7628.475999999908,
+ "ttftMs": 7900.143125000002,
+ "modelMs": 6152.889332999999,
+ "toolMs": 0.2836670000106096,
+ "totalMs": 8229.027292000013,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3579.0214999999735,
+ "roundMs": 4671.781167000008,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.3509999999660067
+ "execMs": 0.2836670000106096
}
]
},
{
- "roundMs": 2296.662000000011,
- "firstDeltaMs": 1754.8986250000307,
+ "roundMs": 1481.108165999991,
+ "firstDeltaMs": 1151.3802079999878,
"calls": []
}
]
@@ -13137,39 +13039,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 4701.652416999976,
- "execMs": 0.0044170000473968685
+ "atMs": 3119.0497499999765,
+ "execMs": 0.01683400000911206
}
],
"timings": {
- "gateMs": 1751.1852499999804,
+ "gateMs": 1556.809042000008,
"gateSkipped": false,
- "ttftMs": 6942.212249999982,
- "modelMs": 5471.085625000007,
- "toolMs": 0.0044170000473968685,
- "totalMs": 7222.275292000035,
+ "ttftMs": 4482.1051670000015,
+ "modelMs": 3598.3967500000144,
+ "toolMs": 0.01683400000911206,
+ "totalMs": 5155.222626000032,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2950.3040000000037,
+ "roundMs": 1561.902625000017,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.0044170000473968685
+ "execMs": 0.01683400000911206
}
]
},
{
- "roundMs": 2520.7816250000033,
- "firstDeltaMs": 2240.5343749999884,
+ "roundMs": 2036.4941249999974,
+ "firstDeltaMs": 1362.9762089999858,
"calls": []
}
]
@@ -13177,39 +13080,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 5276.571083999996,
- "execMs": 0.006333000026643276
+ "atMs": 3150.4295409999904,
+ "execMs": 0.01108399999793619
}
],
"timings": {
- "gateMs": 1985.790417000011,
+ "gateMs": 1154.4163340000086,
"gateSkipped": false,
- "ttftMs": 7095.409709000029,
- "modelMs": 5303.137416999962,
- "toolMs": 0.006333000026643276,
- "totalMs": 7288.934166999999,
+ "ttftMs": 4409.976208000007,
+ "modelMs": 3998.0732920000155,
+ "toolMs": 0.01108399999793619,
+ "totalMs": 5152.500710000022,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3290.5345419999794,
+ "roundMs": 1995.893458000006,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.006333000026643276
+ "execMs": 0.01108399999793619
}
]
},
{
- "roundMs": 2012.6028749999823,
- "firstDeltaMs": 1818.8179159999709,
+ "roundMs": 2002.1798340000096,
+ "firstDeltaMs": 1259.4923750000016,
"calls": []
}
]
@@ -13224,27 +13128,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1814,
- "p95": 1928,
- "mean": 1822,
+ "p50": 1175,
+ "p95": 1987,
+ "mean": 1356,
"n": 3
},
"ttftMs": {
- "p50": 6583,
- "p95": 7650,
- "mean": 6922,
+ "p50": 4440,
+ "p95": 4596,
+ "mean": 4466,
"n": 3
},
"totalMs": {
- "p50": 6870,
- "p95": 8147,
- "mean": 7261,
+ "p50": 4964,
+ "p95": 5260,
+ "mean": 4961,
"n": 3
},
"modelMs": {
- "p50": 5146,
- "p95": 6219,
- "mean": 5439,
+ "p50": 3484,
+ "p95": 4059,
+ "mean": 3605,
"n": 3
},
"toolMs": {
@@ -13261,39 +13165,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 5062.255333999987,
- "execMs": 0.32216699997661635
+ "atMs": 3345.2059169999848,
+ "execMs": 0.12487500000861473
}
],
"timings": {
- "gateMs": 1723.2970830000122,
+ "gateMs": 1175.4720419999794,
"gateSkipped": false,
- "ttftMs": 6583.332500000019,
- "modelMs": 5146.164166000031,
- "toolMs": 0.32216699997661635,
- "totalMs": 6869.78341600002,
+ "ttftMs": 4440.416541999992,
+ "modelMs": 3483.681251000002,
+ "toolMs": 0.12487500000861473,
+ "totalMs": 4659.27816799999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3338.470250000013,
+ "roundMs": 2169.053833999991,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.32216699997661635
+ "execMs": 0.12487500000861473
}
]
},
{
- "roundMs": 1807.6939160000184,
- "firstDeltaMs": 1520.6492080000462,
+ "roundMs": 1314.6274170000106,
+ "firstDeltaMs": 1095.0553329999966,
"calls": []
}
]
@@ -13301,39 +13206,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 5024.6506660000305,
- "execMs": 0.012917000043671578
+ "atMs": 2927.8327500000014,
+ "execMs": 0.01900000000023283
}
],
"timings": {
- "gateMs": 1813.7052500000573,
+ "gateMs": 905.2064590000082,
"gateSkipped": false,
- "ttftMs": 6531.536666000029,
- "modelMs": 4953.527791999979,
- "toolMs": 0.012917000043671578,
- "totalMs": 6767.24595900008,
+ "ttftMs": 4362.666291000001,
+ "modelMs": 4058.9732920000097,
+ "toolMs": 0.01900000000023283,
+ "totalMs": 4964.198751000018,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3210.3758329999982,
+ "roundMs": 2022.255124999996,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.012917000043671578
+ "execMs": 0.01900000000023283
}
]
},
{
- "roundMs": 1743.1519589999807,
- "firstDeltaMs": 1506.8106670000125,
+ "roundMs": 2036.7181670000136,
+ "firstDeltaMs": 1434.734208000009,
"calls": []
}
]
@@ -13341,39 +13247,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_animation",
"round": 0,
- "atMs": 5524.049666999956,
- "execMs": 0.020124999980907887
+ "atMs": 3423.551999999996,
+ "execMs": 0.021499999973457307
}
],
"timings": {
- "gateMs": 1928.2912500000093,
+ "gateMs": 1987.058583999984,
"gateSkipped": false,
- "ttftMs": 7650.248082999955,
- "modelMs": 6218.590290999971,
- "toolMs": 0.020124999980907887,
- "totalMs": 8146.901665999962,
+ "ttftMs": 4595.964584000001,
+ "modelMs": 3273.0979579999985,
+ "toolMs": 0.021499999973457307,
+ "totalMs": 5260.178041999956,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3595.5161659999867,
+ "roundMs": 1436.343790999992,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.020124999980907887
+ "execMs": 0.021499999973457307
}
]
},
{
- "roundMs": 2623.0741249999846,
- "firstDeltaMs": 2126.0995419999817,
+ "roundMs": 1836.7541670000064,
+ "firstDeltaMs": 1172.3484999999928,
"calls": []
}
]
@@ -13388,32 +13295,32 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1918,
- "p95": 2220,
- "mean": 1918,
+ "p50": 985,
+ "p95": 1719,
+ "mean": 1224,
"n": 3
},
"ttftMs": {
- "p50": 7347,
- "p95": 10456,
- "mean": 8145,
+ "p50": 4182,
+ "p95": 4475,
+ "mean": 4091,
"n": 3
},
"totalMs": {
- "p50": 7700,
- "p95": 10800,
- "mean": 8507,
+ "p50": 4461,
+ "p95": 5159,
+ "mean": 4473,
"n": 3
},
"modelMs": {
- "p50": 5782,
- "p95": 8580,
- "mean": 6589,
+ "p50": 3441,
+ "p95": 3476,
+ "mean": 3249,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
@@ -13425,39 +13332,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 4891.097958000028,
- "execMs": 0.5018330000457354
+ "atMs": 2562.2455419999897,
+ "execMs": 0.35775000002468005
}
],
"timings": {
- "gateMs": 1615.4432079999824,
+ "gateMs": 968.4248329999973,
"gateSkipped": false,
- "ttftMs": 6632.028416000016,
- "modelMs": 5404.93841599999,
- "toolMs": 0.5018330000457354,
- "totalMs": 7020.883457000018,
+ "ttftMs": 3616.2149169999757,
+ "modelMs": 2828.9400840000308,
+ "toolMs": 0.35775000002468005,
+ "totalMs": 3797.7226670000528,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3275.3255409999983,
+ "roundMs": 1593.3552500000224,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.5018330000457354
+ "execMs": 0.35775000002468005
}
]
},
{
- "roundMs": 2129.6128749999916,
- "firstDeltaMs": 1740.3807500000112,
+ "roundMs": 1235.5848340000084,
+ "firstDeltaMs": 1053.5677500000165,
"calls": []
}
]
@@ -13465,39 +13373,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 8714.212375000003,
- "execMs": 0.04433300002710894
+ "atMs": 3069.3231249999953,
+ "execMs": 0.006375000026309863
}
],
"timings": {
- "gateMs": 2220.3682089999784,
+ "gateMs": 984.6744590000017,
"gateSkipped": false,
- "ttftMs": 10456.240916000039,
- "modelMs": 8579.748541000008,
- "toolMs": 0.04433300002710894,
- "totalMs": 10800.161083000014,
+ "ttftMs": 4181.90174999999,
+ "modelMs": 3476.0403330000117,
+ "toolMs": 0.006375000026309863,
+ "totalMs": 4460.72116700004,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6493.490082999982,
+ "roundMs": 2084.431707999989,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.04433300002710894
+ "execMs": 0.006375000026309863
}
]
},
{
- "roundMs": 2086.2584580000257,
- "firstDeltaMs": 1741.8165000000154,
+ "roundMs": 1391.6086250000226,
+ "firstDeltaMs": 1112.5617090000014,
"calls": []
}
]
@@ -13505,39 +13414,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 5281.54583300004,
- "execMs": 0.047292000032030046
+ "atMs": 3014.860791999992,
+ "execMs": 0.01929199998266995
}
],
"timings": {
- "gateMs": 1917.9400000000023,
+ "gateMs": 1718.6782499999972,
"gateSkipped": false,
- "ttftMs": 7346.928458000009,
- "modelMs": 5781.709665999981,
- "toolMs": 0.047292000032030046,
- "totalMs": 7699.696958000015,
+ "ttftMs": 4475.350584,
+ "modelMs": 3440.6362910000316,
+ "toolMs": 0.01929199998266995,
+ "totalMs": 5159.3338330000115,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3363.287916000001,
+ "roundMs": 1296.0479160000104,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.047292000032030046
+ "execMs": 0.01929199998266995
}
]
},
{
- "roundMs": 2418.4217499999795,
- "firstDeltaMs": 2065.2482919999748,
+ "roundMs": 2144.588375000021,
+ "firstDeltaMs": 1460.4369170000136,
"calls": []
}
]
@@ -13552,27 +13462,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1731,
- "p95": 2089,
- "mean": 1815,
+ "p50": 1063,
+ "p95": 1192,
+ "mean": 1089,
"n": 3
},
"ttftMs": {
- "p50": 8036,
- "p95": 10815,
- "mean": 8931,
+ "p50": 3805,
+ "p95": 4275,
+ "mean": 3912,
"n": 3
},
"totalMs": {
- "p50": 8426,
- "p95": 11152,
- "mean": 9319,
+ "p50": 3991,
+ "p95": 5207,
+ "mean": 4359,
"n": 3
},
"modelMs": {
- "p50": 6800,
- "p95": 9063,
- "mean": 7504,
+ "p50": 2980,
+ "p95": 4014,
+ "mean": 3271,
"n": 3
},
"toolMs": {
@@ -13589,49 +13499,50 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 6020.149040999939,
- "execMs": 0.40095900010783225
+ "atMs": 2450.6561670000083,
+ "execMs": 0.0811249999969732
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6020.625291000004,
- "execMs": 0.013582999934442341
+ "atMs": 2450.7446669999918,
+ "execMs": 0.00237499998183921
}
],
"timings": {
- "gateMs": 1625.708209000004,
+ "gateMs": 1063.1950840000063,
"gateSkipped": false,
- "ttftMs": 7943.340332999942,
- "modelMs": 6799.851334000006,
- "toolMs": 0.4145420000422746,
- "totalMs": 8425.974085000053,
+ "ttftMs": 3656.489957999991,
+ "modelMs": 2817.7453749999986,
+ "toolMs": 0.08349999997881241,
+ "totalMs": 3881.0239589999837,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4393.620916999993,
+ "roundMs": 1387.1173750000016,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.40095900010783225
+ "execMs": 0.0811249999969732
},
{
"name": "set_animation",
- "execMs": 0.013582999934442341
+ "execMs": 0.00237499998183921
}
]
},
{
- "roundMs": 2406.2304170000134,
- "firstDeltaMs": 1922.6451670000097,
+ "roundMs": 1430.627999999997,
+ "firstDeltaMs": 1205.7331669999985,
"calls": []
}
]
@@ -13639,49 +13550,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 6059.75766699994,
- "execMs": 0.005250000045634806
+ "atMs": 2637.917791999993,
+ "execMs": 0.0030830000177957118
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6059.765999999945,
- "execMs": 0.002208000048995018
+ "atMs": 2637.9229999999807,
+ "execMs": 0.001791000016964972
}
],
"timings": {
- "gateMs": 1731.0577079999493,
+ "gateMs": 1010.913750000007,
"gateSkipped": false,
- "ttftMs": 8035.538541999995,
- "modelMs": 6649.356832000078,
- "toolMs": 0.007458000094629824,
- "totalMs": 8380.421998000122,
+ "ttftMs": 3805.391875000001,
+ "modelMs": 2979.925291999971,
+ "toolMs": 0.004874000034760684,
+ "totalMs": 3990.8439160000125,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4328.521041000029,
+ "roundMs": 1626.8004169999913,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.005250000045634806
+ "execMs": 0.0030830000177957118
},
{
"name": "set_animation",
- "execMs": 0.002208000048995018
+ "execMs": 0.001791000016964972
}
]
},
{
- "roundMs": 2320.8357910000486,
- "firstDeltaMs": 1975.7577909999527,
+ "roundMs": 1353.1248749999795,
+ "firstDeltaMs": 1167.459458999976,
"calls": []
}
]
@@ -13689,49 +13601,50 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 8312.456624999992,
- "execMs": 0.010915999999269843
+ "atMs": 2786.637333999999,
+ "execMs": 0.004124999977648258
},
{
"name": "set_animation",
"round": 0,
- "atMs": 8312.47795900004,
- "execMs": 0.003374999971129
+ "atMs": 2786.644916999998,
+ "execMs": 0.0023339999897871166
}
],
"timings": {
- "gateMs": 2089.3352500000037,
+ "gateMs": 1192.437292000017,
"gateSkipped": false,
- "ttftMs": 10815.298708999995,
- "modelMs": 9062.727083000005,
- "toolMs": 0.014290999970398843,
- "totalMs": 11152.07662399998,
+ "ttftMs": 4275.035875000001,
+ "modelMs": 4014.1705839999777,
+ "toolMs": 0.006458999967435375,
+ "totalMs": 5206.614334999962,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6222.899582999991,
+ "roundMs": 1593.7901669999992,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.010915999999269843
+ "execMs": 0.004124999977648258
},
{
"name": "set_animation",
- "execMs": 0.003374999971129
+ "execMs": 0.0023339999897871166
}
]
},
{
- "roundMs": 2839.827500000014,
- "firstDeltaMs": 2502.7766670000274,
+ "roundMs": 2420.3804169999785,
+ "firstDeltaMs": 1488.379667000001,
"calls": []
}
]
@@ -13742,384 +13655,159 @@
{
"name": "filler-words-cut-with-editor",
"bucket": "multi-tool",
- "passRate": 1,
+ "passRate": 0.6666666666666666,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1645,
- "p95": 1696,
- "mean": 1635,
- "n": 3
+ "p50": 1183,
+ "p95": 1356,
+ "mean": 1269,
+ "n": 2
},
"ttftMs": {
- "p50": 90706,
- "p95": 188011,
- "mean": 116785,
- "n": 3
+ "p50": 5265,
+ "p95": 6426,
+ "mean": 5846,
+ "n": 2
},
"totalMs": {
- "p50": 91406,
- "p95": 106361,
- "mean": 89939,
- "n": 3
+ "p50": 5550,
+ "p95": 6651,
+ "mean": 6100,
+ "n": 2
},
"modelMs": {
- "p50": 89838,
- "p95": 104663,
- "mean": 88301,
- "n": 3
+ "p50": 4194,
+ "p95": 5468,
+ "mean": 4831,
+ "n": 2
},
"toolMs": {
- "p50": 3,
- "p95": 3,
- "mean": 2,
- "n": 3
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 2
},
"rounds": {
- "mean": 20.3,
- "max": 24
+ "mean": 3,
+ "max": 3
}
},
"runs": [
{
- "pass": true,
- "intent": "work",
- "notes": [],
+ "pass": false,
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [
+ "mutating tools called: remove_gap, remove_gap"
+ ],
"trace": [
{
- "name": "detect_silence",
+ "name": "split_at",
"round": 0,
- "atMs": 8078.191084000049,
- "execMs": 0.08587499998975545
- },
- {
- "name": "listen_audio",
- "round": 1,
- "atMs": 10495.594792000018,
- "execMs": 0.014874999993480742
+ "atMs": 2592.2833329999994,
+ "execMs": 1.2816659999953117
},
{
"name": "split_at",
- "round": 2,
- "atMs": 18867.112667000038,
- "execMs": 0.27012500003911555
- },
- {
- "name": "get_state",
- "round": 3,
- "atMs": 23688.99370900006,
- "execMs": 0.8629169999621809
+ "round": 0,
+ "atMs": 2593.5823750000272,
+ "execMs": 0.036417000024812296
},
{
"name": "split_at",
- "round": 4,
- "atMs": 25695.297791999998,
- "execMs": 0.13229199999477714
- },
- {
- "name": "get_state",
- "round": 5,
- "atMs": 27449.016541999998,
- "execMs": 0.08649999997578561
+ "round": 0,
+ "atMs": 2593.6204580000194,
+ "execMs": 0.012083000008715317
},
{
"name": "split_at",
- "round": 6,
- "atMs": 30029.628083999967,
- "execMs": 0.07529099995736033
- },
- {
- "name": "get_state",
- "round": 7,
- "atMs": 31487.84720900003,
- "execMs": 0.05429199989885092
+ "round": 0,
+ "atMs": 2593.633292000013,
+ "execMs": 0.013708000013139099
},
{
"name": "delete_item",
- "round": 8,
- "atMs": 35912.87624999997,
- "execMs": 0.5394999999552965
- },
- {
- "name": "get_state",
- "round": 9,
- "atMs": 40666.844584000064,
- "execMs": 0.02887499995995313
+ "round": 1,
+ "atMs": 7210.745583000011,
+ "execMs": 0.36799999998766
},
{
"name": "delete_item",
- "round": 10,
- "atMs": 42885.74266700004,
- "execMs": 0.033042000024579465
- },
- {
- "name": "get_state",
- "round": 11,
- "atMs": 44348.440542,
- "execMs": 0.09870799991767853
- },
- {
- "name": "refine_speech_cuts",
- "round": 12,
- "atMs": 47600.32416700001,
- "execMs": 0.4031250000698492
- },
- {
- "name": "get_state",
- "round": 13,
- "atMs": 49805.87554200005,
- "execMs": 0.016042000032030046
- },
- {
- "name": "update_cue",
- "round": 14,
- "atMs": 55048.848499999964,
- "execMs": 0.04925000004004687
- },
- {
- "name": "update_cue",
- "round": 15,
- "atMs": 57047.80466700008,
- "execMs": 0.057874999940395355
- },
- {
- "name": "delete_cue",
- "round": 16,
- "atMs": 63798.55229200004,
- "execMs": 0.10075000009965152
- },
- {
- "name": "update_cue",
- "round": 17,
- "atMs": 65628.12954200001,
- "execMs": 0.03662499994970858
+ "round": 1,
+ "atMs": 7211.12066700001,
+ "execMs": 0.022875000024214387
},
{
- "name": "update_cue",
- "round": 18,
- "atMs": 72128.61083400005,
- "execMs": 0.03845799993723631
+ "name": "remove_gap",
+ "round": 1,
+ "atMs": 7211.144875000027,
+ "execMs": 0.014582999981939793
},
{
- "name": "get_state",
- "round": 19,
- "atMs": 82566.396709,
- "execMs": 0.030125000048428774
+ "name": "remove_gap",
+ "round": 1,
+ "atMs": 7211.160167000024,
+ "execMs": 0.02104200000758283
}
],
"timings": {
- "gateMs": 1565.0049159999471,
+ "gateMs": 1100.4079999999958,
"gateSkipped": false,
- "ttftMs": 90706.04108400003,
- "modelMs": 89838.21912599995,
- "toolMs": 3.014541999786161,
- "totalMs": 91406.23858399969,
+ "ttftMs": 8467.044833000022,
+ "modelMs": 7647.857334000029,
+ "toolMs": 1.7703740000433754,
+ "totalMs": 8750.035708000069,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6512.509499999927,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.08587499998975545
- }
- ]
- },
- {
- "roundMs": 2417.2397499999497,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.014874999993480742
- }
- ]
- },
- {
- "roundMs": 8371.476334000006,
+ "roundMs": 1491.2452090000152,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.27012500003911555
- }
- ]
- },
- {
- "roundMs": 4821.563124999986,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.8629169999621809
- }
- ]
- },
- {
- "roundMs": 2005.3240409999853,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 1.2816659999953117
+ },
{
"name": "split_at",
- "execMs": 0.13229199999477714
- }
- ]
- },
- {
- "roundMs": 1753.515875000041,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.08649999997578561
- }
- ]
- },
- {
- "roundMs": 2580.4611659999937,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.036417000024812296
+ },
{
"name": "split_at",
- "execMs": 0.07529099995736033
- }
- ]
- },
- {
- "roundMs": 1458.0750000000698,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.012083000008715317
+ },
{
- "name": "get_state",
- "execMs": 0.05429199989885092
+ "name": "split_at",
+ "execMs": 0.013708000013139099
}
]
},
{
- "roundMs": 4424.913667000015,
+ "roundMs": 4617.059291000012,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.5394999999552965
- }
- ]
- },
- {
- "roundMs": 4753.3744579999475,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.02887499995995313
- }
- ]
- },
- {
- "roundMs": 2218.8438340000575,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.36799999998766
+ },
{
"name": "delete_item",
- "execMs": 0.033042000024579465
- }
- ]
- },
- {
- "roundMs": 1462.620458999998,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.09870799991767853
- }
- ]
- },
- {
- "roundMs": 3251.7185419999296,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "refine_speech_cuts",
- "execMs": 0.4031250000698492
- }
- ]
- },
- {
- "roundMs": 2205.1095830000704,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.016042000032030046
- }
- ]
- },
- {
- "roundMs": 5242.92270799994,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.04925000004004687
- }
- ]
- },
- {
- "roundMs": 1998.8538330000592,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.057874999940395355
- }
- ]
- },
- {
- "roundMs": 6750.62274999998,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_cue",
- "execMs": 0.10075000009965152
- }
- ]
- },
- {
- "roundMs": 1829.4050830000779,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.03662499994970858
- }
- ]
- },
- {
- "roundMs": 6500.371959000011,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.022875000024214387
+ },
{
- "name": "update_cue",
- "execMs": 0.03845799993723631
- }
- ]
- },
- {
- "roundMs": 10437.703458999982,
- "firstDeltaMs": null,
- "calls": [
+ "name": "remove_gap",
+ "execMs": 0.014582999981939793
+ },
{
- "name": "get_state",
- "execMs": 0.030125000048428774
+ "name": "remove_gap",
+ "execMs": 0.02104200000758283
}
]
},
{
- "roundMs": 8841.593999999925,
- "firstDeltaMs": 8139.600041999947,
+ "roundMs": 1539.552834000002,
+ "firstDeltaMs": 1255.8510419999948,
"calls": []
}
]
@@ -14127,303 +13815,106 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
- "name": "list_skills",
+ "name": "split_at",
"round": 0,
- "atMs": 4909.41183300002,
- "execMs": 0.08287499996367842
+ "atMs": 3398.329666000005,
+ "execMs": 0.04033400001935661
},
{
- "name": "read_skill",
- "round": 1,
- "atMs": 10567.365417000023,
- "execMs": 0.012208000058308244
- },
- {
- "name": "detect_silence",
- "round": 2,
- "atMs": 13879.85291700007,
- "execMs": 0.07550000003539026
- },
- {
- "name": "listen_audio",
- "round": 3,
- "atMs": 17339.961708000046,
- "execMs": 0.052207999979145825
- },
- {
- "name": "detect_silence",
- "round": 4,
- "atMs": 22352.724042000016,
- "execMs": 0.028624999918974936
+ "name": "split_at",
+ "round": 0,
+ "atMs": 3398.3729999999923,
+ "execMs": 0.005167000024812296
},
{
"name": "split_at",
- "round": 5,
- "atMs": 36228.00041700003,
- "execMs": 0.05045800004154444
+ "round": 0,
+ "atMs": 3398.378708000004,
+ "execMs": 0.011124999989988282
},
{
"name": "split_at",
- "round": 5,
- "atMs": 36228.07625000004,
- "execMs": 0.016417000093497336
+ "round": 0,
+ "atMs": 3398.390290999989,
+ "execMs": 0.0072920000238809735
},
{
"name": "split_at",
- "round": 5,
- "atMs": 36228.09475000005,
- "execMs": 0.11158400005660951
- },
- {
- "name": "get_state",
- "round": 6,
- "atMs": 38200.165917000035,
- "execMs": 0.10187499993480742
+ "round": 0,
+ "atMs": 3398.397915999987,
+ "execMs": 0.010417000012239441
},
{
"name": "delete_item",
- "round": 7,
- "atMs": 41997.094333000015,
- "execMs": 0.03904199996031821
+ "round": 1,
+ "atMs": 5398.92633300001,
+ "execMs": 0.07583399998839013
},
{
"name": "delete_item",
- "round": 7,
- "atMs": 41997.13954200002,
- "execMs": 0.018416999955661595
- },
- {
- "name": "get_state",
- "round": 8,
- "atMs": 44088.77720800007,
- "execMs": 0.046083000022917986
- },
- {
- "name": "read_skill",
- "round": 9,
- "atMs": 48809.74095800007,
- "execMs": 0.03429199999663979
- },
- {
- "name": "refine_speech_cuts",
- "round": 10,
- "atMs": 53839.83558299998,
- "execMs": 0.060999999986961484
- },
- {
- "name": "subtitles_generate",
- "round": 11,
- "atMs": 59056.432750000036,
- "execMs": 0.00820799998473376
- },
- {
- "name": "get_state",
- "round": 12,
- "atMs": 61039.00829200004,
- "execMs": 0.044916999991983175
- },
- {
- "name": "subtitles_remove_track",
- "round": 13,
- "atMs": 65677.45429200004,
- "execMs": 0.016667000018060207
- },
- {
- "name": "subtitles_generate",
- "round": 13,
- "atMs": 65677.47499999998,
- "execMs": 0.001917000045068562
- },
- {
- "name": "get_state",
- "round": 14,
- "atMs": 67486.14100000006,
- "execMs": 0.013417000067420304
+ "round": 1,
+ "atMs": 5399.0067500000005,
+ "execMs": 0.007458000007318333
}
],
"timings": {
- "gateMs": 1645.4372079999885,
+ "gateMs": 1182.5463340000133,
"gateSkipped": false,
- "ttftMs": 71637.94141700002,
- "modelMs": 70402.21300200024,
- "toolMs": 0.8157100001117215,
- "totalMs": 72048.46592000034,
+ "ttftMs": 6426.304957999993,
+ "modelMs": 5467.79912500002,
+ "toolMs": 0.15762700006598607,
+ "totalMs": 6650.503086000099,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3263.3943750000326,
+ "roundMs": 2215.659875000012,
"firstDeltaMs": null,
"calls": [
{
- "name": "list_skills",
- "execMs": 0.08287499996367842
- }
- ]
- },
- {
- "roundMs": 5657.744291999959,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.012208000058308244
- }
- ]
- },
- {
- "roundMs": 3312.409875000012,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.07550000003539026
- }
- ]
- },
- {
- "roundMs": 3459.970583000104,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.052207999979145825
- }
- ]
- },
- {
- "roundMs": 5012.643625000026,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.028624999918974936
- }
- ]
- },
- {
- "roundMs": 13875.179999999935,
- "firstDeltaMs": null,
- "calls": [
+ "name": "split_at",
+ "execMs": 0.04033400001935661
+ },
{
"name": "split_at",
- "execMs": 0.05045800004154444
+ "execMs": 0.005167000024812296
},
{
"name": "split_at",
- "execMs": 0.016417000093497336
+ "execMs": 0.011124999989988282
},
{
"name": "split_at",
- "execMs": 0.11158400005660951
- }
- ]
- },
- {
- "roundMs": 1971.9100000000326,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.0072920000238809735
+ },
{
- "name": "get_state",
- "execMs": 0.10187499993480742
+ "name": "split_at",
+ "execMs": 0.010417000012239441
}
]
},
{
- "roundMs": 3796.7140000000363,
+ "roundMs": 2000.4973750000063,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.03904199996031821
+ "execMs": 0.07583399998839013
},
{
"name": "delete_item",
- "execMs": 0.018416999955661595
- }
- ]
- },
- {
- "roundMs": 2091.576417000033,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.046083000022917986
- }
- ]
- },
- {
- "roundMs": 4720.85833399999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.03429199999663979
- }
- ]
- },
- {
- "roundMs": 5030.000333999982,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "refine_speech_cuts",
- "execMs": 0.060999999986961484
- }
- ]
- },
- {
- "roundMs": 5216.500625000102,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "subtitles_generate",
- "execMs": 0.00820799998473376
- }
- ]
- },
- {
- "roundMs": 1982.530625000014,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.044916999991983175
+ "execMs": 0.007458000007318333
}
]
},
{
- "roundMs": 4638.365584000014,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "subtitles_remove_track",
- "execMs": 0.016667000018060207
- },
- {
- "name": "subtitles_generate",
- "execMs": 0.001917000045068562
- }
- ]
- },
- {
- "roundMs": 1808.6414159999695,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.013417000067420304
- }
- ]
- },
- {
- "roundMs": 4563.772916999995,
- "firstDeltaMs": 4151.766291999957,
+ "roundMs": 1251.6418750000012,
+ "firstDeltaMs": 1027.279292000021,
"calls": []
}
]
@@ -14431,411 +13922,116 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
- "name": "list_skills",
+ "name": "split_at",
"round": 0,
- "atMs": 5361.157041999977,
- "execMs": 0.13974999997299165
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 7085.969749999931,
- "execMs": 0.005459000007249415
- },
- {
- "name": "detect_silence",
- "round": 2,
- "atMs": 10207.777166999993,
- "execMs": 0.055958000011742115
- },
- {
- "name": "listen_audio",
- "round": 3,
- "atMs": 12406.029916999978,
- "execMs": 0.038958000019192696
- },
- {
- "name": "detect_silence",
- "round": 4,
- "atMs": 20559.41287500004,
- "execMs": 0.026833000010810792
+ "atMs": 2778.1454169999924,
+ "execMs": 0.15554199999314733
},
{
"name": "split_at",
- "round": 5,
- "atMs": 25559.428624999942,
- "execMs": 0.18974999990314245
- },
- {
- "name": "get_state",
- "round": 6,
- "atMs": 27776.948957999935,
- "execMs": 0.11004199995659292
+ "round": 0,
+ "atMs": 2778.3062079999945,
+ "execMs": 0.008416000026045367
},
{
"name": "split_at",
- "round": 7,
- "atMs": 30806.889417,
- "execMs": 0.07362499996088445
- },
- {
- "name": "get_state",
- "round": 8,
- "atMs": 32459.154207999934,
- "execMs": 0.0616669999435544
- },
- {
- "name": "delete_item",
- "round": 9,
- "atMs": 35315.53524999996,
- "execMs": 0.030125000048428774
- },
- {
- "name": "get_state",
- "round": 10,
- "atMs": 36989.01429199998,
- "execMs": 0.07887500000651926
- },
- {
- "name": "delete_cue",
- "round": 11,
- "atMs": 40048.98441699997,
- "execMs": 0.08108399994671345
+ "round": 0,
+ "atMs": 2778.315667000017,
+ "execMs": 0.004833999992115423
},
{
"name": "split_at",
- "round": 12,
- "atMs": 116150.54083299998,
- "execMs": 0.5681249999906868
- },
- {
- "name": "get_state",
- "round": 13,
- "atMs": 117949.17804199993,
- "execMs": 0.19279199989978224
- },
- {
- "name": "delete_item",
- "round": 14,
- "atMs": 131778.15566699998,
- "execMs": 0.13683400000445545
- },
- {
- "name": "get_state",
- "round": 15,
- "atMs": 142995.511542,
- "execMs": 0.01762499997857958
- },
- {
- "name": "delete_item",
- "round": 16,
- "atMs": 145886.48474999995,
- "execMs": 0.21266700001433492
- },
- {
- "name": "update_audio",
- "round": 17,
- "atMs": 147605.89949999994,
- "execMs": 0.14729200000874698
- },
- {
- "name": "update_cue",
- "round": 18,
- "atMs": 150255.126375,
- "execMs": 0.019332999945618212
- },
- {
- "name": "update_cue",
- "round": 19,
- "atMs": 154446.49320799997,
- "execMs": 0.03683300002012402
- },
- {
- "name": "update_cue",
- "round": 19,
- "atMs": 154446.540333,
- "execMs": 0.008791000000201166
- },
- {
- "name": "update_cue",
- "round": 19,
- "atMs": 154446.55008299998,
- "execMs": 0.002749999985098839
- },
- {
- "name": "get_state",
- "round": 20,
- "atMs": 176802.41487500002,
- "execMs": 0.30904199997894466
- },
- {
- "name": "refine_speech_cuts",
- "round": 21,
- "atMs": 183656.07608299993,
- "execMs": 0.03479199996218085
+ "round": 0,
+ "atMs": 2778.3209999999963,
+ "execMs": 0.005124999996041879
},
{
- "name": "get_state",
- "round": 22,
- "atMs": 185497.74308299995,
- "execMs": 0.05745800002478063
- }
- ],
- "timings": {
- "gateMs": 1695.6383330000099,
- "gateSkipped": false,
- "ttftMs": 188011.281708,
- "modelMs": 104663.05691699986,
- "toolMs": 2.6364599996013567,
- "totalMs": 106361.33170999947,
- "retryMs": 82791.06795699999,
- "retries": 3,
- "rounds": [
- {
- "roundMs": 3664.894667000044,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.13974999997299165
- }
- ]
- },
- {
- "roundMs": 1724.590499999933,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.005459000007249415
- }
- ]
- },
- {
- "roundMs": 3121.75091599999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.055958000011742115
- }
- ]
- },
- {
- "roundMs": 2198.120750000002,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.038958000019192696
- }
- ]
- },
- {
- "roundMs": 8153.270833999966,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.026833000010810792
- }
- ]
- },
- {
- "roundMs": 4999.916833999916,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.18974999990314245
- }
- ]
- },
- {
- "roundMs": 2217.2694580000825,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.11004199995659292
- }
- ]
- },
- {
- "roundMs": 3029.7853749999776,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.07362499996088445
- }
- ]
- },
- {
- "roundMs": 1652.138707999955,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.0616669999435544
- }
- ]
- },
- {
- "roundMs": 2856.2732090000063,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.030125000048428774
- }
- ]
- },
- {
- "roundMs": 1673.4063340000575,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.07887500000651926
- }
- ]
- },
- {
- "roundMs": 3059.8248329999624,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_cue",
- "execMs": 0.08108399994671345
- }
- ]
- },
- {
- "roundMs": 12829.972415999975,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.5681249999906868
- }
- ]
- },
- {
- "roundMs": 1797.8883330000099,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.19279199989978224
- }
- ]
- },
- {
- "roundMs": 3777.943500000052,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.13683400000445545
- }
- ]
- },
- {
- "roundMs": 1747.6191250000848,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.01762499997857958
- }
- ]
- },
- {
- "roundMs": 2890.9282079999102,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.21266700001433492
- }
- ]
- },
+ "name": "split_at",
+ "round": 0,
+ "atMs": 2778.3266670000157,
+ "execMs": 0.005624999990686774
+ },
+ {
+ "name": "delete_item",
+ "round": 1,
+ "atMs": 4251.072583000001,
+ "execMs": 0.16195799998240545
+ },
+ {
+ "name": "delete_item",
+ "round": 1,
+ "atMs": 4251.246375000017,
+ "execMs": 0.05795899999793619
+ },
+ {
+ "name": "delete_item",
+ "round": 1,
+ "atMs": 4251.307292000012,
+ "execMs": 0.02204199999687262
+ }
+ ],
+ "timings": {
+ "gateMs": 1355.7745829999913,
+ "gateSkipped": false,
+ "ttftMs": 5265.001874999987,
+ "modelMs": 4193.605707999988,
+ "toolMs": 0.42150099997525103,
+ "totalMs": 5549.801791999955,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
{
- "roundMs": 1719.1646250000922,
+ "roundMs": 1422.1232920000039,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_audio",
- "execMs": 0.14729200000874698
- }
- ]
- },
- {
- "roundMs": 2649.0576669999864,
- "firstDeltaMs": null,
- "calls": [
+ "name": "split_at",
+ "execMs": 0.15554199999314733
+ },
{
- "name": "update_cue",
- "execMs": 0.019332999945618212
- }
- ]
- },
- {
- "roundMs": 4191.315541999997,
- "firstDeltaMs": null,
- "calls": [
+ "name": "split_at",
+ "execMs": 0.008416000026045367
+ },
{
- "name": "update_cue",
- "execMs": 0.03683300002012402
+ "name": "split_at",
+ "execMs": 0.004833999992115423
},
{
- "name": "update_cue",
- "execMs": 0.008791000000201166
+ "name": "split_at",
+ "execMs": 0.005124999996041879
},
{
- "name": "update_cue",
- "execMs": 0.002749999985098839
+ "name": "split_at",
+ "execMs": 0.005624999990686774
}
]
},
{
- "roundMs": 22355.700249999994,
+ "roundMs": 1472.690540999989,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.30904199997894466
- }
- ]
- },
- {
- "roundMs": 6853.262416000012,
- "firstDeltaMs": null,
- "calls": [
+ "name": "delete_item",
+ "execMs": 0.16195799998240545
+ },
{
- "name": "refine_speech_cuts",
- "execMs": 0.03479199996218085
- }
- ]
- },
- {
- "roundMs": 1841.5772919999436,
- "firstDeltaMs": null,
- "calls": [
+ "name": "delete_item",
+ "execMs": 0.05795899999793619
+ },
{
- "name": "get_state",
- "execMs": 0.05745800002478063
+ "name": "delete_item",
+ "execMs": 0.02204199999687262
}
]
},
{
- "roundMs": 3657.3851249999134,
- "firstDeltaMs": 2513.4003749999683,
+ "roundMs": 1298.7918749999953,
+ "firstDeltaMs": 1013.6473750000005,
"calls": []
}
]
@@ -14850,27 +14046,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1895,
- "p95": 1962,
- "mean": 1762,
+ "p50": 1073,
+ "p95": 1235,
+ "mean": 1115,
"n": 3
},
"ttftMs": {
- "p50": 24656,
- "p95": 32325,
- "mean": 21411,
+ "p50": 3842,
+ "p95": 4706,
+ "mean": 4056,
"n": 3
},
"totalMs": {
- "p50": 7590,
- "p95": 25040,
- "mean": 12959,
+ "p50": 4587,
+ "p95": 5396,
+ "mean": 4615,
"n": 3
},
"modelMs": {
- "p50": 5628,
- "p95": 23612,
- "mean": 11197,
+ "p50": 3352,
+ "p95": 4359,
+ "mean": 3500,
"n": 3
},
"toolMs": {
@@ -14887,39 +14083,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 5299.196208000067,
- "execMs": 0.006874999962747097
+ "atMs": 3397.256207999948,
+ "execMs": 0.006041999964509159
}
],
"timings": {
- "gateMs": 1962.1910829999251,
+ "gateMs": 1037.703916999977,
"gateSkipped": false,
- "ttftMs": 7254.026000000071,
- "modelMs": 5627.594498999999,
- "toolMs": 0.006874999962747097,
- "totalMs": 7589.792456999887,
+ "ttftMs": 4705.729499999958,
+ "modelMs": 4358.615707999968,
+ "toolMs": 0.006041999964509159,
+ "totalMs": 5396.32566699991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3336.6662080000388,
+ "roundMs": 2359.0491659999825,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.006874999962747097
+ "execMs": 0.006041999964509159
}
]
},
{
- "roundMs": 2290.92829099996,
- "firstDeltaMs": 1954.7702079999726,
+ "roundMs": 1999.566541999986,
+ "firstDeltaMs": 1308.416499999992,
"calls": []
}
]
@@ -14927,39 +14124,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 20952.293457999942,
- "execMs": 0.024999999906867743
+ "atMs": 2468.955541000003,
+ "execMs": 0.007542000035755336
}
],
"timings": {
- "gateMs": 1895.4627079999773,
+ "gateMs": 1073.1390419999952,
"gateSkipped": false,
- "ttftMs": 32324.52641599998,
- "modelMs": 4351.016667000018,
- "toolMs": 0.024999999906867743,
- "totalMs": 6246.504374999902,
- "retryMs": 26382.186418000027,
- "retries": 3,
+ "ttftMs": 3620.630833000003,
+ "modelMs": 2789.2719170000055,
+ "toolMs": 0.007542000035755336,
+ "totalMs": 3862.4185010000365,
+ "retryMs": 0,
+ "retries": 0,
"rounds": [
{
- "roundMs": 2447.139542000019,
+ "roundMs": 1395.458958000003,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.024999999906867743
+ "execMs": 0.007542000035755336
}
]
},
{
- "roundMs": 1903.877124999999,
- "firstDeltaMs": 1599.0556250000373,
+ "roundMs": 1393.8129590000026,
+ "firstDeltaMs": 1151.6234169999952,
"calls": []
}
]
@@ -14967,39 +14165,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 4329.292707999935,
- "execMs": 0.004791000043042004
+ "atMs": 2488.5043339999975,
+ "execMs": 0.026541999948676676
}
],
"timings": {
- "gateMs": 1428.136999999988,
+ "gateMs": 1234.6572500000475,
"gateSkipped": false,
- "ttftMs": 24655.841916000005,
- "modelMs": 23612.341082999832,
- "toolMs": 0.004791000043042004,
- "totalMs": 25040.482873999863,
+ "ttftMs": 3841.875333999982,
+ "modelMs": 3352.2924999999814,
+ "toolMs": 0.026541999948676676,
+ "totalMs": 4586.976291999978,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2901.001791999908,
+ "roundMs": 1253.4998749999795,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.004791000043042004
+ "execMs": 0.026541999948676676
}
]
},
{
- "roundMs": 20711.339290999924,
- "firstDeltaMs": 20326.525874999934,
+ "roundMs": 2098.792625000002,
+ "firstDeltaMs": 1353.2687499999884,
"calls": []
}
]
@@ -15014,124 +14213,77 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1855,
- "p95": 1955,
- "mean": 1888,
+ "p50": 1031,
+ "p95": 1899,
+ "mean": 1312,
"n": 3
},
"ttftMs": {
- "p50": 9513,
- "p95": 150497,
- "mean": 56065,
+ "p50": 3635,
+ "p95": 4893,
+ "mean": 4045,
"n": 3
},
"totalMs": {
- "p50": 9941,
- "p95": 150992,
- "mean": 56532,
+ "p50": 4091,
+ "p95": 5716,
+ "mean": 4589,
"n": 3
},
"modelMs": {
- "p50": 8087,
- "p95": 149037,
- "mean": 54644,
+ "p50": 3085,
+ "p95": 3816,
+ "mean": 3276,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
"rounds": {
- "mean": 3,
- "max": 5
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 56199.72558299999,
- "execMs": 0.34112499991897494
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 58048.66479099996,
- "execMs": 0.029624999966472387
- },
- {
- "name": "listen_audio",
- "round": 2,
- "atMs": 72017.88354099996,
- "execMs": 0.053875000099651515
- },
{
"name": "generate_scene",
- "round": 3,
- "atMs": 144456.59574999998,
- "execMs": 0.17541599995456636
+ "round": 0,
+ "atMs": 3445.7627499999944,
+ "execMs": 0.31825000001117587
}
],
"timings": {
- "gateMs": 1954.548542000004,
+ "gateMs": 1899.0022079999908,
"gateSkipped": false,
- "ttftMs": 150496.85716600006,
- "modelMs": 149037.106041,
- "toolMs": 0.6000409999396652,
- "totalMs": 150992.25462399994,
+ "ttftMs": 4892.771874999977,
+ "modelMs": 3816.3551669999724,
+ "toolMs": 0.31825000001117587,
+ "totalMs": 5715.675624999974,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 54244.532458,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.34112499991897494
- }
- ]
- },
- {
- "roundMs": 1848.5133749999804,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.029624999966472387
- }
- ]
- },
- {
- "roundMs": 13969.014582999982,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.053875000099651515
- }
- ]
- },
- {
- "roundMs": 72438.59874999989,
+ "roundMs": 1546.0895419999724,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.17541599995456636
+ "execMs": 0.31825000001117587
}
]
},
{
- "roundMs": 6536.44687500014,
- "firstDeltaMs": 6040.064083999954,
+ "roundMs": 2270.265625,
+ "firstDeltaMs": 1446.6444580000243,
"calls": []
}
]
@@ -15139,39 +14291,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 7385.8605840001255,
- "execMs": 0.025750000029802322
+ "atMs": 2501.9412499999744,
+ "execMs": 0.023250000027474016
}
],
"timings": {
- "gateMs": 1854.655291999923,
+ "gateMs": 1005.4594579999684,
"gateSkipped": false,
- "ttftMs": 9512.54108399991,
- "modelMs": 8086.517959000077,
- "toolMs": 0.025750000029802322,
- "totalMs": 9941.19900100003,
+ "ttftMs": 3605.4224169999943,
+ "modelMs": 3085.2516659999965,
+ "toolMs": 0.023250000027474016,
+ "totalMs": 4090.7343739999924,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5530.949709000066,
+ "roundMs": 1496.2741249999963,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.025750000029802322
+ "execMs": 0.023250000027474016
}
]
},
{
- "roundMs": 2555.568250000011,
- "firstDeltaMs": 2126.625416000141,
+ "roundMs": 1588.9775410000002,
+ "firstDeltaMs": 1103.421541000018,
"calls": []
}
]
@@ -15179,39 +14332,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 5760.838500000071,
- "execMs": 0.009166999952867627
+ "atMs": 2613.179291999957,
+ "execMs": 0.03950000001350418
}
],
"timings": {
- "gateMs": 1853.9568749999162,
+ "gateMs": 1031.3022079999791,
"gateSkipped": false,
- "ttftMs": 8184.1908750000875,
- "modelMs": 6809.578625000082,
- "toolMs": 0.009166999952867627,
- "totalMs": 8663.544666999951,
+ "ttftMs": 3635.3549999999814,
+ "modelMs": 2927.7917510000407,
+ "toolMs": 0.03950000001350418,
+ "totalMs": 3959.1334590000333,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3906.7915420001373,
+ "roundMs": 1581.5134170000092,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.009166999952867627
+ "execMs": 0.03950000001350418
}
]
},
{
- "roundMs": 2902.7870829999447,
- "firstDeltaMs": 2423.330125000095,
+ "roundMs": 1346.2783340000315,
+ "firstDeltaMs": 1022.0488340000156,
"calls": []
}
]
@@ -15226,27 +14380,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1746,
- "p95": 2897,
- "mean": 2062,
+ "p50": 1449,
+ "p95": 1462,
+ "mean": 1323,
"n": 3
},
"ttftMs": {
- "p50": 9658,
- "p95": 9737,
- "mean": 9636,
+ "p50": 3983,
+ "p95": 4086,
+ "mean": 3992,
"n": 3
},
"totalMs": {
- "p50": 10092,
- "p95": 10203,
- "mean": 10070,
+ "p50": 4501,
+ "p95": 4663,
+ "mean": 4522,
"n": 3
},
"modelMs": {
- "p50": 8457,
- "p95": 8550,
- "mean": 8008,
+ "p50": 3213,
+ "p95": 3442,
+ "mean": 3199,
"n": 3
},
"toolMs": {
@@ -15263,39 +14417,40 @@
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 7343.86795900017,
- "execMs": 0.09333299985155463
+ "atMs": 2890.453333000012,
+ "execMs": 0.28154200001154095
}
],
"timings": {
- "gateMs": 2897.011417000089,
+ "gateMs": 1449.3699999999953,
"gateSkipped": false,
- "ttftMs": 9514.484209000133,
- "modelMs": 7016.148249999853,
- "toolMs": 0.09333299985155463,
- "totalMs": 9913.252999999793,
+ "ttftMs": 4085.5166249999893,
+ "modelMs": 3213.305124000006,
+ "toolMs": 0.28154200001154095,
+ "totalMs": 4662.956666000013,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4446.300667000003,
+ "roundMs": 1440.6408330000122,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.09333299985155463
+ "execMs": 0.28154200001154095
}
]
},
{
- "roundMs": 2569.8475829998497,
- "firstDeltaMs": 2170.510166999884,
+ "roundMs": 1772.6642909999937,
+ "firstDeltaMs": 1194.7272499999963,
"calls": []
}
]
@@ -15303,39 +14458,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 7633.012416000012,
- "execMs": 0.021708999993279576
+ "atMs": 2926.0276669999585,
+ "execMs": 0.024291999987326562
}
],
"timings": {
- "gateMs": 1541.4845000000205,
+ "gateMs": 1461.5212499999907,
"gateSkipped": false,
- "ttftMs": 9736.69200000004,
- "modelMs": 8550.450332999928,
- "toolMs": 0.021708999993279576,
- "totalMs": 10091.956541999942,
+ "ttftMs": 3983.1796669999603,
+ "modelMs": 2941.3201659999904,
+ "toolMs": 0.024291999987326562,
+ "totalMs": 4402.865707999968,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6091.375958000077,
+ "roundMs": 1464.326000000001,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.021708999993279576
+ "execMs": 0.024291999987326562
}
]
},
{
- "roundMs": 2459.074374999851,
- "firstDeltaMs": 2103.6097079999745,
+ "roundMs": 1476.9941659999895,
+ "firstDeltaMs": 1057.099791000015,
"calls": []
}
]
@@ -15343,39 +14499,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 7538.122915999964,
- "execMs": 0.01566599984653294
+ "atMs": 2630.225832999975,
+ "execMs": 0.049832999997306615
}
],
"timings": {
- "gateMs": 1746.045124999946,
+ "gateMs": 1059.5700419999775,
"gateSkipped": false,
- "ttftMs": 9658.212708000094,
- "modelMs": 8457.418874999974,
- "toolMs": 0.01566599984653294,
- "totalMs": 10203.479665999766,
+ "ttftMs": 3907.8239169999724,
+ "modelMs": 3441.7006249999395,
+ "toolMs": 0.049832999997306615,
+ "totalMs": 4501.320499999914,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5791.495124999899,
+ "roundMs": 1570.123041999992,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.01566599984653294
+ "execMs": 0.049832999997306615
}
]
},
{
- "roundMs": 2665.9237500000745,
- "firstDeltaMs": 2120.059833999956,
+ "roundMs": 1871.5775829999475,
+ "firstDeltaMs": 1277.5052499999874,
"calls": []
}
]
@@ -15396,21 +14553,21 @@
"n": 3
},
"ttftMs": {
- "p50": 10524,
- "p95": 19520,
- "mean": 13149,
+ "p50": 3775,
+ "p95": 5461,
+ "mean": 4005,
"n": 3
},
"totalMs": {
- "p50": 10968,
- "p95": 19851,
- "mean": 13451,
+ "p50": 4546,
+ "p95": 11403,
+ "mean": 6369,
"n": 3
},
"modelMs": {
- "p50": 10968,
- "p95": 19851,
- "mean": 13450,
+ "p50": 4546,
+ "p95": 11402,
+ "mean": 6369,
"n": 3
},
"toolMs": {
@@ -15420,46 +14577,47 @@
"n": 3
},
"rounds": {
- "mean": 3,
- "max": 5
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 4424.4065409998875,
- "execMs": 0.30137500003911555
+ "atMs": 1379.9913749999832,
+ "execMs": 0.38820899999700487
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 9401.296290999977,
- "modelMs": 9532.143208999885,
- "toolMs": 0.30137500003911555,
- "totalMs": 9532.444583999924,
+ "ttftMs": 5460.66745899996,
+ "modelMs": 11402.306167000032,
+ "toolMs": 0.38820899999700487,
+ "totalMs": 11402.694376000029,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4423.831291999901,
+ "roundMs": 1379.540417000011,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.30137500003911555
+ "execMs": 0.38820899999700487
}
]
},
{
- "roundMs": 5108.3119169999845,
- "firstDeltaMs": 4976.51316699991,
+ "roundMs": 10022.76575000002,
+ "firstDeltaMs": 4080.2415829999954,
"calls": []
}
]
@@ -15467,39 +14625,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "generate_scene",
"round": 0,
- "atMs": 7947.714042000007,
- "execMs": 0.016707999864593148
+ "atMs": 2255.989583000017,
+ "execMs": 0.025166999956127256
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 10524.085959000047,
- "modelMs": 10968.315292000072,
- "toolMs": 0.016707999864593148,
- "totalMs": 10968.331999999937,
+ "ttftMs": 3774.678249999997,
+ "modelMs": 4546.057084000029,
+ "toolMs": 0.025166999956127256,
+ "totalMs": 4546.082250999985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7947.459833000088,
+ "roundMs": 2255.790292000049,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.016707999864593148
+ "execMs": 0.025166999956127256
}
]
},
{
- "roundMs": 3020.855458999984,
- "firstDeltaMs": 2576.342249999987,
+ "roundMs": 2290.2667919999803,
+ "firstDeltaMs": 1518.647500000021,
"calls": []
}
]
@@ -15507,87 +14666,40 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 4109.571583000012,
- "execMs": 0.08525000000372529
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 5662.970540999901,
- "execMs": 0.025625000009313226
- },
- {
- "name": "listen_audio",
- "round": 2,
- "atMs": 7722.060999999987,
- "execMs": 0.012790999840945005
- },
{
"name": "generate_scene",
- "round": 3,
- "atMs": 17411.47220800002,
- "execMs": 0.016916999826207757
+ "round": 0,
+ "atMs": 1261.1126250000088,
+ "execMs": 0.02812499995343387
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 19520.12358299992,
- "modelMs": 19850.646666999906,
- "toolMs": 0.14058299968019128,
- "totalMs": 19850.787249999586,
+ "ttftMs": 2779.6992919999757,
+ "modelMs": 3159.465041999996,
+ "toolMs": 0.02812499995343387,
+ "totalMs": 3159.4931669999496,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4109.411334000062,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.08525000000372529
- }
- ]
- },
- {
- "roundMs": 1553.0074579999782,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.025625000009313226
- }
- ]
- },
- {
- "roundMs": 2059.002542000031,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.012790999840945005
- }
- ]
- },
- {
- "roundMs": 9689.363374999957,
+ "roundMs": 1260.8780830000178,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.016916999826207757
+ "execMs": 0.02812499995343387
}
]
},
{
- "roundMs": 2439.8619579998776,
- "firstDeltaMs": 2108.6132079998497,
+ "roundMs": 1898.5869589999784,
+ "firstDeltaMs": 1518.5193749999744,
"calls": []
}
]
@@ -15602,27 +14714,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1861,
- "p95": 2507,
- "mean": 2066,
+ "p50": 1783,
+ "p95": 1805,
+ "mean": 1697,
"n": 3
},
"ttftMs": {
- "p50": 8336,
- "p95": 16959,
- "mean": 10855,
+ "p50": 2946,
+ "p95": 3134,
+ "mean": 2912,
"n": 3
},
"totalMs": {
- "p50": 9017,
- "p95": 17698,
- "mean": 11458,
+ "p50": 3650,
+ "p95": 3947,
+ "mean": 3626,
"n": 3
},
"modelMs": {
- "p50": 6509,
- "p95": 15837,
- "mean": 9391,
+ "p50": 1868,
+ "p95": 2142,
+ "mean": 1929,
"n": 3
},
"toolMs": {
@@ -15632,29 +14744,30 @@
"n": 3
},
"rounds": {
- "mean": 1.7,
- "max": 3
+ "mean": 1,
+ "max": 1
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1830.2608750001527,
+ "gateMs": 1804.7535420000204,
"gateSkipped": false,
- "ttftMs": 7270.8436670000665,
- "modelMs": 5827.920708999969,
+ "ttftMs": 3133.6739579999703,
+ "modelMs": 2142.4990830000024,
"toolMs": 0,
- "totalMs": 7658.181584000122,
+ "totalMs": 3947.252625000023,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5827.920708999969,
- "firstDeltaMs": 5440.134917000076,
+ "roundMs": 2142.4990830000024,
+ "firstDeltaMs": 1328.6962079999503,
"calls": []
}
]
@@ -15662,55 +14775,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 4907.358875000151,
- "execMs": 0.018417000072076917
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 6570.017875000136,
- "execMs": 0.011500000022351742
- }
- ],
+ "trace": [],
"timings": {
- "gateMs": 1861.1924169999547,
+ "gateMs": 1503.3175000000047,
"gateSkipped": false,
- "ttftMs": 16958.63462500018,
- "modelMs": 15836.942209,
- "toolMs": 0.02991700009442866,
- "totalMs": 17698.16454300005,
+ "ttftMs": 2655.687540999963,
+ "modelMs": 1778.1699590000208,
+ "toolMs": 0,
+ "totalMs": 3281.4874590000254,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3045.7635840000585,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.018417000072076917
- }
- ]
- },
- {
- "roundMs": 1662.5172500000335,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.011500000022351742
- }
- ]
- },
- {
- "roundMs": 11128.661374999909,
- "firstDeltaMs": 10388.542875000043,
+ "roundMs": 1778.1699590000208,
+ "firstDeltaMs": 1152.2402920000022,
"calls": []
}
]
@@ -15718,22 +14799,23 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2507.452917000046,
+ "gateMs": 1782.8941670000204,
"gateSkipped": false,
- "ttftMs": 8335.72666699998,
- "modelMs": 6509.443708000006,
+ "ttftMs": 2946.3546250000363,
+ "modelMs": 1867.5433329999796,
"toolMs": 0,
- "totalMs": 9016.896625000052,
+ "totalMs": 3650.4375,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6509.443708000006,
- "firstDeltaMs": 5827.947458000155,
+ "roundMs": 1867.5433329999796,
+ "firstDeltaMs": 1163.3432909999974,
"calls": []
}
]
@@ -15754,21 +14836,21 @@
"n": 3
},
"ttftMs": {
- "p50": 25170,
- "p95": 26731,
- "mean": 25594,
+ "p50": 5880,
+ "p95": 9207,
+ "mean": 6396,
"n": 3
},
"totalMs": {
- "p50": 25461,
- "p95": 27210,
- "mean": 25999,
+ "p50": 6156,
+ "p95": 9545,
+ "mean": 6881,
"n": 3
},
"modelMs": {
- "p50": 25460,
- "p95": 27210,
- "mean": 25998,
+ "p50": 6156,
+ "p95": 9544,
+ "mean": 6881,
"n": 3
},
"toolMs": {
@@ -15778,120 +14860,111 @@
"n": 3
},
"rounds": {
- "mean": 4.7,
+ "mean": 4,
"max": 6
}
},
"runs": [
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 7632.725291999988,
- "execMs": 0.609209000132978
+ "atMs": 1409.8684580000117,
+ "execMs": 0.7161250000353903
},
{
"name": "watch_video",
"round": 1,
- "atMs": 11243.164542000042,
- "execMs": 0.037583000026643276
+ "atMs": 2779.4305000000168,
+ "execMs": 0.03008400002727285
},
{
"name": "watch_video",
- "round": 1,
- "atMs": 11243.24116700003,
- "execMs": 0.002624999964609742
- },
- {
- "name": "recut_scene",
"round": 2,
- "atMs": 18551.571542000165,
- "execMs": 0.011042000027373433
+ "atMs": 4079.7625000000116,
+ "execMs": 0.021499999973457307
},
{
- "name": "wait_for_renders",
+ "name": "regenerate_shot",
"round": 3,
- "atMs": 21137.21529199998,
- "execMs": 0.013583000050857663
+ "atMs": 6190.854082999984,
+ "execMs": 0.06066600000485778
},
{
- "name": "get_state",
+ "name": "regenerate_shot",
"round": 4,
- "atMs": 22981.304625000106,
- "execMs": 0.021333999931812286
+ "atMs": 7980.315750000009,
+ "execMs": 0.03337499999906868
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 25170.49254200002,
- "modelMs": 25460.122539999895,
- "toolMs": 0.6953760001342744,
- "totalMs": 25460.81791600003,
+ "ttftMs": 9207.178916000004,
+ "modelMs": 9543.787540999998,
+ "toolMs": 0.8617500000400469,
+ "totalMs": 9544.649291000038,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7632.472832999891,
+ "roundMs": 1408.9324999999953,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.609209000132978
+ "execMs": 0.7161250000353903
}
]
},
{
- "roundMs": 3609.6450839999598,
+ "roundMs": 1368.6994159999886,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.037583000026643276
- },
- {
- "name": "watch_video",
- "execMs": 0.002624999964609742
+ "execMs": 0.03008400002727285
}
]
},
{
- "roundMs": 7308.2820409999695,
+ "roundMs": 1300.2059579999768,
"firstDeltaMs": null,
"calls": [
{
- "name": "recut_scene",
- "execMs": 0.011042000027373433
+ "name": "watch_video",
+ "execMs": 0.021499999973457307
}
]
},
{
- "roundMs": 2585.601958000101,
+ "roundMs": 2110.8000839999877,
"firstDeltaMs": null,
"calls": [
{
- "name": "wait_for_renders",
- "execMs": 0.013583000050857663
+ "name": "regenerate_shot",
+ "execMs": 0.06066600000485778
}
]
},
{
- "roundMs": 1844.033666000003,
+ "roundMs": 1789.2530830000178,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.021333999931812286
+ "name": "regenerate_shot",
+ "execMs": 0.03337499999906868
}
]
},
{
- "roundMs": 2480.0869579999708,
- "firstDeltaMs": 2189.147333000088,
+ "roundMs": 1565.8965000000317,
+ "firstDeltaMs": 1226.7461250000051,
"calls": []
}
]
@@ -15899,91 +14972,86 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
- "name": "listen_audio",
+ "name": "watch_video",
"round": 0,
- "atMs": 4210.046833000146,
- "execMs": 0.021582999965175986
+ "atMs": 1293.8800419999752,
+ "execMs": 0.05791600001975894
},
{
"name": "watch_video",
- "round": 1,
- "atMs": 9391.733417000156,
- "execMs": 0.04858299996703863
+ "round": 0,
+ "atMs": 1293.9789170000004,
+ "execMs": 0.008040999993681908
},
{
- "name": "watch_video",
- "round": 1,
- "atMs": 9391.82362499996,
- "execMs": 0.008416999829933047
+ "name": "listen_audio",
+ "round": 0,
+ "atMs": 1294.0072919999948,
+ "execMs": 0.0020830000285059214
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 19628.182958000107,
- "execMs": 0.02583299996331334
+ "round": 1,
+ "atMs": 2692.9634589999914,
+ "execMs": 0.035334000014699996
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 19628.213833000045,
- "execMs": 0.004541999893262982
+ "round": 1,
+ "atMs": 2693.007458999986,
+ "execMs": 0.007582999998703599
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 26730.926582999993,
- "modelMs": 27210.120207999833,
- "toolMs": 0.10895799961872399,
- "totalMs": 27210.22916599945,
+ "ttftMs": 4101.407666999963,
+ "modelMs": 4942.625582999957,
+ "toolMs": 0.11095700005535036,
+ "totalMs": 4942.736540000013,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4209.712624999927,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.021582999965175986
- }
- ]
- },
- {
- "roundMs": 5181.459167000139,
+ "roundMs": 1293.7399159999914,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.04858299996703863
+ "execMs": 0.05791600001975894
},
{
"name": "watch_video",
- "execMs": 0.008416999829933047
+ "execMs": 0.008040999993681908
+ },
+ {
+ "name": "listen_audio",
+ "execMs": 0.0020830000285059214
}
]
},
{
- "roundMs": 10236.238707999932,
+ "roundMs": 1398.8245829999796,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.02583299996331334
+ "execMs": 0.035334000014699996
},
{
"name": "regenerate_shot",
- "execMs": 0.004541999893262982
+ "execMs": 0.007582999998703599
}
]
},
{
- "roundMs": 7582.709707999835,
- "firstDeltaMs": 7102.684790999861,
+ "roundMs": 2250.0610839999863,
+ "firstDeltaMs": 1408.35275000002,
"calls": []
}
]
@@ -15991,91 +15059,86 @@
},
{
"pass": true,
- "intent": "work",
+ "intent": "complex",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"trace": [
{
- "name": "listen_audio",
+ "name": "watch_video",
"round": 0,
- "atMs": 3586.0944169999566,
- "execMs": 0.035124999936670065
+ "atMs": 1898.9302080000052,
+ "execMs": 0.028624999977182597
},
{
"name": "watch_video",
- "round": 1,
- "atMs": 7627.177957999986,
- "execMs": 0.022209000075235963
+ "round": 0,
+ "atMs": 1898.9744580000406,
+ "execMs": 0.001791000016964972
},
{
- "name": "watch_video",
- "round": 1,
- "atMs": 7627.222666999791,
- "execMs": 0.029124999884516
+ "name": "listen_audio",
+ "round": 0,
+ "atMs": 1898.9773750000168,
+ "execMs": 0.001083000039216131
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 22500.232249999885,
- "execMs": 0.08012499986216426
+ "round": 1,
+ "atMs": 4500.381250000035,
+ "execMs": 0.09658300003502518
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 22500.322791999904,
- "execMs": 0.010624999878928065
+ "round": 1,
+ "atMs": 4500.488750000019,
+ "execMs": 0.013124999997671694
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 24881.571499999845,
- "modelMs": 25324.963082000148,
- "toolMs": 0.17720899963751435,
- "totalMs": 25325.140290999785,
+ "ttftMs": 5879.886916000047,
+ "modelMs": 6156.300373999984,
+ "toolMs": 0.14120700006606057,
+ "totalMs": 6156.44158100005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3585.6859170000535,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.035124999936670065
- }
- ]
- },
- {
- "roundMs": 4040.954541000072,
+ "roundMs": 1898.5506670000032,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.022209000075235963
+ "execMs": 0.028624999977182597
},
{
"name": "watch_video",
- "execMs": 0.029124999884516
+ "execMs": 0.001791000016964972
+ },
+ {
+ "name": "listen_audio",
+ "execMs": 0.001083000039216131
}
]
},
{
- "roundMs": 14872.887457999866,
+ "roundMs": 2601.3342909999774,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.08012499986216426
+ "execMs": 0.09658300003502518
},
{
"name": "regenerate_shot",
- "execMs": 0.010624999878928065
+ "execMs": 0.013124999997671694
}
]
},
{
- "roundMs": 2825.4351660001557,
- "firstDeltaMs": 2381.1570830000564,
+ "roundMs": 1656.4154160000035,
+ "firstDeltaMs": 1379.3476249999949,
"calls": []
}
]