Skip to content

Open source CapCut alternative.

TypeScript

chore: chat eval baseline on the judge

David Le3bedec7a4171
diff --git a/evals/cut-chat.latest-report.json b/evals/cut-chat.latest-report.json
index 2da1f6cf..23970b4a 100644
--- a/evals/cut-chat.latest-report.json
+++ b/evals/cut-chat.latest-report.json
@@ -1,82 +1,81 @@
{
"schema": "cut-chat-eval/v1",
- "generatedAt": "2026-09-03T05:49:07.705Z",
+ "generatedAt": "2026-09-17T12:17:25.851Z",
"base": "http://localhost:3000",
- "runsPerCase": 3,
+ "runsPerCase": 2,
"configs": [
{
"label": "config",
"chatModel": "gemini-3.1-flash-lite | complex→gemini-3.8-flash",
- "gateModel": "gemini-3.1-flash-lite",
"buckets": {
"chat": {
"cases": 9,
"passRate": 1,
"gateMs": {
- "p50": 985,
- "p95": 1154,
- "mean": 814,
+ "p50": 251,
+ "p95": 304,
+ "mean": 249,
"n": 9
},
"ttftMs": {
- "p50": 3323,
- "p95": 5212,
- "mean": 3137,
+ "p50": 2728,
+ "p95": 6411,
+ "mean": 3041,
"n": 9
},
"totalMs": {
- "p50": 3663,
- "p95": 5381,
- "mean": 3601,
+ "p50": 3280,
+ "p95": 6511,
+ "mean": 3372,
"n": 9
},
- "meanRounds": 1.6
+ "meanRounds": 1.5
},
"multi-tool": {
- "cases": 8,
+ "cases": 9,
"passRate": 1,
"gateMs": {
- "p50": 1133,
- "p95": 1670,
- "mean": 1035,
- "n": 8
+ "p50": 245,
+ "p95": 301,
+ "mean": 252,
+ "n": 9
},
"ttftMs": {
- "p50": 5452,
- "p95": 33169,
- "mean": 10717,
- "n": 8
+ "p50": 5792,
+ "p95": 32285,
+ "mean": 10035,
+ "n": 9
},
"totalMs": {
- "p50": 5703,
- "p95": 33649,
- "mean": 11138,
- "n": 8
+ "p50": 6118,
+ "p95": 32550,
+ "mean": 10335,
+ "n": 9
},
- "meanRounds": 4.3
+ "meanRounds": 4.8
},
"single-tool": {
"cases": 24,
- "passRate": 0.99,
+ "passRate": 0.94,
"gateMs": {
- "p50": 1100,
- "p95": 1763,
- "mean": 1105,
+ "p50": 255,
+ "p95": 319,
+ "mean": 289,
"n": 24
},
"ttftMs": {
- "p50": 4144,
- "p95": 7502,
- "mean": 4777,
+ "p50": 3166,
+ "p95": 11762,
+ "mean": 5085,
"n": 24
},
"totalMs": {
- "p50": 4684,
- "p95": 7796,
- "mean": 5165,
+ "p50": 3433,
+ "p95": 12041,
+ "mean": 5332,
"n": 24
},
- "meanRounds": 2.3
+ "meanRounds": 2.6
}
},
"cases": [
@@ -87,34 +86,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 1,
- "mean": 0,
- "n": 3
+ "p50": 304,
+ "p95": 812,
+ "mean": 558,
+ "n": 2
},
"ttftMs": {
- "p50": 3051,
- "p95": 8580,
- "mean": 4800,
- "n": 3
+ "p50": 2728,
+ "p95": 3610,
+ "mean": 3169,
+ "n": 2
},
"totalMs": {
- "p50": 3277,
- "p95": 8889,
- "mean": 5046,
- "n": 3
+ "p50": 2928,
+ "p95": 3713,
+ "mean": 3320,
+ "n": 2
},
"modelMs": {
- "p50": 3275,
- "p95": 8884,
- "mean": 5043,
- "n": 3
+ "p50": 2622,
+ "p95": 2891,
+ "mean": 2756,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 1,
@@ -125,50 +124,36 @@
{
"pass": true,
"intent": "complex",
+ "skill": null,
+ "areas": [
+ "timeline",
+ "inspector",
+ "subtitles",
+ "audio",
+ "library",
+ "side_panel",
+ "editor"
+ ],
+ "declaredTools": 91,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 0.7730830000000424,
- "gateSkipped": true,
- "ttftMs": 8579.68975,
- "modelMs": 8883.585833,
- "toolMs": 0,
- "totalMs": 8889.499,
- "judgeMs": 1284.7025840000006,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 8883.585833,
- "firstDeltaMs": 8574.95825,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 0.11662500000056752,
+ "gateMs": 811.7028329999999,
"gateSkipped": true,
- "ttftMs": 3051.334334000001,
- "modelMs": 3274.6637919999994,
+ "ttftMs": 3609.8061660000003,
+ "modelMs": 2891.1585,
"toolMs": 0,
- "totalMs": 3276.7689999999984,
- "judgeMs": 1050.6233329999995,
+ "totalMs": 3712.7595410000004,
+ "judgeMs": 637.5777499999995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3274.6637919999994,
- "firstDeltaMs": 3049.3071249999994,
+ "roundMs": 2891.1585,
+ "firstDeltaMs": 2789.6068339999997,
"calls": []
}
]
@@ -177,24 +162,35 @@
{
"pass": true,
"intent": "complex",
+ "skill": null,
+ "areas": [
+ "inspector",
+ "subtitles",
+ "audio",
+ "library",
+ "side_panel",
+ "editor"
+ ],
+ "declaredTools": 63,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 0.08633299999928568,
+ "gateMs": 304.1766669999997,
"gateSkipped": true,
- "ttftMs": 2767.4800829999986,
- "modelMs": 2971.503917000002,
+ "ttftMs": 2727.847667,
+ "modelMs": 2621.7646250000007,
"toolMs": 0,
- "totalMs": 2972.5941669999993,
- "judgeMs": 1071.7190840000003,
+ "totalMs": 2927.532292000001,
+ "judgeMs": 206.19129099999918,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2971.503917000002,
- "firstDeltaMs": 2766.4630829999987,
+ "roundMs": 2621.7646250000007,
+ "firstDeltaMs": 2422.3260420000006,
"calls": []
}
]
@@ -209,34 +205,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 959,
- "p95": 1509,
- "mean": 1137,
- "n": 3
+ "p50": 263,
+ "p95": 316,
+ "mean": 290,
+ "n": 2
},
"ttftMs": {
- "p50": 3530,
- "p95": 3689,
- "mean": 3461,
- "n": 3
+ "p50": 4319,
+ "p95": 30883,
+ "mean": 17601,
+ "n": 2
},
"totalMs": {
- "p50": 3663,
- "p95": 4060,
- "mean": 3672,
- "n": 3
+ "p50": 4319,
+ "p95": 31109,
+ "mean": 17714,
+ "n": 2
},
"modelMs": {
- "p50": 3656,
- "p95": 4058,
- "mean": 3667,
- "n": 3
+ "p50": 4053,
+ "p95": 30789,
+ "mean": 17421,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -247,60 +243,33 @@
{
"pass": true,
"intent": "chat",
+ "skill": null,
+ "areas": [],
+ "declaredTools": 0,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1508.666165999999,
- "gateSkipped": false,
- "ttftMs": 3688.9159169999984,
- "modelMs": 4057.8673329999983,
- "toolMs": 0,
- "totalMs": 4060.382125,
- "judgeMs": 1676.7332500000011,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1508.033625,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2549.8337079999983,
- "firstDeltaMs": 2178.6090829999994,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "chat",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 959.1631670000024,
+ "gateMs": 316.32133299999987,
"gateSkipped": false,
- "ttftMs": 3163.849875,
- "modelMs": 3287.459498999997,
+ "ttftMs": 30882.879916,
+ "modelMs": 30789.048667000003,
"toolMs": 0,
- "totalMs": 3292.176540999997,
- "judgeMs": 1232.218915999998,
+ "totalMs": 31108.895458000003,
+ "judgeMs": 538.7172090000022,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 957.825665999997,
- "firstDeltaMs": null,
+ "roundMs": 3538.2138749999995,
+ "firstDeltaMs": 3307.114125,
"calls": []
},
{
- "roundMs": 2329.633833,
- "firstDeltaMs": 2202.6432920000007,
+ "roundMs": 27250.834792,
+ "firstDeltaMs": 27026.010374999998,
"calls": []
}
]
@@ -309,29 +278,33 @@
{
"pass": true,
"intent": "chat",
+ "skill": null,
+ "areas": [],
+ "declaredTools": 0,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 941.7583339999983,
+ "gateMs": 263.0132920000033,
"gateSkipped": false,
- "ttftMs": 3530.4107079999994,
- "modelMs": 3655.570584000001,
+ "ttftMs": 4318.823958000001,
+ "modelMs": 4053.2067920000045,
"toolMs": 0,
- "totalMs": 3662.8358750000007,
- "judgeMs": 1666.0734169999996,
+ "totalMs": 4319.161749999999,
+ "judgeMs": 606.3485000000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 937.5361669999984,
- "firstDeltaMs": null,
+ "roundMs": 1134.1152920000022,
+ "firstDeltaMs": 1008.7466249999998,
"calls": []
},
{
- "roundMs": 2718.0344170000026,
- "firstDeltaMs": 2585.926459000002,
+ "roundMs": 2919.0915000000023,
+ "firstDeltaMs": 2918.957416999998,
"calls": []
}
]
@@ -346,34 +319,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 985,
- "p95": 1023,
- "mean": 992,
- "n": 3
+ "p50": 251,
+ "p95": 282,
+ "mean": 266,
+ "n": 2
},
"ttftMs": {
- "p50": 4431,
- "p95": 8462,
- "mean": 5639,
- "n": 3
+ "p50": 4282,
+ "p95": 4923,
+ "mean": 4603,
+ "n": 2
},
"totalMs": {
- "p50": 4482,
- "p95": 8617,
- "mean": 5708,
- "n": 3
+ "p50": 4458,
+ "p95": 4972,
+ "mean": 4715,
+ "n": 2
},
"modelMs": {
- "p50": 4480,
- "p95": 8613,
- "mean": 5706,
- "n": 3
+ "p50": 4206,
+ "p95": 4677,
+ "mean": 4441,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -384,60 +357,33 @@
{
"pass": true,
"intent": "chat",
+ "skill": null,
+ "areas": [],
+ "declaredTools": 0,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1022.5701249999984,
- "gateSkipped": false,
- "ttftMs": 4431.229707999999,
- "modelMs": 4480.158624999996,
- "toolMs": 0,
- "totalMs": 4481.987207999999,
- "judgeMs": 1367.1809580000045,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1021.7181249999994,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 3458.440499999997,
- "firstDeltaMs": 3407.7709170000016,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "chat",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": "contains an unnecessary question instead of acknowledging the gratitude or ending the conversation.",
- "trace": [],
- "timings": {
- "gateMs": 967.346166000003,
+ "gateMs": 281.59308299999975,
"gateSkipped": false,
- "ttftMs": 4024.539582999998,
- "modelMs": 4023.6377500000017,
+ "ttftMs": 4922.769625000001,
+ "modelMs": 4676.804083000003,
"toolMs": 0,
- "totalMs": 4025.767749999999,
- "judgeMs": 1250.4864580000067,
+ "totalMs": 4972.416833000003,
+ "judgeMs": 489.70624999999563,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 966.9035000000003,
+ "roundMs": 1204.658457999998,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 3056.7342500000013,
- "firstDeltaMs": 3055.675375000006,
+ "roundMs": 3472.1456250000047,
+ "firstDeltaMs": 3422.9637500000026,
"calls": []
}
]
@@ -446,29 +392,33 @@
{
"pass": true,
"intent": "chat",
+ "skill": null,
+ "areas": [],
+ "declaredTools": 0,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": "conversational turns should acknowledge the thanks rather than prompting with a generic question.",
+ "judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 984.9856250000012,
+ "gateMs": 251.05025000000023,
"gateSkipped": false,
- "ttftMs": 8461.616750000001,
- "modelMs": 8613.311624999995,
+ "ttftMs": 4282.4438329999975,
+ "modelMs": 4205.652916999999,
"toolMs": 0,
- "totalMs": 8616.5245,
- "judgeMs": 1384.0475000000006,
+ "totalMs": 4458.468416999996,
+ "judgeMs": 1035.0701249999984,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 983.5761249999996,
+ "roundMs": 1077.3654590000006,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 7629.735499999995,
- "firstDeltaMs": 7475.032208999997,
+ "roundMs": 3128.287457999999,
+ "firstDeltaMs": 2952.6718330000003,
"calls": []
}
]
@@ -483,34 +433,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1670,
- "p95": 1750,
- "mean": 1565,
- "n": 3
+ "p50": 291,
+ "p95": 332,
+ "mean": 311,
+ "n": 2
},
"ttftMs": {
- "p50": 2900,
- "p95": 3431,
- "mean": 3058,
- "n": 3
+ "p50": 2914,
+ "p95": 3019,
+ "mean": 2967,
+ "n": 2
},
"totalMs": {
- "p50": 3174,
- "p95": 3600,
- "mean": 3298,
- "n": 3
+ "p50": 3090,
+ "p95": 3278,
+ "mean": 3184,
+ "n": 2
},
"modelMs": {
- "p50": 3171,
- "p95": 3591,
- "mean": 3293,
- "n": 3
+ "p50": 2795,
+ "p95": 2941,
+ "mean": 2868,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -521,89 +471,17 @@
{
"pass": true,
"intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 2218.0288329999967,
- "execMs": 0.1832079999949201
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 2218.6628749999945,
- "execMs": 0.0032920000012381934
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 2218.7287499999948,
- "execMs": 0.0027919999993173406
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 2218.9429170000003,
- "execMs": 0.021416999996290542
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 2219.0049579999977,
- "execMs": 0.0018750000017462298
- }
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "library",
+ "editor"
],
- "timings": {
- "gateMs": 1750.174708999999,
- "gateSkipped": false,
- "ttftMs": 3431.481749999999,
- "modelMs": 3590.845540999995,
- "toolMs": 0.2125839999935124,
- "totalMs": 3600.3431249999994,
- "judgeMs": 879.0350000000035,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2210.5563330000004,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.1832079999949201
- },
- {
- "name": "update_cue",
- "execMs": 0.0032920000012381934
- },
- {
- "name": "update_cue",
- "execMs": 0.0027919999993173406
- },
- {
- "name": "delete_cue",
- "execMs": 0.021416999996290542
- },
- {
- "name": "update_cue",
- "execMs": 0.0018750000017462298
- }
- ]
- },
- {
- "roundMs": 1380.2892079999947,
- "firstDeltaMs": 1211.8164169999945,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
+ "declaredTools": 85,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -611,74 +489,79 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 1776.591124999999,
- "execMs": 0.006832999999460299
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1776.7309170000008,
- "execMs": 0.003416999999899417
+ "atMs": 1818.1259169999976,
+ "execMs": 0.28608299999905284,
+ "args": {
+ "text": "so today we're",
+ "id": "cue1"
+ }
},
{
"name": "update_cue",
"round": 0,
- "atMs": 1776.8132499999992,
- "execMs": 0.0039590000014868565
+ "atMs": 1818.9567919999972,
+ "execMs": 0.004207999998470768,
+ "args": {
+ "id": "cue3",
+ "text": "and they absolutely love the water"
+ }
},
{
"name": "delete_cue",
"round": 0,
- "atMs": 1776.8711670000048,
- "execMs": 0.006792000000132248
+ "atMs": 1819.2722079999949,
+ "execMs": 0.02195899999787798,
+ "args": {
+ "id": "cue4"
+ }
},
{
"name": "update_cue",
"round": 0,
- "atMs": 1776.9459170000046,
- "execMs": 0.0032079999946290627
+ "atMs": 1819.3593330000003,
+ "execMs": 0.002542000001994893,
+ "args": {
+ "text": "so let's watch them play",
+ "id": "cue5"
+ }
}
],
"timings": {
- "gateMs": 1670.3589169999977,
+ "gateMs": 331.9095410000009,
"gateSkipped": false,
- "ttftMs": 2899.6146249999947,
- "modelMs": 3171.136832999997,
- "toolMs": 0.024208999995607883,
- "totalMs": 3174.3258330000026,
- "judgeMs": 1037.3140419999982,
+ "ttftMs": 3018.7580420000013,
+ "modelMs": 2940.6652909999975,
+ "toolMs": 0.3147919999973965,
+ "totalMs": 3277.793249999995,
+ "judgeMs": 304.1790420000034,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1774.7459579999995,
+ "roundMs": 1483.0979999999981,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.006832999999460299
- },
- {
- "name": "update_cue",
- "execMs": 0.003416999999899417
+ "execMs": 0.28608299999905284
},
{
"name": "update_cue",
- "execMs": 0.0039590000014868565
+ "execMs": 0.004207999998470768
},
{
"name": "delete_cue",
- "execMs": 0.006792000000132248
+ "execMs": 0.02195899999787798
},
{
"name": "update_cue",
- "execMs": 0.0032079999946290627
+ "execMs": 0.002542000001994893
}
]
},
{
- "roundMs": 1396.3908749999973,
- "firstDeltaMs": 1121.820749999999,
+ "roundMs": 1457.5672909999994,
+ "firstDeltaMs": 1198.645958000001,
"calls": []
}
]
@@ -687,6 +570,17 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "library",
+ "editor"
+ ],
+ "declaredTools": 85,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -694,74 +588,66 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 1790.8949579999899,
- "execMs": 0.0064999999885912985
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1790.9784579999978,
- "execMs": 0.001500000013038516
+ "atMs": 1847.401791999997,
+ "execMs": 0.007083999997121282,
+ "args": {
+ "text": "So today we're",
+ "id": "cue1"
+ }
},
{
"name": "update_cue",
"round": 0,
- "atMs": 1791.0212499999907,
- "execMs": 0.0041249999922001734
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 1791.0573749999894,
- "execMs": 0.004417000003741123
+ "atMs": 1847.5505419999972,
+ "execMs": 0.00391700000182027,
+ "args": {
+ "id": "cue3",
+ "text": "and it's great."
+ }
},
{
- "name": "update_cue",
+ "name": "align_to_audio",
"round": 0,
- "atMs": 1791.094207999995,
- "execMs": 0.0016669999895384535
+ "atMs": 1848.802791999995,
+ "execMs": 0.005500000006577466,
+ "args": {
+ "target": "captions",
+ "track": 0
+ }
}
],
"timings": {
- "gateMs": 1273.5387920000067,
+ "gateMs": 291.03254200000083,
"gateSkipped": false,
- "ttftMs": 2842.837249999997,
- "modelMs": 3117.975207999989,
- "toolMs": 0.018208999987109564,
- "totalMs": 3120.649790999989,
- "judgeMs": 1107.458209000004,
+ "ttftMs": 2914.2705419999984,
+ "modelMs": 2794.726251,
+ "toolMs": 0.016501000005519018,
+ "totalMs": 3090.3525420000005,
+ "judgeMs": 351.53241700000217,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1789.2649579999998,
+ "roundMs": 1554.4014590000006,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.0064999999885912985
+ "execMs": 0.007083999997121282
},
{
"name": "update_cue",
- "execMs": 0.001500000013038516
- },
- {
- "name": "update_cue",
- "execMs": 0.0041249999922001734
- },
- {
- "name": "delete_cue",
- "execMs": 0.004417000003741123
+ "execMs": 0.00391700000182027
},
{
- "name": "update_cue",
- "execMs": 0.0016669999895384535
+ "name": "align_to_audio",
+ "execMs": 0.005500000006577466
}
]
},
{
- "roundMs": 1328.7102499999892,
- "firstDeltaMs": 1051.045207999996,
+ "roundMs": 1240.3247919999994,
+ "firstDeltaMs": 1064.6275000000023,
"calls": []
}
]
@@ -776,34 +662,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1154,
- "p95": 1604,
- "mean": 1250,
- "n": 3
+ "p50": 239,
+ "p95": 253,
+ "mean": 246,
+ "n": 2
},
"ttftMs": {
- "p50": 1450,
- "p95": 1850,
- "mean": 1523,
- "n": 3
+ "p50": 1176,
+ "p95": 1269,
+ "mean": 1222,
+ "n": 2
},
"totalMs": {
- "p50": 1793,
- "p95": 2443,
- "mean": 1946,
- "n": 3
+ "p50": 1276,
+ "p95": 1444,
+ "mean": 1360,
+ "n": 2
},
"modelMs": {
- "p50": 1791,
- "p95": 2441,
- "mean": 1944,
- "n": 3
+ "p50": 1035,
+ "p95": 1187,
+ "mean": 1111,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 1,
@@ -814,50 +700,30 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "timeline"
+ ],
+ "declaredTools": 44,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1603.8369999999995,
- "gateSkipped": false,
- "ttftMs": 1849.7965000000113,
- "modelMs": 2440.9797499999986,
- "toolMs": 0,
- "totalMs": 2442.6970830000064,
- "judgeMs": 1072.788041000007,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2440.9797499999986,
- "firstDeltaMs": 1848.3556669999962,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 992.4839169999905,
+ "gateMs": 253.24716699999408,
"gateSkipped": false,
- "ttftMs": 1449.6951249999984,
- "modelMs": 1790.7664170000062,
+ "ttftMs": 1269.1092500000013,
+ "modelMs": 1187.2047910000038,
"toolMs": 0,
- "totalMs": 1792.6339160000061,
- "judgeMs": 1755.7967500000086,
+ "totalMs": 1444.1679999999978,
+ "judgeMs": 244.8036659999998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1790.7664170000062,
- "firstDeltaMs": 1448.099125000008,
+ "roundMs": 1187.2047910000038,
+ "firstDeltaMs": 1012.2492910000001,
"calls": []
}
]
@@ -866,24 +732,30 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "timeline"
+ ],
+ "declaredTools": 44,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1154.4174159999966,
+ "gateMs": 238.75079099999857,
"gateSkipped": false,
- "ttftMs": 1268.7805829999998,
- "modelMs": 1601.0302919999958,
+ "ttftMs": 1175.5901660000018,
+ "modelMs": 1035.3704170000055,
"toolMs": 0,
- "totalMs": 1603.416000000012,
- "judgeMs": 1054.1132499999949,
+ "totalMs": 1276.0852499999964,
+ "judgeMs": 271.54412500000035,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1601.0302919999958,
- "firstDeltaMs": 1266.5898750000051,
+ "roundMs": 1035.3704170000055,
+ "firstDeltaMs": 935.2327500000029,
"calls": []
}
]
@@ -898,137 +770,114 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
+ "p50": 250,
+ "p95": 274,
+ "mean": 262,
+ "n": 2
},
"ttftMs": {
- "p50": 18315,
- "p95": 24319,
- "mean": 19750,
- "n": 3
+ "p50": 5792,
+ "p95": 7322,
+ "mean": 6557,
+ "n": 2
},
"totalMs": {
- "p50": 18804,
- "p95": 24986,
- "mean": 20186,
- "n": 3
+ "p50": 6118,
+ "p95": 7661,
+ "mean": 6889,
+ "n": 2
},
"modelMs": {
- "p50": 18787,
- "p95": 24975,
- "mean": 20171,
- "n": 3
+ "p50": 5840,
+ "p95": 7407,
+ "mean": 6624,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 5,
- "max": 5
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
"intent": "complex",
+ "skill": "timeline-editing",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 112,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": "incorrectly identified input media as clips and added them after an unspecified beach clip.",
+ "judgeNote": null,
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 7057.296000000002,
- "execMs": 0.008832999999867752
+ "atMs": 3654.9874589999963,
+ "execMs": 0.09112500000628643,
+ "args": {
+ "asset_id": "a-i1"
+ }
},
{
"name": "add_clip",
"round": 0,
- "atMs": 7057.943167000005,
- "execMs": 0.002582999994046986
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 10725.348792000004,
- "execMs": 0.16854099999181926
- },
- {
- "name": "add_clip",
- "round": 2,
- "atMs": 13341.096791999997,
- "execMs": 0.002166999998735264
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 15736.111959000002,
- "execMs": 0.0030409999890252948
+ "atMs": 3655.246874999997,
+ "execMs": 0.003874999994877726,
+ "args": {
+ "asset_id": "a-i2"
+ }
}
],
"timings": {
- "gateMs": 0.23770800000056624,
+ "gateMs": 274.00900000000547,
"gateSkipped": true,
- "ttftMs": 18315.102541999993,
- "modelMs": 18787.237874000013,
- "toolMs": 0.18516499997349456,
- "totalMs": 18804.043959000002,
- "judgeMs": 1111.6966660000035,
+ "ttftMs": 5791.829333999995,
+ "modelMs": 5840.383915999992,
+ "toolMs": 0.09500000000116415,
+ "totalMs": 6118.179999999993,
+ "judgeMs": 772.9698329999956,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7050.482333000007,
+ "roundMs": 3378.5229580000014,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.008832999999867752
+ "execMs": 0.09112500000628643
},
{
"name": "add_clip",
- "execMs": 0.002582999994046986
- }
- ]
- },
- {
- "roundMs": 3662.932792000007,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.16854099999181926
+ "execMs": 0.003874999994877726
}
]
},
{
- "roundMs": 2612.8652079999883,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.002166999998735264
- }
- ]
- },
- {
- "roundMs": 2394.347708000001,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0030409999890252948
- }
- ]
- },
- {
- "roundMs": 3066.6098330000095,
- "firstDeltaMs": 2578.2003750000003,
+ "roundMs": 2461.86095799999,
+ "firstDeltaMs": 2135.6040829999984,
"calls": []
}
]
@@ -1037,107 +886,25 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "add_clip",
- "round": 0,
- "atMs": 3450.7752089999994,
- "execMs": 0.00304100000357721
- },
- {
- "name": "add_clip",
- "round": 0,
- "atMs": 3450.9862500000017,
- "execMs": 0.0012910000077681616
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 5830.954666999998,
- "execMs": 0.04470799998671282
- },
- {
- "name": "add_clip",
- "round": 2,
- "atMs": 10253.465542000005,
- "execMs": 0.002749999999650754
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 21312.949334000004,
- "execMs": 0.0054579999996349216
- }
+ "skill": "timeline-editing",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
],
- "timings": {
- "gateMs": 0.2736250000016298,
- "gateSkipped": true,
- "ttftMs": 24318.856875000012,
- "modelMs": 24975.346416999993,
- "toolMs": 0.05724799999734387,
- "totalMs": 24986.103292,
- "judgeMs": 1099.9086249999964,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3447.73666699999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.00304100000357721
- },
- {
- "name": "add_clip",
- "execMs": 0.0012910000077681616
- }
- ]
- },
- {
- "roundMs": 2378.5878330000123,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.04470799998671282
- }
- ]
- },
- {
- "roundMs": 4420.350292000003,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.002749999999650754
- }
- ]
- },
- {
- "roundMs": 11057.87999999999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0054579999996349216
- }
- ]
- },
- {
- "roundMs": 3670.791624999998,
- "firstDeltaMs": 3004.3520419999986,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
+ "declaredTools": 112,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
@@ -1145,92 +912,52 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 3768.695583000008,
- "execMs": 0.01120900000387337
+ "atMs": 5664.788457999995,
+ "execMs": 0.009791000004042871,
+ "args": {
+ "asset_id": "a-i1",
+ "index": 1
+ }
},
{
"name": "add_clip",
"round": 0,
- "atMs": 3769.12337500001,
- "execMs": 0.0036659999896073714
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 6617.595541000017,
- "execMs": 0.027916999999433756
- },
- {
- "name": "add_clip",
- "round": 2,
- "atMs": 9614.423500000004,
- "execMs": 0.010208000021521002
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 12585.030666000006,
- "execMs": 0.011582999984966591
+ "atMs": 5665.031625000003,
+ "execMs": 0.0037089999968884513,
+ "args": {
+ "index": 2,
+ "asset_id": "a-i2"
+ }
}
],
"timings": {
- "gateMs": 0.38445800000044983,
+ "gateMs": 249.76866600000358,
"gateSkipped": true,
- "ttftMs": 16614.922166000004,
- "modelMs": 16749.355291,
- "toolMs": 0.06458299999940209,
- "totalMs": 16768.363333000016,
- "judgeMs": 1097.0650000000023,
+ "ttftMs": 7321.556125000003,
+ "modelMs": 7407.458249999996,
+ "toolMs": 0.013500000000931323,
+ "totalMs": 7660.595541000002,
+ "judgeMs": 519.989167000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3764.0111250000045,
+ "roundMs": 5413.399124999996,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.01120900000387337
+ "execMs": 0.009791000004042871
},
{
"name": "add_clip",
- "execMs": 0.0036659999896073714
- }
- ]
- },
- {
- "roundMs": 2845.323417000007,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.027916999999433756
- }
- ]
- },
- {
- "roundMs": 2992.8737919999985,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.010208000021521002
- }
- ]
- },
- {
- "roundMs": 2967.459415999998,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.011582999984966591
+ "execMs": 0.0037089999968884513
}
]
},
{
- "roundMs": 4179.687540999992,
- "firstDeltaMs": 4026.7184579999885,
+ "roundMs": 1994.0591249999998,
+ "firstDeltaMs": 1655.4207499999902,
"calls": []
}
]
@@ -1245,34 +972,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
+ "p50": 230,
+ "p95": 254,
+ "mean": 242,
+ "n": 2
},
"ttftMs": {
- "p50": 1705,
- "p95": 5381,
- "mean": 2921,
- "n": 3
+ "p50": 2151,
+ "p95": 2347,
+ "mean": 2249,
+ "n": 2
},
"totalMs": {
- "p50": 3438,
- "p95": 8246,
- "mean": 4911,
- "n": 3
+ "p50": 3280,
+ "p95": 3711,
+ "mean": 3495,
+ "n": 2
},
"modelMs": {
- "p50": 3435,
- "p95": 8241,
- "mean": 4907,
- "n": 3
+ "p50": 3046,
+ "p95": 3453,
+ "mean": 3250,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 1,
@@ -1283,50 +1010,44 @@
{
"pass": true,
"intent": "complex",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 112,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": "includes filler and unnecessary headings.",
- "trace": [],
- "timings": {
- "gateMs": 0.38387499999953434,
- "gateSkipped": true,
- "ttftMs": 5380.736750000011,
- "modelMs": 8240.964915999997,
- "toolMs": 0,
- "totalMs": 8245.811125000007,
- "judgeMs": 2102.2117909999797,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 8240.964915999997,
- "firstDeltaMs": 5376.637916000007,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": "includes self-praise (\"dynamic,\" \"gentle,\" \"upbeat\") and violates conciseness with bulleted fluff.",
+ "judgeNote": "concise factual 0.21; upsell 0.84; long 1.9",
"trace": [],
"timings": {
- "gateMs": 0.0857079999987036,
+ "gateMs": 230.47950000000128,
"gateSkipped": true,
- "ttftMs": 1705.4309170000197,
- "modelMs": 3045.89291699999,
+ "ttftMs": 2150.78966699999,
+ "modelMs": 3045.852583,
"toolMs": 0,
- "totalMs": 3047.524208000017,
- "judgeMs": 1892.83712500002,
+ "totalMs": 3279.6020419999986,
+ "judgeMs": 226.97287500000675,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3045.89291699999,
- "firstDeltaMs": 1704.2735000000102,
+ "roundMs": 3045.852583,
+ "firstDeltaMs": 1917.5350000000035,
"calls": []
}
]
@@ -1335,24 +1056,43 @@
{
"pass": true,
"intent": "complex",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "elements",
+ "effects",
+ "transitions",
+ "subtitles",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 110,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": "contains filler and self-referential introductory phrasing.",
+ "judgeNote": "concise factual 0.20; upsell 0.55; long 2.0",
"trace": [],
"timings": {
- "gateMs": 0.3089169999875594,
+ "gateMs": 254.1837910000031,
"gateSkipped": true,
- "ttftMs": 1676.4723330000124,
- "modelMs": 3434.980707999988,
+ "ttftMs": 2346.775207999992,
+ "modelMs": 3453.3652920000022,
"toolMs": 0,
- "totalMs": 3438.4217500000086,
- "judgeMs": 1920.5850420000206,
+ "totalMs": 3711.0452919999952,
+ "judgeMs": 332.4827920000098,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3434.980707999988,
- "firstDeltaMs": 1673.3244579999882,
+ "roundMs": 3453.3652920000022,
+ "firstDeltaMs": 2090.231500000009,
"calls": []
}
]
@@ -1367,37 +1107,37 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1119,
- "p95": 1814,
- "mean": 1275,
- "n": 3
+ "p50": 259,
+ "p95": 263,
+ "mean": 261,
+ "n": 2
},
"ttftMs": {
- "p50": 5212,
- "p95": 8193,
- "mean": 6005,
- "n": 3
+ "p50": 1434,
+ "p95": 8678,
+ "mean": 5056,
+ "n": 2
},
"totalMs": {
- "p50": 5381,
- "p95": 8480,
- "mean": 6297,
- "n": 3
+ "p50": 1696,
+ "p95": 8806,
+ "mean": 5251,
+ "n": 2
},
"modelMs": {
- "p50": 5377,
- "p95": 8476,
- "mean": 6293,
- "n": 3
+ "p50": 1432,
+ "p95": 8539,
+ "mean": 4985,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2,
+ "mean": 1.5,
"max": 2
}
},
@@ -1405,60 +1145,38 @@
{
"pass": true,
"intent": "chat",
+ "skill": null,
+ "areas": [
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 0,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1813.7096660000097,
- "gateSkipped": false,
- "ttftMs": 8192.862374999997,
- "modelMs": 8475.762707999995,
- "toolMs": 0,
- "totalMs": 8480.434959000006,
- "judgeMs": 1790.3574999999837,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1811.7376249999797,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 6664.025083000015,
- "firstDeltaMs": 6376.769916999998,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "chat",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 893.1134169999859,
+ "gateMs": 259.3814999999886,
"gateSkipped": false,
- "ttftMs": 4609.015749999991,
- "modelMs": 5026.633417000005,
+ "ttftMs": 8677.94183299999,
+ "modelMs": 8538.731751,
"toolMs": 0,
- "totalMs": 5030.841416999989,
- "judgeMs": 1081.1937090000138,
+ "totalMs": 8805.912207999994,
+ "judgeMs": 548.3632499999949,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 890.0865829999966,
+ "roundMs": 3911.315333999999,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 4136.546834000008,
- "firstDeltaMs": 3714.9482920000155,
+ "roundMs": 4627.416417,
+ "firstDeltaMs": 4499.9619590000075,
"calls": []
}
]
@@ -1466,30 +1184,34 @@
},
{
"pass": true,
- "intent": "chat",
- "roundModel": "gemini-3.8-flash",
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 38,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1118.7804590000014,
+ "gateMs": 262.62966599999345,
"gateSkipped": false,
- "ttftMs": 5212.1137089999975,
- "modelMs": 5376.935291000002,
+ "ttftMs": 1433.572291000004,
+ "modelMs": 1431.7364170000073,
"toolMs": 0,
- "totalMs": 5380.887999999977,
- "judgeMs": 1147.4468749999942,
+ "totalMs": 1696.436333000005,
+ "judgeMs": 273.283208000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1116.468332999997,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 4260.4669580000045,
- "firstDeltaMs": 4092.0635000000184,
+ "roundMs": 1431.7364170000073,
+ "firstDeltaMs": 1169.2181249999994,
"calls": []
}
]
@@ -1504,34 +1226,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1036,
- "p95": 1191,
- "mean": 1086,
- "n": 3
+ "p50": 263,
+ "p95": 266,
+ "mean": 265,
+ "n": 2
},
"ttftMs": {
- "p50": 2849,
- "p95": 2983,
- "mean": 2777,
- "n": 3
+ "p50": 3166,
+ "p95": 3209,
+ "mean": 3188,
+ "n": 2
},
"totalMs": {
- "p50": 3104,
- "p95": 3192,
- "mean": 3017,
- "n": 3
+ "p50": 3433,
+ "p95": 3436,
+ "mean": 3434,
+ "n": 2
},
"modelMs": {
- "p50": 3098,
- "p95": 3181,
- "mean": 3010,
- "n": 3
+ "p50": 3158,
+ "p95": 3166,
+ "mean": 3162,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -1542,6 +1264,26 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 113,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -1549,34 +1291,37 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 1824.6631669999915,
- "execMs": 0.00741700001526624
+ "atMs": 1752.7311659999978,
+ "execMs": 0.01779099999112077,
+ "args": {
+ "prompt": "A joyful dog running and playing in a sunny park, cinematic lighting."
+ }
}
],
"timings": {
- "gateMs": 1190.7280409999948,
+ "gateMs": 266.34341600000334,
"gateSkipped": false,
- "ttftMs": 2982.562458999979,
- "modelMs": 3181.330875000014,
- "toolMs": 0.00741700001526624,
- "totalMs": 3191.993374999991,
- "judgeMs": 1907.2397919999785,
+ "ttftMs": 3209.0444579999894,
+ "modelMs": 3158.371082999991,
+ "toolMs": 0.01779099999112077,
+ "totalMs": 3432.8083329999936,
+ "judgeMs": 260.37766700000793,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1815.6071250000095,
+ "roundMs": 1480.4649579999968,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.00741700001526624
+ "execMs": 0.01779099999112077
}
]
},
{
- "roundMs": 1365.7237500000047,
- "firstDeltaMs": 1156.5904169999994,
+ "roundMs": 1677.906124999994,
+ "firstDeltaMs": 1454.6185830000031,
"calls": []
}
]
@@ -1585,6 +1330,24 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 107,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -1592,42 +1355,115 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 1441.5456669999985,
- "execMs": 0.011458000022685155
+ "atMs": 1754.778250000003,
+ "execMs": 0.009125000011408702,
+ "args": {
+ "prompt": "A cheerful dog running happily through a lush green park, sunny day, high definition, slow motion."
+ }
}
],
"timings": {
- "gateMs": 1031.8328750000219,
+ "gateMs": 263.310083999997,
"gateSkipped": false,
- "ttftMs": 2498.831708999991,
- "modelMs": 2749.6239159999823,
- "toolMs": 0.011458000022685155,
- "totalMs": 2755.8695000000007,
- "judgeMs": 1121.7033330000122,
+ "ttftMs": 3166.4713750000083,
+ "modelMs": 3166.453999999998,
+ "toolMs": 0.009125000011408702,
+ "totalMs": 3435.7610419999983,
+ "judgeMs": 277.9326669999864,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1438.0455409999995,
+ "roundMs": 1488.7930000000051,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.011458000022685155
+ "execMs": 0.009125000011408702
}
]
},
{
- "roundMs": 1311.5783749999828,
- "firstDeltaMs": 1054.8739999999816,
+ "roundMs": 1677.6609999999928,
+ "firstDeltaMs": 1408.8366249999963,
"calls": []
}
]
}
+ }
+ ]
+ },
+ {
+ "name": "video-ask-with-size-and-length",
+ "bucket": "single-tool",
+ "passRate": 0.5,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 1124,
+ "p95": 1124,
+ "mean": 1124,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 15281,
+ "p95": 15281,
+ "mean": 15281,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 16012,
+ "p95": 16012,
+ "mean": 16012,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 15599,
+ "p95": 15599,
+ "mean": 15599,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
+ {
+ "pass": false,
+ "intent": "work",
+ "notes": [],
+ "trace": [],
+ "timings": null,
+ "error": "This operation was aborted"
},
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "animation",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 97,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -1635,34 +1471,39 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 1493.3179999999993,
- "execMs": 0.010334000020520762
+ "atMs": 10523.85370800001,
+ "execMs": 0.0022080000053392723,
+ "args": {
+ "prompt": "a paper boat drifting across a puddle, cinematic lighting, 1080p",
+ "duration_seconds": 6,
+ "resolution": "1080p"
+ }
}
],
"timings": {
- "gateMs": 1035.556082999974,
+ "gateMs": 1124.1443750000035,
"gateSkipped": false,
- "ttftMs": 2848.558250000002,
- "modelMs": 3097.7195420000353,
- "toolMs": 0.010334000020520762,
- "totalMs": 3104.093709000008,
- "judgeMs": 1146.1801250000135,
+ "ttftMs": 15281.440125000008,
+ "modelMs": 15598.731249999997,
+ "toolMs": 0.0022080000053392723,
+ "totalMs": 16012.236250000002,
+ "judgeMs": 711.551833999998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1489.509417000023,
+ "roundMs": 10118.337124999991,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.010334000020520762
+ "execMs": 0.0022080000053392723
}
]
},
{
- "roundMs": 1608.2101250000123,
- "firstDeltaMs": 1352.9489590000012,
+ "roundMs": 5480.394125000006,
+ "firstDeltaMs": 4750.2749579999945,
"calls": []
}
]
@@ -1671,85 +1512,108 @@
]
},
{
- "name": "video-ask-with-size-and-length",
+ "name": "iterate-keeps-existing-clip",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1585,
- "p95": 1947,
- "mean": 1589,
- "n": 3
+ "p50": 303,
+ "p95": 322,
+ "mean": 313,
+ "n": 2
},
"ttftMs": {
- "p50": 3338,
- "p95": 3654,
- "mean": 3312,
- "n": 3
+ "p50": 6209,
+ "p95": 11595,
+ "mean": 8902,
+ "n": 2
},
"totalMs": {
- "p50": 3818,
- "p95": 3920,
- "mean": 3656,
- "n": 3
+ "p50": 6431,
+ "p95": 11925,
+ "mean": 9178,
+ "n": 2
},
"modelMs": {
- "p50": 3572,
- "p95": 3809,
- "mean": 3535,
- "n": 3
+ "p50": 6118,
+ "p95": 11573,
+ "mean": 8846,
+ "n": 2
},
"toolMs": {
"p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
+ "p95": 3,
+ "mean": 1,
+ "n": 2
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 3
}
},
"runs": [
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "skill": "color-grading",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "effects",
+ "video_gen",
+ "scene",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 88,
+ "routingMisses": [],
+ "roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": "did not state the approval gate for rendering credits",
+ "judgeNote": null,
"trace": [
{
"name": "generate_video",
- "round": 0,
- "atMs": 2034.9309169999906,
- "execMs": 0.3302910000202246
+ "round": 1,
+ "atMs": 7802.556916000001,
+ "execMs": 2.6554999999934807,
+ "args": {
+ "reference_asset_id": "a-v1",
+ "prompt": "Scenic beach at golden-hour sunset, warm glowing sunlight reflecting across ocean waves and wet sand, gentle cinematic camera motion, peaceful evening light"
+ }
}
],
"timings": {
- "gateMs": 1585.101874999993,
+ "gateMs": 321.89491600000474,
"gateSkipped": false,
- "ttftMs": 3337.9151249999995,
- "modelMs": 3809.324958000012,
- "toolMs": 0.3302910000202246,
- "totalMs": 3817.597167,
- "judgeMs": 1066.2834589999984,
+ "ttftMs": 11595.309333000012,
+ "modelMs": 11573.099707999994,
+ "toolMs": 2.6554999999934807,
+ "totalMs": 11924.54104099999,
+ "judgeMs": 540.6497089999903,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2030.3348750000005,
+ "roundMs": 1767.5983750000014,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 5688.663333000004,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.3302910000202246
+ "execMs": 2.6554999999934807
}
]
},
{
- "roundMs": 1778.9900830000115,
- "firstDeltaMs": 1299.5712080000085,
+ "roundMs": 4116.837999999989,
+ "firstDeltaMs": 3787.974208999978,
"calls": []
}
]
@@ -1757,85 +1621,68 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "skill": "color-grading",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "effects",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 97,
+ "routingMisses": [],
+ "roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "generate_video",
- "round": 0,
- "atMs": 1947.681917000009,
- "execMs": 0.008916999999200925
+ "round": 1,
+ "atMs": 4222.986291999987,
+ "execMs": 0.014374999998835847,
+ "args": {
+ "reference_asset_id": "a-v1",
+ "prompt": "A scenic beach bathed in warm golden-hour sunset light, gentle ocean waves lapping at the shore, soft warm orange and pink horizon, cinematic camera motion",
+ "aspect": "9:16"
+ }
}
],
"timings": {
- "gateMs": 1947.460207999975,
+ "gateMs": 303.25975000002654,
"gateSkipped": false,
- "ttftMs": 3653.5570000000007,
- "modelMs": 3571.8519579999847,
- "toolMs": 0.008916999999200925,
- "totalMs": 3920.2177500000107,
- "judgeMs": 1189.4654169999994,
+ "ttftMs": 6209.459000000003,
+ "modelMs": 6117.910833000002,
+ "toolMs": 0.014374999998835847,
+ "totalMs": 6431.130208999995,
+ "judgeMs": 497.73662499999045,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1601.7580829999933,
+ "roundMs": 1569.9868330000027,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_video",
- "execMs": 0.008916999999200925
- }
- ]
- },
- {
- "roundMs": 1970.0938749999914,
- "firstDeltaMs": 1703.7032089999993,
"calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": "failed to state the approval gate for rendering credits.",
- "trace": [
- {
- "name": "generate_video",
- "round": 0,
- "atMs": 1749.7870839999814,
- "execMs": 0.01208399998722598
- }
- ],
- "timings": {
- "gateMs": 1235.3598750000237,
- "gateSkipped": false,
- "ttftMs": 2944.857833999995,
- "modelMs": 3224.909375999996,
- "toolMs": 0.01208399998722598,
- "totalMs": 3231.5591669999994,
- "judgeMs": 1657.3542080000043,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ },
{
- "roundMs": 1746.4149169999873,
+ "roundMs": 2341.4284159999806,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.01208399998722598
+ "execMs": 0.014374999998835847
}
]
},
{
- "roundMs": 1478.4944590000086,
- "firstDeltaMs": 1191.947709,
+ "roundMs": 2206.4955840000184,
+ "firstDeltaMs": 1984.9706250000163,
"calls": []
}
]
@@ -1844,138 +1691,127 @@
]
},
{
- "name": "iterate-keeps-existing-clip",
+ "name": "mute-all-does-not-redo-prior-turn",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1062,
- "p95": 1098,
- "mean": 1047,
- "n": 3
+ "p50": 217,
+ "p95": 227,
+ "mean": 222,
+ "n": 2
},
"ttftMs": {
- "p50": 6018,
- "p95": 11300,
- "mean": 7420,
- "n": 3
+ "p50": 2833,
+ "p95": 2954,
+ "mean": 2894,
+ "n": 2
},
"totalMs": {
- "p50": 6366,
- "p95": 11692,
- "mean": 8069,
- "n": 3
+ "p50": 3014,
+ "p95": 3059,
+ "mean": 3037,
+ "n": 2
},
"modelMs": {
- "p50": 6359,
- "p95": 11682,
- "mean": 8061,
- "n": 3
+ "p50": 2793,
+ "p95": 2823,
+ "mean": 2808,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 3,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "audio",
+ "editor"
+ ],
+ "declaredTools": 69,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_video",
- "round": 1,
- "atMs": 8575.124791999988,
- "execMs": 0.2596250000060536
- }
- ],
- "timings": {
- "gateMs": 1097.5425000000105,
- "gateSkipped": false,
- "ttftMs": 11299.935582999984,
- "modelMs": 11682.138166999997,
- "toolMs": 0.2596250000060536,
- "totalMs": 11691.560874999996,
- "judgeMs": 976.9583330000169,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1095.608333000011,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 7473.187416999979,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_video",
- "execMs": 0.2596250000060536
- }
- ]
- },
- {
- "roundMs": 3113.342417000007,
- "firstDeltaMs": 2722.0463330000057,
- "calls": []
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 1902.9193749999977,
+ "execMs": 0.06995800000731833,
+ "args": {
+ "clipId": "c1",
+ "muted": true
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
+ },
{
- "name": "generate_video",
- "round": 1,
- "atMs": 3151.3382920000004,
- "execMs": 0.012541000003693625
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 1903.2229999999981,
+ "execMs": 0.0027500000142026693,
+ "args": {
+ "muted": true,
+ "clipId": "c2"
+ }
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 1903.465750000003,
+ "execMs": 0.003874999994877726,
+ "args": {
+ "muted": true,
+ "clipId": "c3"
+ }
}
],
"timings": {
- "gateMs": 983.2099579999922,
+ "gateMs": 226.62166699999943,
"gateSkipped": false,
- "ttftMs": 4943.595667000016,
- "modelMs": 6143.798416999984,
- "toolMs": 0.012541000003693625,
- "totalMs": 6150.661042000022,
- "judgeMs": 1022.3878329999861,
+ "ttftMs": 2953.701915999991,
+ "modelMs": 2823.3891669999866,
+ "toolMs": 0.07658300001639873,
+ "totalMs": 3058.836290999985,
+ "judgeMs": 206.96162500002538,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 981.5204579999845,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2163.9426669999957,
+ "roundMs": 1669.7185839999875,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 0.012541000003693625
+ "name": "set_clip_muted",
+ "execMs": 0.06995800000731833
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0027500000142026693
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.003874999994877726
}
]
},
{
- "roundMs": 2998.3352920000034,
- "firstDeltaMs": 1791.5535420000087,
+ "roundMs": 1153.6705829999992,
+ "firstDeltaMs": 1048.986957999994,
"calls": []
}
]
@@ -1983,47 +1819,84 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "audio",
+ "editor"
+ ],
+ "declaredTools": 69,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_video",
- "round": 1,
- "atMs": 3316.487333999976,
- "execMs": 0.004458000010345131
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 1580.9433750000026,
+ "execMs": 0.01179200000478886,
+ "args": {
+ "clipId": "c1",
+ "muted": true
+ }
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 1581.1917919999978,
+ "execMs": 0.0027920000138692558,
+ "args": {
+ "clipId": "c2",
+ "muted": true
+ }
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 1581.402833,
+ "execMs": 0.002207999990787357,
+ "args": {
+ "muted": true,
+ "clipId": "c3"
+ }
}
],
"timings": {
- "gateMs": 1061.5086250000168,
+ "gateMs": 217.39104200000293,
"gateSkipped": false,
- "ttftMs": 6017.925041999988,
- "modelMs": 6358.501540999976,
- "toolMs": 0.004458000010345131,
- "totalMs": 6366.125624999986,
- "judgeMs": 1388.3779170000053,
+ "ttftMs": 2833.381625000009,
+ "modelMs": 2792.722999999998,
+ "toolMs": 0.016792000009445474,
+ "totalMs": 3014.457208000007,
+ "judgeMs": 218.88420800000313,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1058.6841249999998,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2251.597665999987,
+ "roundMs": 1361.5271249999932,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 0.004458000010345131
+ "name": "set_clip_muted",
+ "execMs": 0.01179200000478886
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0027920000138692558
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.002207999990787357
}
]
},
{
- "roundMs": 3048.2197499999893,
- "firstDeltaMs": 2700.2572079999954,
+ "roundMs": 1431.195875000005,
+ "firstDeltaMs": 1250.511415999994,
"calls": []
}
]
@@ -2032,105 +1905,111 @@
]
},
{
- "name": "mute-all-does-not-redo-prior-turn",
+ "name": "imported-photo-edit-references-photo",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1137,
- "p95": 1854,
- "mean": 1308,
- "n": 3
+ "p50": 220,
+ "p95": 266,
+ "mean": 243,
+ "n": 2
},
"ttftMs": {
- "p50": 3256,
- "p95": 5978,
- "mean": 4086,
- "n": 3
+ "p50": 3314,
+ "p95": 3858,
+ "mean": 3586,
+ "n": 2
},
"totalMs": {
- "p50": 3395,
- "p95": 6082,
- "mean": 4211,
- "n": 3
+ "p50": 3667,
+ "p95": 4203,
+ "mean": 3935,
+ "n": 2
},
"modelMs": {
- "p50": 3387,
- "p95": 6070,
- "mean": 4202,
- "n": 3
+ "p50": 3438,
+ "p95": 3931,
+ "mean": 3685,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2.3,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 103,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 1956.8394580000022,
- "execMs": 0.1183749999909196
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 1957.2883750000037,
- "execMs": 0.0023339999897871166
- },
- {
- "name": "set_clip_muted",
+ "name": "generate_image",
"round": 0,
- "atMs": 1957.4432080000115,
- "execMs": 0.0017080000252462924
+ "atMs": 1988.851915999985,
+ "execMs": 0.2047080000047572,
+ "args": {
+ "reference_asset_ids": [
+ "a-tw1"
+ ],
+ "prompt": "a photo of the person in the reference image wearing a santa hat, maintaining the style, lighting, and composition of the original photo"
+ }
}
],
"timings": {
- "gateMs": 1854.478207999986,
+ "gateMs": 219.92966699998942,
"gateSkipped": false,
- "ttftMs": 3255.5517080000136,
- "modelMs": 3386.586915999971,
- "toolMs": 0.12241700000595301,
- "totalMs": 3394.6058330000087,
- "judgeMs": 1030.1028749999823,
+ "ttftMs": 3313.9157499999856,
+ "modelMs": 3438.0066660000302,
+ "toolMs": 0.2047080000047572,
+ "totalMs": 3667.044790999993,
+ "judgeMs": 244.47749999997905,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1952.3399999999965,
+ "roundMs": 1761.965166000009,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.1183749999909196
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0023339999897871166
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0017080000252462924
+ "name": "generate_image",
+ "execMs": 0.2047080000047572
}
]
},
{
- "roundMs": 1434.2469159999746,
- "firstDeltaMs": 1295.4658749999944,
+ "roundMs": 1676.0415000000212,
+ "firstDeltaMs": 1323.256625000009,
"calls": []
}
]
@@ -2139,109 +2018,68 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 103,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 1585.1683750000084,
- "execMs": 0.007333000015933067
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 1585.6196669999918,
- "execMs": 0.0031669999880250543
- },
- {
- "name": "set_clip_muted",
+ "name": "generate_image",
"round": 0,
- "atMs": 1585.8193329999922,
- "execMs": 0.002082999999402091
- }
- ],
- "timings": {
- "gateMs": 1137.1493750000081,
- "gateSkipped": false,
- "ttftMs": 3025.0112500000105,
- "modelMs": 3148.0454169999866,
- "toolMs": 0.012583000003360212,
- "totalMs": 3155.611124999996,
- "judgeMs": 1124.7066669999913,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1581.2874999999767,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_clip_muted",
- "execMs": 0.007333000015933067
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0031669999880250543
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.002082999999402091
- }
- ]
- },
- {
- "roundMs": 1566.75791700001,
- "firstDeltaMs": 1436.392084000021,
- "calls": []
+ "atMs": 2701.744250000018,
+ "execMs": 0.012959000014234334,
+ "args": {
+ "reference_asset_ids": [
+ "a-tw1"
+ ],
+ "prompt": "A high-quality recreation of the girl in the \"first snow of the year\" photo, now wearing a festive red Santa hat, maintaining the original composition and aesthetic.",
+ "aspect": "9:16"
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_track_muted",
- "round": 1,
- "atMs": 3302.646624999994,
- "execMs": 0.021041999978478998
}
],
"timings": {
- "gateMs": 933.1382919999887,
+ "gateMs": 266.1987909999734,
"gateSkipped": false,
- "ttftMs": 5977.528416999994,
- "modelMs": 6070.120874999993,
- "toolMs": 0.021041999978478998,
- "totalMs": 6082.119959000003,
- "judgeMs": 1051.1293329999899,
+ "ttftMs": 3858.036208000005,
+ "modelMs": 3931.0774989999773,
+ "toolMs": 0.012959000014234334,
+ "totalMs": 4203.301917000004,
+ "judgeMs": 224.05791699999827,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 930.7594169999938,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2362.8220419999852,
+ "roundMs": 2431.2394159999967,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_track_muted",
- "execMs": 0.021041999978478998
+ "name": "generate_image",
+ "execMs": 0.012959000014234334
}
]
},
{
- "roundMs": 2776.539416000014,
- "firstDeltaMs": 2672.144874999998,
+ "roundMs": 1499.8380829999805,
+ "firstDeltaMs": 1154.8791249999776,
"calls": []
}
]
@@ -2250,90 +2088,104 @@
]
},
{
- "name": "imported-photo-edit-references-photo",
+ "name": "audio-ask-single-generate",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1041,
- "p95": 1876,
- "mean": 1279,
- "n": 3
+ "p50": 224,
+ "p95": 248,
+ "mean": 236,
+ "n": 2
},
"ttftMs": {
- "p50": 6520,
- "p95": 10809,
- "mean": 7537,
- "n": 3
+ "p50": 2787,
+ "p95": 3334,
+ "mean": 3061,
+ "n": 2
},
"totalMs": {
- "p50": 7434,
- "p95": 11327,
- "mean": 8140,
- "n": 3
+ "p50": 3004,
+ "p95": 3585,
+ "mean": 3295,
+ "n": 2
},
"modelMs": {
- "p50": 7410,
- "p95": 11318,
- "mean": 8126,
- "n": 3
+ "p50": 2750,
+ "p95": 3355,
+ "mean": 3052,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 3.3,
- "max": 4
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "audio",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 106,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_image",
- "round": 1,
- "atMs": 3667.9483329999784,
- "execMs": 0.11249999998835847
+ "name": "voiceover_generate",
+ "round": 0,
+ "atMs": 1881.8186250000144,
+ "execMs": 0.007541999977547675,
+ "args": {
+ "script": "Welcome to the dog show."
+ }
}
],
"timings": {
- "gateMs": 920.3077500000072,
+ "gateMs": 223.6105830000015,
"gateSkipped": false,
- "ttftMs": 5280.741917000007,
- "modelMs": 5649.037040999974,
- "toolMs": 0.11249999998835847,
- "totalMs": 5659.403332999995,
- "judgeMs": 1054.5527500000317,
+ "ttftMs": 3334.4220000000205,
+ "modelMs": 3355.0260420000122,
+ "toolMs": 0.007541999977547675,
+ "totalMs": 3584.7433330000204,
+ "judgeMs": 254.3882910000102,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 918.6604159999988,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2741.6396669999813,
+ "roundMs": 1653.8843749999942,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_image",
- "execMs": 0.11249999998835847
+ "name": "voiceover_generate",
+ "execMs": 0.007541999977547675
}
]
},
{
- "roundMs": 1988.736957999994,
- "firstDeltaMs": 1611.2525000000023,
+ "roundMs": 1701.141667000018,
+ "firstDeltaMs": 1451.2494170000136,
"calls": []
}
]
@@ -2341,52 +2193,214 @@
},
{
"pass": true,
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "audio",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 106,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "voiceover_generate",
+ "round": 0,
+ "atMs": 1573.5448749999923,
+ "execMs": 0.00987500001792796,
+ "args": {
+ "script": "Welcome to the dog show."
+ }
+ }
+ ],
+ "timings": {
+ "gateMs": 248.26612499999464,
+ "gateSkipped": false,
+ "ttftMs": 2787.0850840000203,
+ "modelMs": 2749.9022919999843,
+ "toolMs": 0.00987500001792796,
+ "totalMs": 3004.278250000003,
+ "judgeMs": 221.06387500002165,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1322.1817919999885,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "voiceover_generate",
+ "execMs": 0.00987500001792796
+ }
+ ]
+ },
+ {
+ "roundMs": 1427.7204999999958,
+ "firstDeltaMs": 1212.0876250000147,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "music-ask-single-generate",
+ "bucket": "single-tool",
+ "passRate": 0.5,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 268,
+ "p95": 268,
+ "mean": 268,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 9835,
+ "p95": 9835,
+ "mean": 9835,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 10184,
+ "p95": 10184,
+ "mean": 10184,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 9912,
+ "p95": 9912,
+ "mean": 9912,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
+ },
+ "rounds": {
+ "mean": 3,
+ "max": 3
+ }
+ },
+ "runs": [
+ {
+ "pass": false,
"intent": "complex",
+ "skill": "audio-and-subtitles",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "audio",
+ "stock",
+ "library",
+ "side_panel"
+ ],
+ "declaredTools": 94,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
- "notes": [],
+ "notes": [
+ "mutating tools called: add_clip"
+ ],
"judgeNote": null,
"trace": [
{
- "name": "generate_image",
+ "name": "generate_music",
+ "round": 1,
+ "atMs": 5008.586624999996,
+ "execMs": 0.17216600000392646,
+ "args": {
+ "instrumental": true,
+ "length": "clip",
+ "prompt": "Chill, relaxed lofi ambient acoustic background music, warm gentle guitar and soft synth pads, mellow and calming vibe",
+ "start": 0,
+ "add_to_timeline": true,
+ "volume": 0.35
+ }
+ },
+ {
+ "name": "get_state",
"round": 2,
- "atMs": 4939.702082999982,
- "execMs": 0.05333299998892471
+ "atMs": 19322.093041999993,
+ "execMs": 0.1537909999897238,
+ "args": {}
+ },
+ {
+ "name": "add_clip",
+ "round": 3,
+ "atMs": 28052.555291999975,
+ "execMs": 0.008332999976119027,
+ "args": {
+ "lane": 1,
+ "asset_id": "a-music1",
+ "start": 0
+ }
}
],
"timings": {
- "gateMs": 1875.5652920000139,
+ "gateMs": 246.73874999998952,
"gateSkipped": false,
- "ttftMs": 6519.987499999988,
- "modelMs": 7410.202958000009,
- "toolMs": 0.05333299998892471,
- "totalMs": 7433.988790999982,
- "judgeMs": 1780.1149580000201,
+ "ttftMs": 41920.403416999994,
+ "modelMs": 41960.425292,
+ "toolMs": 0.33428999996976927,
+ "totalMs": 42219.28408299998,
+ "judgeMs": 554.3412499999977,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1860.576207999955,
+ "roundMs": 1830.7174170000071,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 0.02954200003296137,
+ "roundMs": 2924.800416000013,
"firstDeltaMs": null,
- "calls": []
+ "calls": [
+ {
+ "name": "generate_music",
+ "execMs": 0.17216600000392646
+ }
+ ]
},
{
- "roundMs": 3058.6285830000415,
+ "roundMs": 14310.745166999986,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_image",
- "execMs": 0.05333299998892471
+ "name": "get_state",
+ "execMs": 0.1537909999897238
+ }
+ ]
+ },
+ {
+ "roundMs": 8728.772792000003,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "add_clip",
+ "execMs": 0.008332999976119027
}
]
},
{
- "roundMs": 2490.9686249999795,
- "firstDeltaMs": 1577.5048330000136,
+ "roundMs": 14165.38949999999,
+ "firstDeltaMs": 13866.961291999993,
"calls": []
}
]
@@ -2395,46 +2409,68 @@
{
"pass": true,
"intent": "complex",
+ "skill": "audio-and-subtitles",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "audio",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 98,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_image",
+ "name": "generate_music",
"round": 1,
- "atMs": 3785.538499999966,
- "execMs": 0.017667000007349998
+ "atMs": 7789.254709000001,
+ "execMs": 0.013916999974753708,
+ "args": {
+ "length": "clip",
+ "add_to_timeline": true,
+ "volume": 0.4,
+ "start": 0,
+ "prompt": "Chill lofi ambient background music, relaxed acoustic beats, mellow guitar and soft synth, warm summer beach vibe",
+ "instrumental": true
+ }
}
],
"timings": {
- "gateMs": 1040.7640000000247,
+ "gateMs": 268.3392500000191,
"gateSkipped": false,
- "ttftMs": 10808.984667000012,
- "modelMs": 11317.741582999995,
- "toolMs": 0.017667000007349998,
- "totalMs": 11327.278833999997,
- "judgeMs": 1412.5709170000046,
+ "ttftMs": 9835.427792000002,
+ "modelMs": 9911.988834999967,
+ "toolMs": 0.013916999974753708,
+ "totalMs": 10184.317209,
+ "judgeMs": 459.09216700002435,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1038.5029999999679,
+ "roundMs": 4441.42420899999,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 2741.2167910000426,
+ "roundMs": 3076.8457089999865,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_image",
- "execMs": 0.017667000007349998
+ "name": "generate_music",
+ "execMs": 0.013916999974753708
}
]
},
{
- "roundMs": 7538.021791999985,
- "firstDeltaMs": 7020.978583999968,
+ "roundMs": 2393.718916999991,
+ "firstDeltaMs": 2044.9508330000099,
"calls": []
}
]
@@ -2443,40 +2479,40 @@
]
},
{
- "name": "audio-ask-single-generate",
+ "name": "trim-ask-single-tool",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1215,
- "p95": 6877,
- "mean": 3044,
- "n": 3
+ "p50": 218,
+ "p95": 287,
+ "mean": 252,
+ "n": 2
},
"ttftMs": {
- "p50": 5930,
- "p95": 8366,
- "mean": 5831,
- "n": 3
+ "p50": 2617,
+ "p95": 2766,
+ "mean": 2691,
+ "n": 2
},
"totalMs": {
- "p50": 6019,
- "p95": 8543,
- "mean": 5987,
- "n": 3
+ "p50": 2761,
+ "p95": 2916,
+ "mean": 2838,
+ "n": 2
},
"modelMs": {
- "p50": 6009,
- "p95": 8527,
- "mean": 5976,
- "n": 3
+ "p50": 2538,
+ "p95": 2627,
+ "mean": 2582,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -2487,84 +2523,56 @@
{
"pass": true,
"intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": "editing turn failed to report the tool action taken or state what changed in the project.",
- "trace": [
- {
- "name": "voiceover_generate",
- "round": 0,
- "atMs": 2042.0562079999945,
- "execMs": 0.00712500000372529
- }
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "library",
+ "editor"
],
- "timings": {
- "gateMs": 1214.6472920000087,
- "gateSkipped": false,
- "ttftMs": 5929.829542000021,
- "modelMs": 6008.703083000029,
- "toolMs": 0.00712500000372529,
- "totalMs": 6018.970666999987,
- "judgeMs": 1785.3153750000056,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2034.2913750000298,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "voiceover_generate",
- "execMs": 0.00712500000372529
- }
- ]
- },
- {
- "roundMs": 3974.4117079999996,
- "firstDeltaMs": 3885.8348750000005,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
+ "declaredTools": 72,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "voiceover_generate",
+ "name": "trim_clip",
"round": 0,
- "atMs": 1807.043874999974,
- "execMs": 0.026624999998603016
+ "atMs": 1551.334583000018,
+ "execMs": 0.005499999999301508,
+ "args": {
+ "clipId": "c1",
+ "in": 0,
+ "out": 5
+ }
}
],
"timings": {
- "gateMs": 1040.195291000011,
+ "gateMs": 217.80354200000875,
"gateSkipped": false,
- "ttftMs": 3196.6332499999553,
- "modelMs": 3391.085583999986,
- "toolMs": 0.026624999998603016,
- "totalMs": 3398.12820799998,
- "judgeMs": 1165.7955000000075,
+ "ttftMs": 2617.0373330000148,
+ "modelMs": 2538.0882920000004,
+ "toolMs": 0.005499999999301508,
+ "totalMs": 2761.014583000011,
+ "judgeMs": 221.55958299999475,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1803.567166999972,
+ "roundMs": 1329.2464169999876,
"firstDeltaMs": null,
"calls": [
{
- "name": "voiceover_generate",
- "execMs": 0.026624999998603016
+ "name": "trim_clip",
+ "execMs": 0.005499999999301508
}
]
},
{
- "roundMs": 1587.5184170000139,
- "firstDeltaMs": 1386.3612089999951,
+ "roundMs": 1208.8418750000128,
+ "firstDeltaMs": 1064.9870419999934,
"calls": []
}
]
@@ -2573,41 +2581,55 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "library",
+ "editor"
+ ],
+ "declaredTools": 72,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": "the tool did not run as requested, and the assistant failed to perform the editing task.",
+ "judgeNote": null,
"trace": [
{
- "name": "voiceover_generate",
+ "name": "trim_clip",
"round": 0,
- "atMs": 6876.957374999998,
- "execMs": 0.005667000019457191
+ "atMs": 1514.858332999982,
+ "execMs": 0.004165999998804182,
+ "args": {
+ "out": 5,
+ "clipId": "c1"
+ }
}
],
"timings": {
- "gateMs": 6876.627457999974,
+ "gateMs": 286.5764579999959,
"gateSkipped": false,
- "ttftMs": 8366.120875000022,
- "modelMs": 8526.764082999958,
- "toolMs": 0.005667000019457191,
- "totalMs": 8543.480250000022,
- "judgeMs": 1152.1452500000014,
+ "ttftMs": 2765.8231249999953,
+ "modelMs": 2626.754917000013,
+ "toolMs": 0.004165999998804182,
+ "totalMs": 2915.7682909999858,
+ "judgeMs": 177.7972909999953,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6862.413082999992,
+ "roundMs": 1226.7500419999997,
"firstDeltaMs": null,
"calls": [
{
- "name": "voiceover_generate",
- "execMs": 0.005667000019457191
+ "name": "trim_clip",
+ "execMs": 0.004165999998804182
}
]
},
{
- "roundMs": 1664.350999999966,
- "firstDeltaMs": 1487.2907499999856,
+ "roundMs": 1400.0048750000133,
+ "firstDeltaMs": 1250.1533750000235,
"calls": []
}
]
@@ -2616,85 +2638,102 @@
]
},
{
- "name": "music-ask-single-generate",
+ "name": "transition-ask-styled",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1763,
- "p95": 1834,
- "mean": 1592,
- "n": 3
+ "p50": 215,
+ "p95": 215,
+ "mean": 215,
+ "n": 2
},
"ttftMs": {
- "p50": 4144,
- "p95": 4826,
- "mean": 4234,
- "n": 3
+ "p50": 2857,
+ "p95": 6292,
+ "mean": 4574,
+ "n": 2
},
"totalMs": {
- "p50": 4984,
- "p95": 5366,
- "mean": 4907,
- "n": 3
+ "p50": 2966,
+ "p95": 6440,
+ "mean": 4703,
+ "n": 2
},
"modelMs": {
- "p50": 4980,
- "p95": 5355,
- "mean": 4900,
- "n": 3
+ "p50": 2748,
+ "p95": 6221,
+ "mean": 4484,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 2.5,
+ "max": 3
}
},
"runs": [
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "animation",
+ "transitions",
+ "editor"
+ ],
+ "declaredTools": 68,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_music",
+ "name": "set_transition",
"round": 0,
- "atMs": 3034.780374999973,
- "execMs": 0.0959170000278391
+ "atMs": 1766.6607920000097,
+ "execMs": 0.08154099999228492,
+ "args": {
+ "style": "pushdown",
+ "clipId": "c1",
+ "seconds": 0.5
+ }
}
],
"timings": {
- "gateMs": 1833.6837090000045,
+ "gateMs": 215.1701250000042,
"gateSkipped": false,
- "ttftMs": 4826.294500000018,
- "modelMs": 5354.766709000047,
- "toolMs": 0.0959170000278391,
- "totalMs": 5366.453832999978,
- "judgeMs": 1688.5583329999936,
+ "ttftMs": 2856.6242499999935,
+ "modelMs": 2747.8471250000002,
+ "toolMs": 0.08154099999228492,
+ "totalMs": 2966.304708999989,
+ "judgeMs": 262.8814999999886,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3025.3701670000446,
+ "roundMs": 1549.3415410000016,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_music",
- "execMs": 0.0959170000278391
+ "name": "set_transition",
+ "execMs": 0.08154099999228492
}
]
},
{
- "roundMs": 2329.3965420000022,
- "firstDeltaMs": 1789.382459000044,
+ "roundMs": 1198.5055839999986,
+ "firstDeltaMs": 1088.9722919999913,
"calls": []
}
]
@@ -2702,85 +2741,65 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "skill": "transitions-and-fades",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "animation",
+ "transitions",
+ "library",
+ "editor"
+ ],
+ "declaredTools": 75,
+ "routingMisses": [],
+ "roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_music",
- "round": 0,
- "atMs": 2020.850792000012,
- "execMs": 0.002709000022150576
+ "name": "set_transition",
+ "round": 1,
+ "atMs": 3791.700832999981,
+ "execMs": 0.009999999980209395,
+ "args": {
+ "seconds": 0.5,
+ "clipId": "c1",
+ "style": "pushdown"
+ }
}
],
"timings": {
- "gateMs": 1180.0750000000116,
+ "gateMs": 215.35533300001407,
"gateSkipped": false,
- "ttftMs": 3731.2131250000093,
- "modelMs": 4366.238834000018,
- "toolMs": 0.002709000022150576,
- "totalMs": 4369.9761249999865,
- "judgeMs": 1704.6961250000168,
+ "ttftMs": 6291.746541999979,
+ "modelMs": 6221.118248999963,
+ "toolMs": 0.009999999980209395,
+ "totalMs": 6440.362166999985,
+ "judgeMs": 518.4899999999907,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2018.122875000001,
+ "roundMs": 1501.8468329999887,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_music",
- "execMs": 0.002709000022150576
- }
- ]
- },
- {
- "roundMs": 2348.115959000017,
- "firstDeltaMs": 1709.4713750000228,
"calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "generate_music",
- "round": 0,
- "atMs": 2536.9601250000414,
- "execMs": 0.0036250000121071935
- }
- ],
- "timings": {
- "gateMs": 1763.0954159999965,
- "gateSkipped": false,
- "ttftMs": 4144.107541000005,
- "modelMs": 4980.17083399999,
- "toolMs": 0.0036250000121071935,
- "totalMs": 4983.538750000007,
- "judgeMs": 1018.406707999995,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ },
{
- "roundMs": 2534.671208999993,
+ "roundMs": 2071.82699999999,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_music",
- "execMs": 0.0036250000121071935
+ "name": "set_transition",
+ "execMs": 0.009999999980209395
}
]
},
{
- "roundMs": 2445.4996249999967,
- "firstDeltaMs": 1606.2081670000334,
+ "roundMs": 2647.444415999984,
+ "firstDeltaMs": 2499.012290999992,
"calls": []
}
]
@@ -2789,85 +2808,106 @@
]
},
{
- "name": "trim-ask-single-tool",
+ "name": "slide-in-direction-correct",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 979,
- "p95": 1578,
- "mean": 1157,
- "n": 3
+ "p50": 260,
+ "p95": 263,
+ "mean": 261,
+ "n": 2
},
"ttftMs": {
- "p50": 4147,
- "p95": 4434,
- "mean": 4166,
- "n": 3
+ "p50": 2700,
+ "p95": 4221,
+ "mean": 3460,
+ "n": 2
},
"totalMs": {
- "p50": 4684,
- "p95": 4858,
- "mean": 4704,
- "n": 3
+ "p50": 2788,
+ "p95": 4408,
+ "mean": 3598,
+ "n": 2
},
"modelMs": {
- "p50": 4678,
- "p95": 4856,
- "mean": 4699,
- "n": 3
+ "p50": 2526,
+ "p95": 4140,
+ "mean": 3333,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 2.5,
+ "max": 3
}
},
"runs": [
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "animation",
+ "transitions"
+ ],
+ "declaredTools": 63,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "trim_clip",
- "round": 0,
- "atMs": 2533.0589579999796,
- "execMs": 0.012208000000100583
+ "name": "set_animation",
+ "round": 1,
+ "atMs": 3143.269540999987,
+ "execMs": 0.07474999999976717,
+ "args": {
+ "which": "in",
+ "style": "slideright",
+ "clipId": "c2",
+ "seconds": 0.5
+ }
}
],
"timings": {
- "gateMs": 1577.6686249999912,
+ "gateMs": 263.201040999993,
"gateSkipped": false,
- "ttftMs": 3916.629292000027,
- "modelMs": 4563.14970800007,
- "toolMs": 0.012208000000100583,
- "totalMs": 4569.493917000014,
- "judgeMs": 1119.327333000023,
+ "ttftMs": 4220.703832999978,
+ "modelMs": 4139.535084000032,
+ "toolMs": 0.07474999999976717,
+ "totalMs": 4408.397790999996,
+ "judgeMs": 542.0186660000181,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2528.740833000047,
+ "roundMs": 1399.340125000017,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 1476.0649590000103,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.012208000000100583
+ "name": "set_animation",
+ "execMs": 0.07474999999976717
}
]
},
{
- "roundMs": 2034.4088750000228,
- "firstDeltaMs": 1381.7640409999876,
+ "roundMs": 1264.1300000000047,
+ "firstDeltaMs": 1076.5062500000058,
"calls": []
}
]
@@ -2876,126 +2916,100 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "animation",
+ "transitions"
+ ],
+ "declaredTools": 64,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "trim_clip",
+ "name": "set_animation",
"round": 0,
- "atMs": 2372.023749999993,
- "execMs": 0.00466699997195974
+ "atMs": 1625.3120000000054,
+ "execMs": 0.003915999986929819,
+ "args": {
+ "seconds": 0.5,
+ "clipId": "c2",
+ "style": "slideright",
+ "which": "in"
+ }
}
],
"timings": {
- "gateMs": 979.357874999987,
+ "gateMs": 259.50512499999604,
"gateSkipped": false,
- "ttftMs": 4147.184000000008,
- "modelMs": 4678.363332999987,
- "toolMs": 0.00466699997195974,
- "totalMs": 4684.062333000009,
- "judgeMs": 1038.531667000032,
+ "ttftMs": 2700.167665999994,
+ "modelMs": 2526.1960839999956,
+ "toolMs": 0.003915999986929819,
+ "totalMs": 2787.675125000009,
+ "judgeMs": 325.18249999999534,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2367.833166999975,
+ "roundMs": 1364.479833999998,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.00466699997195974
+ "name": "set_animation",
+ "execMs": 0.003915999986929819
}
]
},
{
- "roundMs": 2310.5301660000114,
- "firstDeltaMs": 1773.836874999979,
+ "roundMs": 1161.7162499999977,
+ "firstDeltaMs": 1074.3126660000125,
"calls": []
}
]
}
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "trim_clip",
- "round": 0,
- "atMs": 2884.0214169999817,
- "execMs": 0.00712500000372529
- }
- ],
- "timings": {
- "gateMs": 913.4087500000023,
- "gateSkipped": false,
- "ttftMs": 4433.711708999996,
- "modelMs": 4856.052708000003,
- "toolMs": 0.00712500000372529,
- "totalMs": 4858.051334000018,
- "judgeMs": 1024.622040999995,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2882.901165999996,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "trim_clip",
- "execMs": 0.00712500000372529
- }
- ]
- },
- {
- "roundMs": 1973.1515420000069,
- "firstDeltaMs": 1548.9225839999854,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "transition-ask-styled",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 953,
- "p95": 1072,
- "mean": 976,
- "n": 3
+ }
+ ]
+ },
+ {
+ "name": "look-ask-single-tool",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 240,
+ "p95": 290,
+ "mean": 265,
+ "n": 2
},
"ttftMs": {
- "p50": 4062,
- "p95": 7804,
- "mean": 5298,
- "n": 3
+ "p50": 2938,
+ "p95": 2994,
+ "mean": 2966,
+ "n": 2
},
"totalMs": {
- "p50": 4585,
- "p95": 8416,
- "mean": 5860,
- "n": 3
+ "p50": 3166,
+ "p95": 3201,
+ "mean": 3184,
+ "n": 2
},
"modelMs": {
- "p50": 4582,
- "p95": 8411,
- "mean": 5854,
- "n": 3
+ "p50": 2910,
+ "p95": 2926,
+ "mean": 2918,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -3006,84 +3020,62 @@
{
"pass": true,
"intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_transition",
- "round": 0,
- "atMs": 2584.1580829999875,
- "execMs": 0.04566599999088794
- }
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "effects",
+ "video_gen",
+ "scene",
+ "library",
+ "side_panel",
+ "top_bar"
],
- "timings": {
- "gateMs": 904.3172499999637,
- "gateSkipped": false,
- "ttftMs": 4062.422332999995,
- "modelMs": 4581.75462500006,
- "toolMs": 0.04566599999088794,
- "totalMs": 4584.975166999968,
- "judgeMs": 1100.5815000000293,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2581.870333000028,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.04566599999088794
- }
- ]
- },
- {
- "roundMs": 1999.8842920000316,
- "firstDeltaMs": 1477.4113750000251,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
+ "declaredTools": 95,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
+ "name": "add_effect",
"round": 0,
- "atMs": 5773.465083000017,
- "execMs": 0.008833999978378415
+ "atMs": 1861.2977920000267,
+ "execMs": 0.037958000000799075,
+ "args": {
+ "effect": "vhs",
+ "start": 6,
+ "end": 12.5,
+ "amount": 0.5
+ }
}
],
"timings": {
- "gateMs": 1072.0947089999681,
+ "gateMs": 289.69391599998926,
"gateSkipped": false,
- "ttftMs": 7804.316708000028,
- "modelMs": 8411.398291000049,
- "toolMs": 0.008833999978378415,
- "totalMs": 8415.619458000001,
- "judgeMs": 1154.455707999994,
+ "ttftMs": 2994.052250000008,
+ "modelMs": 2909.613624999998,
+ "toolMs": 0.037958000000799075,
+ "totalMs": 3200.9923750000016,
+ "judgeMs": 234.41870800001197,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5771.006458000047,
+ "roundMs": 1570.4394169999869,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.008833999978378415
+ "name": "add_effect",
+ "execMs": 0.037958000000799075
}
]
},
{
- "roundMs": 2640.3918330000015,
- "firstDeltaMs": 2029.4281670000055,
+ "roundMs": 1339.1742080000113,
+ "firstDeltaMs": 1132.3141659999965,
"calls": []
}
]
@@ -3092,41 +3084,62 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "effects",
+ "video_gen",
+ "scene",
+ "library",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 95,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
+ "name": "add_effect",
"round": 0,
- "atMs": 2601.827667000005,
- "execMs": 0.011624999984633178
+ "atMs": 1858.4160839999968,
+ "execMs": 0.0029999999969732016,
+ "args": {
+ "end": 12.5,
+ "start": 6,
+ "amount": 0.5,
+ "effect": "vhs"
+ }
}
],
"timings": {
- "gateMs": 952.7090829999652,
+ "gateMs": 239.67337499998393,
"gateSkipped": false,
- "ttftMs": 4028.250624999986,
- "modelMs": 4570.054124999966,
- "toolMs": 0.011624999984633178,
- "totalMs": 4579.3036250000005,
- "judgeMs": 996.1543329999549,
+ "ttftMs": 2937.9317089999968,
+ "modelMs": 2925.514500999998,
+ "toolMs": 0.0029999999969732016,
+ "totalMs": 3166.4014170000155,
+ "judgeMs": 265.204000000027,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2594.164417000022,
+ "roundMs": 1618.0305419999931,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.011624999984633178
+ "name": "add_effect",
+ "execMs": 0.0029999999969732016
}
]
},
{
- "roundMs": 1975.8897079999442,
- "firstDeltaMs": 1425.0307079999475,
+ "roundMs": 1307.4839590000047,
+ "firstDeltaMs": 1079.0800839999865,
"calls": []
}
]
@@ -3135,40 +3148,40 @@
]
},
{
- "name": "slide-in-direction-correct",
+ "name": "audio-effect-single-tool",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1125,
- "p95": 1943,
- "mean": 1392,
- "n": 3
+ "p50": 251,
+ "p95": 356,
+ "mean": 304,
+ "n": 2
},
"ttftMs": {
- "p50": 3033,
- "p95": 3084,
- "mean": 3021,
- "n": 3
+ "p50": 2636,
+ "p95": 2729,
+ "mean": 2683,
+ "n": 2
},
"totalMs": {
- "p50": 3229,
- "p95": 3292,
- "mean": 3237,
- "n": 3
+ "p50": 2894,
+ "p95": 2894,
+ "mean": 2894,
+ "n": 2
},
"modelMs": {
- "p50": 3187,
- "p95": 3222,
- "mean": 3059,
- "n": 3
+ "p50": 2534,
+ "p95": 2640,
+ "mean": 2587,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -3179,84 +3192,56 @@
{
"pass": true,
"intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_animation",
- "round": 0,
- "atMs": 1841.8112499999697,
- "execMs": 0.11758299998473376
- }
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "audio",
+ "side_panel"
],
- "timings": {
- "gateMs": 1107.772209000017,
- "gateSkipped": false,
- "ttftMs": 3032.627207999991,
- "modelMs": 3222.3546240000287,
- "toolMs": 0.11758299998473376,
- "totalMs": 3228.6325419999775,
- "judgeMs": 1117.929999999993,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1837.6956660000142,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_animation",
- "execMs": 0.11758299998473376
- }
- ]
- },
- {
- "roundMs": 1384.6589580000145,
- "firstDeltaMs": 1188.760083000001,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
+ "declaredTools": 73,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_animation",
+ "name": "set_clip_sound",
"round": 0,
- "atMs": 1943.4262919999892,
- "execMs": 0.008416999946348369
+ "atMs": 1633.934459000011,
+ "execMs": 0.11125000001629815,
+ "args": {
+ "clipId": "c2",
+ "preset": "phone"
+ }
}
],
"timings": {
- "gateMs": 1943.1847089999937,
+ "gateMs": 250.75916599997436,
"gateSkipped": false,
- "ttftMs": 3084.376542000042,
- "modelMs": 2766.168874999974,
- "toolMs": 0.008416999946348369,
- "totalMs": 3292.352332999988,
- "judgeMs": 1031.8133340000059,
+ "ttftMs": 2729.4537919999857,
+ "modelMs": 2640.348874999967,
+ "toolMs": 0.11125000001629815,
+ "totalMs": 2893.5675420000043,
+ "judgeMs": 215.6016670000099,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1418.3320829999866,
+ "roundMs": 1381.5887089999742,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.008416999946348369
+ "name": "set_clip_sound",
+ "execMs": 0.11125000001629815
}
]
},
{
- "roundMs": 1347.8367919999873,
- "firstDeltaMs": 1139.997583999997,
+ "roundMs": 1258.7601659999928,
+ "firstDeltaMs": 1094.755749999982,
"calls": []
}
]
@@ -3265,41 +3250,56 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "audio",
+ "side_panel"
+ ],
+ "declaredTools": 73,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_animation",
+ "name": "set_clip_sound",
"round": 0,
- "atMs": 1593.6002499999595,
- "execMs": 0.002957999997306615
+ "atMs": 1617.3443329999864,
+ "execMs": 0.021084000007249415,
+ "args": {
+ "preset": "phone",
+ "clipId": "c2"
+ }
}
],
"timings": {
- "gateMs": 1124.607250000001,
+ "gateMs": 356.3399999999965,
"gateSkipped": false,
- "ttftMs": 2946.189207999967,
- "modelMs": 3187.0392909999937,
- "toolMs": 0.002957999997306615,
- "totalMs": 3190.6774999999907,
- "judgeMs": 1269.1875420000288,
+ "ttftMs": 2636.099332999991,
+ "modelMs": 2534.3065840000054,
+ "toolMs": 0.021084000007249415,
+ "totalMs": 2893.990832999989,
+ "judgeMs": 229.26095799999894,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1591.003000000026,
+ "roundMs": 1259.4970420000027,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.002957999997306615
+ "name": "set_clip_sound",
+ "execMs": 0.021084000007249415
}
]
},
{
- "roundMs": 1596.0362909999676,
- "firstDeltaMs": 1352.103041000024,
+ "roundMs": 1274.8095420000027,
+ "firstDeltaMs": 1016.9855420000094,
"calls": []
}
]
@@ -3308,40 +3308,40 @@
]
},
{
- "name": "look-ask-single-tool",
+ "name": "sound-dissolve-single-tool",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1079,
- "p95": 1124,
- "mean": 1094,
- "n": 3
+ "p50": 225,
+ "p95": 296,
+ "mean": 261,
+ "n": 2
},
"ttftMs": {
- "p50": 3895,
- "p95": 4201,
- "mean": 3963,
- "n": 3
+ "p50": 2623,
+ "p95": 2923,
+ "mean": 2773,
+ "n": 2
},
"totalMs": {
- "p50": 4493,
- "p95": 4856,
- "mean": 4570,
- "n": 3
+ "p50": 2867,
+ "p95": 3142,
+ "mean": 3005,
+ "n": 2
},
"modelMs": {
- "p50": 4487,
- "p95": 4851,
- "mean": 4565,
- "n": 3
+ "p50": 2640,
+ "p95": 2844,
+ "mean": 2742,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -3352,84 +3352,58 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "transitions",
+ "audio",
+ "editor"
+ ],
+ "declaredTools": 70,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "add_effect",
+ "name": "set_transition",
"round": 0,
- "atMs": 2384.055916000041,
- "execMs": 0.07483299996238202
- }
- ],
- "timings": {
- "gateMs": 1078.47766699997,
- "gateSkipped": false,
- "ttftMs": 3895.0314160000416,
- "modelMs": 4487.489624999987,
- "toolMs": 0.07483299996238202,
- "totalMs": 4492.896958000027,
- "judgeMs": 1775.0307080000057,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2380.4290000000037,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_effect",
- "execMs": 0.07483299996238202
- }
- ]
- },
- {
- "roundMs": 2107.0606249999837,
- "firstDeltaMs": 1509.3370829999913,
- "calls": []
+ "atMs": 1747.8170419999806,
+ "execMs": 0.07725000000209548,
+ "args": {
+ "clipId": "c1",
+ "style": "audiocross",
+ "seconds": 0.3
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": "imprecise; the edit was applied to the \"remainder of the project\" instead of just the \"second clip\" as requested.",
- "trace": [
- {
- "name": "add_effect",
- "round": 0,
- "atMs": 2720.327500000014,
- "execMs": 0.01816600002348423
}
],
"timings": {
- "gateMs": 1079.444749999966,
+ "gateMs": 296.3960420000076,
"gateSkipped": false,
- "ttftMs": 4200.830667000031,
- "modelMs": 4851.141416000028,
- "toolMs": 0.01816600002348423,
- "totalMs": 4855.825500000035,
- "judgeMs": 1120.0190839999705,
+ "ttftMs": 2922.7595419999852,
+ "modelMs": 2843.9709999999905,
+ "toolMs": 0.07725000000209548,
+ "totalMs": 3141.900667000009,
+ "judgeMs": 302.0467920000083,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2718.8158750000293,
+ "roundMs": 1450.6633750000037,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_effect",
- "execMs": 0.01816600002348423
+ "name": "set_transition",
+ "execMs": 0.07725000000209548
}
]
},
{
- "roundMs": 2132.3255409999983,
- "firstDeltaMs": 1477.4586250000284,
+ "roundMs": 1393.3076249999867,
+ "firstDeltaMs": 1174.4249589999963,
"calls": []
}
]
@@ -3438,41 +3412,58 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "transitions",
+ "audio",
+ "editor"
+ ],
+ "declaredTools": 70,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "add_effect",
+ "name": "set_transition",
"round": 0,
- "atMs": 2306,
- "execMs": 0.016042000032030046
+ "atMs": 1571.1429999999818,
+ "execMs": 0.006624999979976565,
+ "args": {
+ "clipId": "c1",
+ "seconds": 0.3,
+ "style": "audiocross"
+ }
}
],
"timings": {
- "gateMs": 1124.0804999999818,
+ "gateMs": 225.32741700002225,
"gateSkipped": false,
- "ttftMs": 3791.8588749999763,
- "modelMs": 4356.02362599998,
- "toolMs": 0.016042000032030046,
- "totalMs": 4362.34604199999,
- "judgeMs": 1631.0080839999719,
+ "ttftMs": 2623.195624999993,
+ "modelMs": 2639.6257500000065,
+ "toolMs": 0.006624999979976565,
+ "totalMs": 2867.185999999987,
+ "judgeMs": 279.27512499998556,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2302.5827089999802,
+ "roundMs": 1344.689000000013,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_effect",
- "execMs": 0.016042000032030046
+ "name": "set_transition",
+ "execMs": 0.006624999979976565
}
]
},
{
- "roundMs": 2053.440917,
- "firstDeltaMs": 1483.1146670000162,
+ "roundMs": 1294.9367499999935,
+ "firstDeltaMs": 1051.1058339999872,
"calls": []
}
]
@@ -3481,128 +3472,121 @@
]
},
{
- "name": "audio-effect-single-tool",
+ "name": "animation-overrides-transition",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1170,
- "p95": 1779,
- "mean": 1348,
- "n": 3
+ "p50": 255,
+ "p95": 301,
+ "mean": 278,
+ "n": 2
},
"ttftMs": {
- "p50": 2934,
- "p95": 2962,
- "mean": 2853,
- "n": 3
+ "p50": 4192,
+ "p95": 6068,
+ "mean": 5130,
+ "n": 2
},
"totalMs": {
- "p50": 3122,
- "p95": 3232,
- "mean": 3059,
- "n": 3
+ "p50": 4350,
+ "p95": 6422,
+ "mean": 5386,
+ "n": 2
},
"modelMs": {
- "p50": 2814,
- "p95": 3220,
- "mean": 2914,
- "n": 3
+ "p50": 4091,
+ "p95": 6119,
+ "mean": 5105,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 2.5,
+ "max": 3
}
},
"runs": [
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "animation",
+ "transitions",
+ "editor"
+ ],
+ "declaredTools": 67,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_clip_sound",
+ "name": "remove_transition",
"round": 0,
- "atMs": 1821.2804579999647,
- "execMs": 0.36429200001293793
+ "atMs": 1751.9912089999998,
+ "execMs": 0.09441600000718608,
+ "args": {
+ "transitionId": "tr-c1"
+ }
+ },
+ {
+ "name": "set_animation",
+ "round": 1,
+ "atMs": 3176.6528340000077,
+ "execMs": 0.0044169999891892076,
+ "args": {
+ "clipId": "c2",
+ "style": "pop",
+ "seconds": 0.5,
+ "which": "in"
+ }
}
],
"timings": {
- "gateMs": 1170.4378340000403,
+ "gateMs": 254.50633299996844,
"gateSkipped": false,
- "ttftMs": 2962.154249999963,
- "modelMs": 3220.2934589999495,
- "toolMs": 0.36429200001293793,
- "totalMs": 3231.8749579999712,
- "judgeMs": 921.7363749999786,
+ "ttftMs": 4192.076667000016,
+ "modelMs": 4090.9922099999967,
+ "toolMs": 0.09883299999637529,
+ "totalMs": 4349.64837499999,
+ "judgeMs": 589.4487920000101,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1813.8348339999793,
+ "roundMs": 1495.0590840000077,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_sound",
- "execMs": 0.36429200001293793
+ "name": "remove_transition",
+ "execMs": 0.09441600000718608
}
]
},
{
- "roundMs": 1406.4586249999702,
- "firstDeltaMs": 1138.0572079999838,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_clip_sound",
- "round": 0,
- "atMs": 1779.4611250000307,
- "execMs": 0.027082999993581325
- }
- ],
- "timings": {
- "gateMs": 1778.5980410000193,
- "gateSkipped": false,
- "ttftMs": 2933.819125000038,
- "modelMs": 2707.81570799998,
- "toolMs": 0.027082999993581325,
- "totalMs": 3122.486375000037,
- "judgeMs": 1725.4163749999716,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1368.481415999995,
+ "roundMs": 1423.5612919999985,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_sound",
- "execMs": 0.027082999993581325
+ "name": "set_animation",
+ "execMs": 0.0044169999891892076
}
]
},
{
- "roundMs": 1339.334291999985,
- "firstDeltaMs": 1151.2451250000158,
+ "roundMs": 1172.3718339999905,
+ "firstDeltaMs": 1014.9619999999995,
"calls": []
}
]
@@ -3611,41 +3595,71 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "animation",
+ "transitions",
+ "editor"
+ ],
+ "declaredTools": 67,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_clip_sound",
+ "name": "remove_transition",
+ "round": 0,
+ "atMs": 2244.4309999999823,
+ "execMs": 0.0015410000341944396,
+ "args": {
+ "transitionId": "c1-transition-0"
+ }
+ },
+ {
+ "name": "set_animation",
"round": 0,
- "atMs": 1391.466875000042,
- "execMs": 0.021083999949041754
+ "atMs": 2244.4865839999984,
+ "execMs": 0.000792000035289675,
+ "args": {
+ "style": "pop",
+ "clipId": "c2",
+ "seconds": 0.2,
+ "which": "in"
+ }
}
],
"timings": {
- "gateMs": 1094.5957499999786,
+ "gateMs": 301.0832919999957,
"gateSkipped": false,
- "ttftMs": 2663.2801250000484,
- "modelMs": 2814.385332999984,
- "toolMs": 0.021083999949041754,
- "totalMs": 2822.6786660000216,
- "judgeMs": 1047.2447909999755,
+ "ttftMs": 6067.817124999943,
+ "modelMs": 6119.131208999956,
+ "toolMs": 0.0023330000694841146,
+ "totalMs": 6421.992083999969,
+ "judgeMs": 509.14308300003177,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1386.7783749999944,
+ "roundMs": 1942.0425839999807,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_sound",
- "execMs": 0.021083999949041754
+ "name": "remove_transition",
+ "execMs": 0.0015410000341944396
+ },
+ {
+ "name": "set_animation",
+ "execMs": 0.000792000035289675
}
]
},
{
- "roundMs": 1427.6069579999894,
- "firstDeltaMs": 1268.6642920000013,
+ "roundMs": 4177.088624999975,
+ "firstDeltaMs": 3822.990208000003,
"calls": []
}
]
@@ -3654,860 +3668,396 @@
]
},
{
- "name": "sound-dissolve-single-tool",
- "bucket": "single-tool",
+ "name": "filler-words-cut-with-editor",
+ "bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1573,
- "p95": 1852,
- "mean": 1526,
- "n": 3
+ "p50": 218,
+ "p95": 290,
+ "mean": 254,
+ "n": 2
},
"ttftMs": {
- "p50": 5844,
- "p95": 12238,
- "mean": 7115,
- "n": 3
+ "p50": 32285,
+ "p95": 38799,
+ "mean": 35542,
+ "n": 2
},
"totalMs": {
- "p50": 6136,
- "p95": 12581,
- "mean": 7380,
- "n": 3
+ "p50": 32550,
+ "p95": 39155,
+ "mean": 35852,
+ "n": 2
},
"modelMs": {
- "p50": 6129,
- "p95": 12571,
- "mean": 7321,
- "n": 3
+ "p50": 32237,
+ "p95": 38907,
+ "mean": 35572,
+ "n": 2
},
"toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
+ "p50": 1,
+ "p95": 1,
+ "mean": 1,
+ "n": 2
},
"rounds": {
- "mean": 2.3,
- "max": 3
+ "mean": 16,
+ "max": 17
}
},
"runs": [
{
"pass": true,
"intent": "complex",
+ "skill": "watching-and-cutting",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "transitions",
+ "subtitles",
+ "audio",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 100,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": null,
+ "judgeNote": "upsell 0.54",
"trace": [
{
- "name": "set_transition",
+ "name": "find_filler",
"round": 1,
- "atMs": 9939.379875000042,
- "execMs": 0.33724999998230487
- }
- ],
- "timings": {
- "gateMs": 1573.3168329999899,
- "gateSkipped": false,
- "ttftMs": 12237.521334000048,
- "modelMs": 12571.074790000042,
- "toolMs": 0.33724999998230487,
- "totalMs": 12580.730875000008,
- "judgeMs": 1038.5593339999905,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1571.9622910000035,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 8360.06016600004,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.33724999998230487
- }
- ]
- },
- {
- "roundMs": 2639.0523329999996,
- "firstDeltaMs": 2296.247541000019,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_transition",
- "round": 0,
- "atMs": 4541.94079100003,
- "execMs": 0.01295800000661984
- }
- ],
- "timings": {
- "gateMs": 1154.1812500000233,
- "gateSkipped": false,
- "ttftMs": 5843.677166000009,
- "modelMs": 6128.993918000022,
- "toolMs": 0.01295800000661984,
- "totalMs": 6135.5470000000205,
- "judgeMs": 1081.6055840000045,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 4538.053084000014,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.01295800000661984
- }
- ]
- },
- {
- "roundMs": 1590.9408340000082,
- "firstDeltaMs": 1300.1020840000128,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_transition",
- "round": 0,
- "atMs": 1852.4392499999958,
- "execMs": 0.021957999968435615
- }
- ],
- "timings": {
- "gateMs": 1851.8504170000087,
- "gateSkipped": false,
- "ttftMs": 3263.625708000036,
- "modelMs": 3262.354249999975,
- "toolMs": 0.021957999968435615,
- "totalMs": 3423.8789170000236,
- "judgeMs": 1122.176167000027,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1692.7075839999598,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.021957999968435615
- }
- ]
- },
- {
- "roundMs": 1569.646666000015,
- "firstDeltaMs": 1409.7089159999741,
- "calls": []
+ "atMs": 7303.833249999967,
+ "execMs": 0.2982499999925494,
+ "args": {
+ "clip_id": "c1",
+ "track": 0
}
- ]
- }
- }
- ]
- },
- {
- "name": "animation-overrides-transition",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1187,
- "p95": 1745,
- "mean": 1302,
- "n": 3
- },
- "ttftMs": {
- "p50": 7502,
- "p95": 7638,
- "mean": 7082,
- "n": 3
- },
- "totalMs": {
- "p50": 7796,
- "p95": 7987,
- "mean": 7419,
- "n": 3
- },
- "modelMs": {
- "p50": 7793,
- "p95": 7980,
- "mean": 7412,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 3,
- "max": 3
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_transition",
- "round": 1,
- "atMs": 5533.123625000007,
- "execMs": 0.14137500000651926
},
{
- "name": "set_animation",
- "round": 1,
- "atMs": 5533.449458999967,
- "execMs": 0.003874999994877726
- }
- ],
- "timings": {
- "gateMs": 1186.8304999999818,
- "gateSkipped": false,
- "ttftMs": 7637.508458999975,
- "modelMs": 7979.711833999958,
- "toolMs": 0.14525000000139698,
- "totalMs": 7986.935624999984,
- "judgeMs": 864.0507499999949,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1184.97537499998,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 4342.449084000022,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.14137500000651926
- },
- {
- "name": "set_animation",
- "execMs": 0.003874999994877726
- }
- ]
- },
- {
- "roundMs": 2452.2873749999562,
- "firstDeltaMs": 2102.9659999999567,
- "calls": []
+ "name": "detect_silence",
+ "round": 2,
+ "atMs": 10485.969624999969,
+ "execMs": 0.019415999995544553,
+ "args": {
+ "to": 6,
+ "clip_id": "c1",
+ "from": 0
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_transition",
- "round": 1,
- "atMs": 4696.082500000019,
- "execMs": 0.0020000000367872417
},
{
- "name": "set_animation",
- "round": 1,
- "atMs": 4696.180290999997,
- "execMs": 0.0014169999631121755
- }
- ],
- "timings": {
- "gateMs": 973.6624580000062,
- "gateSkipped": false,
- "ttftMs": 7502.332666000002,
- "modelMs": 7792.872332000057,
- "toolMs": 0.003416999999899417,
- "totalMs": 7796.209916000022,
- "judgeMs": 993.4386250000098,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 972.874375000014,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 3721.09591600002,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.0020000000367872417
- },
- {
- "name": "set_animation",
- "execMs": 0.0014169999631121755
- }
- ]
- },
- {
- "roundMs": 3098.902041000023,
- "firstDeltaMs": 2805.442708000017,
- "calls": []
+ "name": "watch_video",
+ "round": 3,
+ "atMs": 12500.72104199999,
+ "execMs": 0.029291999991983175,
+ "args": {
+ "clip_id": "c1",
+ "from": 0,
+ "interval_seconds": 1,
+ "to": 6
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_transition",
- "round": 1,
- "atMs": 4475.805167000042,
- "execMs": 0.010707999987062067
},
{
- "name": "set_animation",
- "round": 1,
- "atMs": 4476.193375000032,
- "execMs": 0.0035829999833367765
- }
- ],
- "timings": {
- "gateMs": 1745.4226660000277,
- "gateSkipped": false,
- "ttftMs": 6107.071875000023,
- "modelMs": 6464.680457000039,
- "toolMs": 0.014290999970398843,
- "totalMs": 6473.639459000027,
- "judgeMs": 1149.5836249999702,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1743.0707910000347,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2725.657750000013,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.010707999987062067
- },
+ "name": "note_source",
+ "round": 4,
+ "atMs": 14568.631832999992,
+ "execMs": 0.014416999998502433,
+ "args": {
+ "clip_id": "c1",
+ "notes": [
{
- "name": "set_animation",
- "execMs": 0.0035829999833367765
+ "from": 0,
+ "to": 6,
+ "text": "Speaker by the beach talking with dogs in view"
}
]
- },
- {
- "roundMs": 1995.951915999991,
- "firstDeltaMs": 1629.7183749999967,
- "calls": []
}
- ]
- }
- }
- ]
- },
- {
- "name": "filler-words-cut-with-editor",
- "bucket": "multi-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1139,
- "p95": 1891,
- "mean": 1360,
- "n": 3
- },
- "ttftMs": {
- "p50": 33169,
- "p95": 35657,
- "mean": 31768,
- "n": 3
- },
- "totalMs": {
- "p50": 33649,
- "p95": 35960,
- "mean": 32151,
- "n": 3
- },
- "modelMs": {
- "p50": 33626,
- "p95": 35929,
- "mean": 32124,
- "n": 3
- },
- "toolMs": {
- "p50": 1,
- "p95": 1,
- "mean": 1,
- "n": 3
- },
- "rounds": {
- "mean": 12,
- "max": 14
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "detect_silence",
- "round": 1,
- "atMs": 3926.368415999983,
- "execMs": 0.044999999983701855
- },
- {
- "name": "listen_audio",
- "round": 2,
- "atMs": 6085.617458000022,
- "execMs": 0.012499999953433871
- },
- {
- "name": "detect_silence",
- "round": 3,
- "atMs": 8397.153041000012,
- "execMs": 0.011916999996174127
- },
- {
- "name": "split_at",
- "round": 4,
- "atMs": 12851.928332999989,
- "execMs": 0.5322089999681339
},
{
- "name": "split_at",
- "round": 4,
- "atMs": 12852.792415999982,
- "execMs": 0.03700000001117587
- },
- {
- "name": "delete_item",
+ "name": "select",
"round": 5,
- "atMs": 19924.53008300002,
- "execMs": 0.07795900001656264
+ "atMs": 16454.25408299995,
+ "execMs": 0.024625000020023435,
+ "args": {
+ "kind": "clip",
+ "id": "c1"
+ }
},
{
- "name": "get_state",
+ "name": "trim_clip",
"round": 6,
- "atMs": 22105.71754099999,
- "execMs": 0.16866700001992285
+ "atMs": 18193.951166999992,
+ "execMs": 0.5514580000308342,
+ "args": {
+ "clipId": "c1",
+ "in": 0.45
+ }
},
{
- "name": "delete_cue",
+ "name": "split_at",
"round": 7,
- "atMs": 24439.046499999997,
- "execMs": 0.014833999972324818
+ "atMs": 19819.45170799998,
+ "execMs": 0.08112499996786937,
+ "args": {
+ "t": 4.1
+ }
},
{
- "name": "align_to_audio",
+ "name": "trim_clip",
"round": 8,
- "atMs": 26445.709124999994,
- "execMs": 0.007833000039681792
- },
- {
- "name": "get_state",
- "round": 9,
- "atMs": 29174.740500000014,
- "execMs": 0.0540839999448508
- },
- {
- "name": "update_cue",
- "round": 10,
- "atMs": 31530.171333000006,
- "execMs": 0.01729200000409037
- },
- {
- "name": "update_cue",
- "round": 10,
- "atMs": 31530.301665999985,
- "execMs": 0.025750000029802322
- }
- ],
- "timings": {
- "gateMs": 1049.1254159999662,
- "gateSkipped": false,
- "ttftMs": 33169.077583000006,
- "modelMs": 33625.68183499988,
- "toolMs": 1.0050449999398552,
- "totalMs": 33649.34962499997,
- "judgeMs": 865.2688750000088,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1048.6091669999878,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2872.769499999995,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.044999999983701855
- }
- ]
- },
- {
- "roundMs": 2157.5817919999827,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.012499999953433871
- }
- ]
- },
- {
- "roundMs": 2310.6631669999915,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.011916999996174127
- }
- ]
- },
- {
- "roundMs": 4451.958166999975,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.5322089999681339
- },
- {
- "name": "split_at",
- "execMs": 0.03700000001117587
- }
- ]
- },
- {
- "roundMs": 7069.517624999979,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.07795900001656264
- }
- ]
- },
- {
- "roundMs": 2180.207374999998,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.16866700001992285
- }
- ]
- },
- {
- "roundMs": 2332.1539169999887,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_cue",
- "execMs": 0.014833999972324818
- }
- ]
- },
- {
- "roundMs": 2003.7060000000056,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "align_to_audio",
- "execMs": 0.007833000039681792
- }
- ]
- },
- {
- "roundMs": 2727.5879580000183,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.0540839999448508
- }
- ]
- },
- {
- "roundMs": 2353.0849999999627,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.01729200000409037
- },
- {
- "name": "update_cue",
- "execMs": 0.025750000029802322
- }
- ]
- },
- {
- "roundMs": 2117.8421669999952,
- "firstDeltaMs": 1637.7451250000158,
- "calls": []
+ "atMs": 21819.94316699996,
+ "execMs": 0.018207999994046986,
+ "args": {
+ "in": 5.6,
+ "clipId": "c1-2"
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "detect_silence",
- "round": 1,
- "atMs": 4347.621625000029,
- "execMs": 0.1914589999942109
- },
- {
- "name": "listen_audio",
- "round": 1,
- "atMs": 4348.42541700002,
- "execMs": 0.03599999996367842
- },
- {
- "name": "watch_video",
- "round": 2,
- "atMs": 9290.12599999999,
- "execMs": 0.006041000015102327
- },
- {
- "name": "note_source",
- "round": 3,
- "atMs": 11946.30479199998,
- "execMs": 0.02108299999963492
- },
- {
- "name": "split_at",
- "round": 4,
- "atMs": 13826.460666999978,
- "execMs": 0.14829199999803677
- },
- {
- "name": "split_at",
- "round": 4,
- "atMs": 13826.88841700001,
- "execMs": 0.06295799999497831
- },
- {
- "name": "delete_item",
- "round": 5,
- "atMs": 15883.943874999997,
- "execMs": 0.10516700003063306
},
{
"name": "get_state",
- "round": 6,
- "atMs": 18439.494292000018,
- "execMs": 0.17183300002943724
+ "round": 9,
+ "atMs": 24824.277000000002,
+ "execMs": 0.14995900000212714,
+ "args": {}
},
{
- "name": "delete_cue",
- "round": 7,
- "atMs": 21690.227208000026,
- "execMs": 0.04141700000036508
+ "name": "refine_speech_cuts",
+ "round": 10,
+ "atMs": 26635.33041699999,
+ "execMs": 0.03062500001396984,
+ "args": {
+ "clip_ids": [
+ "c1-1",
+ "c1-2"
+ ],
+ "pace": "natural"
+ }
},
{
"name": "update_cue",
- "round": 7,
- "atMs": 21690.649957999995,
- "execMs": 0.020499999984167516
+ "round": 11,
+ "atMs": 28624.792749999964,
+ "execMs": 0.013500000000931323,
+ "args": {
+ "end": 1.8,
+ "text": "So today we're",
+ "id": "cue1",
+ "start": 0
+ }
},
{
- "name": "update_cue",
- "round": 7,
- "atMs": 21690.850916999974,
- "execMs": 0.020124999980907887
+ "name": "align_to_audio",
+ "round": 12,
+ "atMs": 30336.36599999998,
+ "execMs": 0.006457999988924712,
+ "args": {
+ "track": 0,
+ "target": "captions"
+ }
},
{
- "name": "update_cue",
- "round": 7,
- "atMs": 21691.02008300001,
- "execMs": 0.015416999987792224
+ "name": "get_state",
+ "round": 13,
+ "atMs": 32870.604082999984,
+ "execMs": 0.09083299996564165,
+ "args": {}
},
{
- "name": "refine_speech_cuts",
- "round": 8,
- "atMs": 23820.785083000024,
- "execMs": 0.03599999996367842
+ "name": "delete_cue",
+ "round": 14,
+ "atMs": 34794.35333299998,
+ "execMs": 0.061999999976251274,
+ "args": {
+ "id": "cue3"
+ }
+ },
+ {
+ "name": "listen_audio",
+ "round": 15,
+ "atMs": 36722.856582999986,
+ "execMs": 0.021999999997206032,
+ "args": {
+ "from": 3.5,
+ "to": 4.55,
+ "clip_id": "c1-1"
+ }
}
],
"timings": {
- "gateMs": 1139.3610410000547,
+ "gateMs": 217.8545419999864,
"gateSkipped": false,
- "ttftMs": 26476.659291999997,
- "modelMs": 26815.80679199996,
- "toolMs": 0.8762919999426231,
- "totalMs": 26842.60483299999,
- "judgeMs": 1039.5110410000198,
+ "ttftMs": 38799.28816699999,
+ "modelMs": 38907.04075100011,
+ "toolMs": 1.4121659999364056,
+ "totalMs": 39155.218957999954,
+ "judgeMs": 582.8360840000096,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1138.2258750000037,
+ "roundMs": 1595.4662499999977,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 3205.038708999986,
+ "roundMs": 5485.319834000024,
"firstDeltaMs": null,
"calls": [
{
- "name": "detect_silence",
- "execMs": 0.1914589999942109
- },
+ "name": "find_filler",
+ "execMs": 0.2982499999925494
+ }
+ ]
+ },
+ {
+ "roundMs": 3179.2717909999774,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "listen_audio",
- "execMs": 0.03599999996367842
+ "name": "detect_silence",
+ "execMs": 0.019415999995544553
}
]
},
{
- "roundMs": 4937.483291999961,
+ "roundMs": 2012.2938750000321,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.006041000015102327
+ "execMs": 0.029291999991983175
}
]
},
{
- "roundMs": 2652.9713749999646,
+ "roundMs": 2065.308707999997,
"firstDeltaMs": null,
"calls": [
{
"name": "note_source",
- "execMs": 0.02108299999963492
+ "execMs": 0.014416999998502433
}
]
},
{
- "roundMs": 1877.8197919999948,
+ "roundMs": 1882.5462919999845,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.14829199999803677
- },
+ "name": "select",
+ "execMs": 0.024625000020023435
+ }
+ ]
+ },
+ {
+ "roundMs": 1738.1774159999914,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "trim_clip",
+ "execMs": 0.5514580000308342
+ }
+ ]
+ },
+ {
+ "roundMs": 1623.5362920000334,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "split_at",
- "execMs": 0.06295799999497831
+ "execMs": 0.08112499996786937
}
]
},
{
- "roundMs": 2055.239082999993,
+ "roundMs": 1999.857958000037,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.10516700003063306
+ "name": "trim_clip",
+ "execMs": 0.018207999994046986
}
]
},
{
- "roundMs": 2554.1294580000103,
+ "roundMs": 3003.715124999988,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.17183300002943724
+ "execMs": 0.14995900000212714
}
]
},
{
- "roundMs": 3247.589791000006,
+ "roundMs": 1809.7702920000302,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_cue",
- "execMs": 0.04141700000036508
- },
+ "name": "refine_speech_cuts",
+ "execMs": 0.03062500001396984
+ }
+ ]
+ },
+ {
+ "roundMs": 1988.8142089999747,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "update_cue",
- "execMs": 0.020499999984167516
- },
+ "execMs": 0.013500000000931323
+ }
+ ]
+ },
+ {
+ "roundMs": 1710.788750000007,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "update_cue",
- "execMs": 0.020124999980907887
- },
+ "name": "align_to_audio",
+ "execMs": 0.006457999988924712
+ }
+ ]
+ },
+ {
+ "roundMs": 2533.3838749999995,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "update_cue",
- "execMs": 0.015416999987792224
+ "name": "get_state",
+ "execMs": 0.09083299996564165
}
]
},
{
- "roundMs": 2126.88629200001,
+ "roundMs": 1922.700708999997,
"firstDeltaMs": null,
"calls": [
{
- "name": "refine_speech_cuts",
- "execMs": 0.03599999996367842
+ "name": "delete_cue",
+ "execMs": 0.061999999976251274
+ }
+ ]
+ },
+ {
+ "roundMs": 1925.6313749999972,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.021999999997206032
}
]
},
{
- "roundMs": 3020.4231250000303,
- "firstDeltaMs": 2655.276332999987,
+ "roundMs": 2430.4580000000424,
+ "firstDeltaMs": 2075.1564160000416,
"calls": []
}
]
@@ -4516,222 +4066,311 @@
{
"pass": true,
"intent": "complex",
+ "skill": "watching-and-cutting",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "transitions",
+ "subtitles",
+ "audio",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 100,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "detect_silence",
+ "name": "find_filler",
"round": 1,
- "atMs": 3754.6000830000266,
- "execMs": 0.13462499994784594
+ "atMs": 4396.68987500004,
+ "execMs": 0.059084000007715076,
+ "args": {
+ "track": 0,
+ "clip_id": "c1"
+ }
},
{
- "name": "watch_video",
+ "name": "detect_silence",
"round": 2,
- "atMs": 6073.943415999995,
- "execMs": 0.019957999931648374
+ "atMs": 6149.900499999989,
+ "execMs": 0.040542000031564385,
+ "args": {
+ "clip_id": "c1"
+ }
},
{
- "name": "note_source",
+ "name": "watch_video",
"round": 3,
- "atMs": 8498.99700000009,
- "execMs": 0.038459000061266124
+ "atMs": 8156.194791000045,
+ "execMs": 0.017125000013038516,
+ "args": {
+ "clip_id": "c1",
+ "to": 6,
+ "interval_seconds": 1,
+ "from": 0
+ }
},
{
- "name": "split_at",
+ "name": "note_source",
"round": 4,
- "atMs": 10369.026458000066,
- "execMs": 0.2454169999109581
+ "atMs": 10120.79916600004,
+ "execMs": 0.017749999999068677,
+ "args": {
+ "asset_id": "a-v1",
+ "notes": [
+ {
+ "from": 0,
+ "text": "Woman at beach talking to camera with dogs in background",
+ "to": 6
+ }
+ ]
+ }
},
{
"name": "split_at",
"round": 5,
- "atMs": 12575.262958000065,
- "execMs": 0.040167000028304756
+ "atMs": 11832.032666000014,
+ "execMs": 0.05033300002105534,
+ "args": {
+ "t": 4.55
+ }
},
{
- "name": "delete_item",
+ "name": "split_at",
"round": 6,
- "atMs": 19651.047833000077,
- "execMs": 0.13429099996574223
+ "atMs": 13592.981583000044,
+ "execMs": 0.08833399997092783,
+ "args": {
+ "t": 5.6
+ }
},
{
- "name": "get_state",
+ "name": "trim_clip",
"round": 7,
- "atMs": 22539.452665999997,
- "execMs": 0.171875
+ "atMs": 15339.422750000027,
+ "execMs": 0.055333999975118786,
+ "args": {
+ "clipId": "c1-1",
+ "in": 0.45
+ }
},
{
- "name": "delete_cue",
+ "name": "delete_item",
"round": 8,
- "atMs": 24393.789457999985,
- "execMs": 0.08608400006778538
+ "atMs": 18217.488500000036,
+ "execMs": 0.022624999983236194,
+ "args": {
+ "kind": "clip",
+ "id": "c1-2-3"
+ }
},
{
- "name": "align_to_audio",
+ "name": "get_state",
"round": 9,
- "atMs": 27528.16279099998,
- "execMs": 0.001959000015631318
+ "atMs": 19943.600416,
+ "execMs": 0.0782500000204891,
+ "args": {}
},
{
- "name": "get_state",
+ "name": "refine_speech_cuts",
"round": 10,
- "atMs": 29366.58058300009,
- "execMs": 0.05816599994432181
+ "atMs": 21950.37174999999,
+ "execMs": 0.04920799995306879,
+ "args": {
+ "pace": "natural",
+ "clip_ids": [
+ "c1-1",
+ "c1-2-4"
+ ]
+ }
},
{
"name": "update_cue",
"round": 11,
- "atMs": 31508.675500000012,
- "execMs": 0.049915999989025295
+ "atMs": 23760.78470800002,
+ "execMs": 0.024000000033993274,
+ "args": {
+ "end": 1.35,
+ "start": 0,
+ "id": "cue1",
+ "text": "so today we're"
+ }
},
{
- "name": "update_cue",
+ "name": "get_state",
"round": 12,
- "atMs": 33691.084291000036,
- "execMs": 0.04766599996946752
+ "atMs": 28713.041333,
+ "execMs": 0.0811659999890253,
+ "args": {}
+ },
+ {
+ "name": "update_cue",
+ "round": 13,
+ "atMs": 30496.30125000002,
+ "execMs": 0.017583000008016825,
+ "args": {
+ "end": 4.1,
+ "start": 4.1,
+ "text": "and",
+ "id": "cue3"
+ }
}
],
"timings": {
- "gateMs": 1891.345874999999,
+ "gateMs": 289.78583300003083,
"gateSkipped": false,
- "ttftMs": 35656.935666000005,
- "modelMs": 35929.187540000305,
- "toolMs": 1.0285829998319969,
- "totalMs": 35960.211291000014,
- "judgeMs": 1020.4677500000689,
+ "ttftMs": 32285.455958000035,
+ "modelMs": 32237.16462699999,
+ "toolMs": 0.6013340000063181,
+ "totalMs": 32549.717833000002,
+ "judgeMs": 500.9048329999787,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1890.14537500008,
+ "roundMs": 1260.811875000014,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 1858.8560410001082,
+ "roundMs": 2842.517625000037,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "find_filler",
+ "execMs": 0.059084000007715076
+ }
+ ]
+ },
+ {
+ "roundMs": 1751.425709000032,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.13462499994784594
+ "execMs": 0.040542000031564385
}
]
},
{
- "roundMs": 2316.3431660000933,
+ "roundMs": 2004.4235419999459,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.019957999931648374
+ "execMs": 0.017125000013038516
}
]
},
{
- "roundMs": 2423.60079200007,
+ "roundMs": 1963.215749999974,
"firstDeltaMs": null,
"calls": [
{
"name": "note_source",
- "execMs": 0.038459000061266124
+ "execMs": 0.017749999999068677
}
]
},
{
- "roundMs": 1867.5900830000173,
+ "roundMs": 1710.0438339999528,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.2454169999109581
+ "execMs": 0.05033300002105534
}
]
},
{
- "roundMs": 2203.1162919999333,
+ "roundMs": 1759.844959000009,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.040167000028304756
+ "execMs": 0.08833399997092783
}
]
},
{
- "roundMs": 7074.600957999937,
+ "roundMs": 1744.530040999991,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.13429099996574223
+ "name": "trim_clip",
+ "execMs": 0.055333999975118786
}
]
},
{
- "roundMs": 2884.946083000046,
+ "roundMs": 2876.2964999999967,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.171875
+ "name": "delete_item",
+ "execMs": 0.022624999983236194
}
]
},
{
- "roundMs": 1851.5185839999467,
+ "roundMs": 1725.439167000004,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_cue",
- "execMs": 0.08608400006778538
+ "name": "get_state",
+ "execMs": 0.0782500000204891
}
]
},
{
- "roundMs": 3131.4597080000676,
+ "roundMs": 2004.9705840000533,
"firstDeltaMs": null,
"calls": [
{
- "name": "align_to_audio",
- "execMs": 0.001959000015631318
+ "name": "refine_speech_cuts",
+ "execMs": 0.04920799995306879
}
]
},
{
- "roundMs": 1837.4747500000522,
+ "roundMs": 1808.8036250000005,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.05816599994432181
+ "name": "update_cue",
+ "execMs": 0.024000000033993274
}
]
},
{
- "roundMs": 2140.5964580000145,
+ "roundMs": 4950.779749999987,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.049915999989025295
+ "name": "get_state",
+ "execMs": 0.0811659999890253
}
]
},
{
- "roundMs": 2181.292541999952,
+ "roundMs": 1781.9015409999993,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.04766599996946752
+ "execMs": 0.017583000008016825
}
]
},
{
- "roundMs": 2267.6467079999857,
- "firstDeltaMs": 1964.9120409999741,
+ "roundMs": 2052.160124999995,
+ "firstDeltaMs": 1788.3024160000496,
"calls": []
}
]
@@ -4746,34 +4385,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1077,
- "p95": 1608,
- "mean": 1242,
- "n": 3
+ "p50": 269,
+ "p95": 291,
+ "mean": 280,
+ "n": 2
},
"ttftMs": {
- "p50": 4358,
- "p95": 7344,
- "mean": 5269,
- "n": 3
+ "p50": 2915,
+ "p95": 5602,
+ "mean": 4258,
+ "n": 2
},
"totalMs": {
- "p50": 4954,
- "p95": 7959,
- "mean": 5862,
- "n": 3
+ "p50": 2930,
+ "p95": 5753,
+ "mean": 4342,
+ "n": 2
},
"modelMs": {
- "p50": 4948,
- "p95": 7951,
- "mean": 5857,
- "n": 3
+ "p50": 2637,
+ "p95": 5480,
+ "mean": 4059,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -4784,49 +4423,19 @@
{
"pass": true,
"intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": "failed to state what changed in concrete terms (e.g., specifying the duration or style of captions).",
- "trace": [
- {
- "name": "subtitles_generate",
- "round": 0,
- "atMs": 5559.232958000037,
- "execMs": 0.020790999988093972
- }
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
],
- "timings": {
- "gateMs": 1076.5102920000209,
- "gateSkipped": false,
- "ttftMs": 7343.600541000022,
- "modelMs": 7951.065792000038,
- "toolMs": 0.020790999988093972,
- "totalMs": 7959.060833000112,
- "judgeMs": 1199.0992090000072,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 5553.572334000026,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "subtitles_generate",
- "execMs": 0.020790999988093972
- }
- ]
- },
- {
- "roundMs": 2397.4934580000117,
- "firstDeltaMs": 1782.182291999925,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
+ "declaredTools": 96,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -4834,34 +4443,37 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 2784.1212499999674,
- "execMs": 0.023625000030733645
+ "atMs": 4294.591166999948,
+ "execMs": 0.00483300001360476,
+ "args": {
+ "track": 0
+ }
}
],
"timings": {
- "gateMs": 1041.9306660000002,
+ "gateMs": 269.41512499999953,
"gateSkipped": false,
- "ttftMs": 4357.628541999962,
- "modelMs": 4948.300123999943,
- "toolMs": 0.023625000030733645,
- "totalMs": 4954.147833999945,
- "judgeMs": 1569.7235409999266,
+ "ttftMs": 5601.570207999961,
+ "modelMs": 5480.3979999999865,
+ "toolMs": 0.00483300001360476,
+ "totalMs": 5753.372499999998,
+ "judgeMs": 520.054833000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2780.5099579999223,
+ "roundMs": 4022.840124999988,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.023625000030733645
+ "execMs": 0.00483300001360476
}
]
},
{
- "roundMs": 2167.7901660000207,
- "firstDeltaMs": 1571.343332999968,
+ "roundMs": 1457.5578749999986,
+ "firstDeltaMs": 1306.034290999989,
"calls": []
}
]
@@ -4870,41 +4482,57 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 96,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": "the assistant did not confirm what specific action was taken or what the current status of the project is (i.e., whether the captions were added to the timeline).",
+ "judgeNote": null,
"trace": [
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 2534.64383299998,
- "execMs": 0.00145800004247576
+ "atMs": 1606.8932919999934,
+ "execMs": 0.006709000037517399,
+ "args": {
+ "track": 0
+ }
}
],
"timings": {
- "gateMs": 1607.5515419999138,
+ "gateMs": 290.6009159999667,
"gateSkipped": false,
- "ttftMs": 4104.592791000032,
- "modelMs": 4670.894874999882,
- "toolMs": 0.00145800004247576,
- "totalMs": 4673.431582999998,
- "judgeMs": 1900.3200420000358,
+ "ttftMs": 2914.5904579999624,
+ "modelMs": 2636.953916999977,
+ "toolMs": 0.006709000037517399,
+ "totalMs": 2929.628708000004,
+ "judgeMs": 228.50675000000047,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2532.9985829999205,
+ "roundMs": 1314.8173749999842,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.00145800004247576
+ "execMs": 0.006709000037517399
}
]
},
{
- "roundMs": 2137.8962919999612,
- "firstDeltaMs": 1569.1106669999426,
+ "roundMs": 1322.136541999993,
+ "firstDeltaMs": 1307.2522920000483,
"calls": []
}
]
@@ -4919,132 +4547,132 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1100,
- "p95": 1578,
- "mean": 1216,
- "n": 3
+ "p50": 275,
+ "p95": 470,
+ "mean": 373,
+ "n": 2
},
"ttftMs": {
- "p50": 6349,
- "p95": 15353,
- "mean": 8856,
- "n": 3
+ "p50": 7286,
+ "p95": 23363,
+ "mean": 15325,
+ "n": 2
},
"totalMs": {
- "p50": 6713,
- "p95": 15609,
- "mean": 9155,
- "n": 3
+ "p50": 7461,
+ "p95": 23869,
+ "mean": 15665,
+ "n": 2
},
"modelMs": {
- "p50": 6708,
- "p95": 15603,
- "mean": 9151,
- "n": 3
+ "p50": 7051,
+ "p95": 23588,
+ "mean": 15319,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 3,
- "max": 3
+ "mean": 4,
+ "max": 4
}
},
"runs": [
{
"pass": true,
"intent": "complex",
+ "skill": "scene-productions",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 127,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "listen_audio",
"round": 1,
- "atMs": 4081.477083000005,
- "execMs": 0.31404199998360127
+ "atMs": 4857.906708000053,
+ "execMs": 0.07583400001749396,
+ "args": {
+ "asset_id": "a-au1"
+ }
+ },
+ {
+ "name": "generate_scene",
+ "round": 2,
+ "atMs": 7902.72316600004,
+ "execMs": 0.08183300000382587,
+ "args": {
+ "style": "Smooth 2D cartoon animation, hand-drawn vector aesthetic, clean lines, vibrant palette, fluid character animation",
+ "brief": "A smooth, fluid 2D hand-drawn animated cartoon illustrating the narration: clean bold outlines, vibrant flat colors, rounded friendly character shapes, continuous animated movement.",
+ "from_audio_asset_id": "a-au1"
+ }
}
],
"timings": {
- "gateMs": 1577.749666000018,
+ "gateMs": 274.9295830000192,
"gateSkipped": false,
- "ttftMs": 6348.967208000016,
- "modelMs": 6707.541334000067,
- "toolMs": 0.31404199998360127,
- "totalMs": 6712.647791999974,
- "judgeMs": 1044.9286250000587,
+ "ttftMs": 23363.34275000001,
+ "modelMs": 23587.79954300006,
+ "toolMs": 0.15766700002131984,
+ "totalMs": 23869.057291000034,
+ "judgeMs": 579.0723330000183,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1577.137875000015,
+ "roundMs": 1727.5787920000148,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 2500.963333999971,
+ "roundMs": 2850.7872920000227,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.31404199998360127
+ "name": "listen_audio",
+ "execMs": 0.07583400001749396
}
]
},
{
- "roundMs": 2629.440125000081,
- "firstDeltaMs": 2265.909459000104,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "generate_scene",
- "round": 1,
- "atMs": 3142.0275830000173,
- "execMs": 0.004167000064626336
- }
- ],
- "timings": {
- "gateMs": 970.7010000000009,
- "gateSkipped": false,
- "ttftMs": 4864.098999999929,
- "modelMs": 5141.384501000051,
- "toolMs": 0.004167000064626336,
- "totalMs": 5144.465749999974,
- "judgeMs": 1280.29770799994,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 970.0226249999832,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2169.5587090000045,
+ "roundMs": 3043.916917000024,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.004167000064626336
+ "execMs": 0.08183300000382587
}
]
},
{
- "roundMs": 2001.8031670000637,
- "firstDeltaMs": 1721.5061670000432,
+ "roundMs": 15965.516541999998,
+ "firstDeltaMs": 15459.97320800001,
"calls": []
}
]
@@ -5053,46 +4681,94 @@
{
"pass": true,
"intent": "complex",
+ "skill": "scene-productions",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 127,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "listen_audio",
"round": 1,
- "atMs": 13302.911958999932,
- "execMs": 0.00783299992326647
+ "atMs": 2976.7070419999654,
+ "execMs": 0.0313750000204891,
+ "args": {
+ "asset_id": "a-au1"
+ }
+ },
+ {
+ "name": "generate_scene",
+ "round": 2,
+ "atMs": 5711.230125000002,
+ "execMs": 0.005917000002227724,
+ "args": {
+ "from_audio_asset_id": "a-au1",
+ "brief": "A charming story following the narration, animated as smooth 2D cartoon animation with clean ink line art, soft vibrant colors, fluid hand-drawn motion, and consistent character design.",
+ "style": "Smooth 2D cartoon animation, hand-drawn vector art style, clean linework, vibrant flat colors, fluid motion"
+ }
}
],
"timings": {
- "gateMs": 1099.7334170000395,
+ "gateMs": 470.4172920000274,
"gateSkipped": false,
- "ttftMs": 15353.481416999944,
- "modelMs": 15603.33262500004,
- "toolMs": 0.00783299992326647,
- "totalMs": 15609.21887500002,
- "judgeMs": 1041.5517080000136,
+ "ttftMs": 7285.673084000009,
+ "modelMs": 7050.987667999987,
+ "toolMs": 0.03729200002271682,
+ "totalMs": 7460.546999999962,
+ "judgeMs": 555.2023749999935,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1099.2170840000035,
+ "roundMs": 66.9012090000324,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 12199.379666000023,
+ "roundMs": 2504.117499999993,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.0313750000204891
+ }
+ ]
+ },
+ {
+ "roundMs": 2732.1188749999856,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.00783299992326647
+ "execMs": 0.005917000002227724
}
]
},
{
- "roundMs": 2304.735875000013,
- "firstDeltaMs": 2049.1084580000024,
+ "roundMs": 1747.850083999976,
+ "firstDeltaMs": 1573.3913339999854,
"calls": []
}
]
@@ -5107,132 +4783,132 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1163,
- "p95": 1473,
- "mean": 1176,
- "n": 3
+ "p50": 319,
+ "p95": 320,
+ "mean": 319,
+ "n": 2
},
"ttftMs": {
- "p50": 8098,
- "p95": 16793,
- "mean": 10812,
- "n": 3
+ "p50": 11762,
+ "p95": 12573,
+ "mean": 12168,
+ "n": 2
},
"totalMs": {
- "p50": 8749,
- "p95": 17462,
- "mean": 11505,
- "n": 3
+ "p50": 12041,
+ "p95": 12908,
+ "mean": 12474,
+ "n": 2
},
"modelMs": {
- "p50": 8743,
- "p95": 17456,
- "mean": 11499,
- "n": 3
+ "p50": 11714,
+ "p95": 12580,
+ "mean": 12147,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 3,
- "max": 3
+ "mean": 4,
+ "max": 4
}
},
"runs": [
{
"pass": true,
"intent": "complex",
+ "skill": "scene-productions",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 120,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "listen_audio",
"round": 1,
- "atMs": 4696.540167000028,
- "execMs": 0.35895799996796995
+ "atMs": 4272.4063750000205,
+ "execMs": 0.009041999990586191,
+ "args": {
+ "asset_id": "a-au1"
+ }
+ },
+ {
+ "name": "generate_scene",
+ "round": 2,
+ "atMs": 7042.81270900002,
+ "execMs": 0.3398750000051223,
+ "args": {
+ "audio_language": "ko-KR",
+ "from_audio_asset_id": "a-au1",
+ "style": "smooth 2D cartoon animation, clean line art, vibrant pastel colors, fluid motion",
+ "brief": "A smooth 2D cartoon animation visualizing the Korean narration about a dog's adventure in a park and beach. Clean lines, charming rounded 2D animated characters, soft vibrant colors, fluid cartoon motion."
+ }
}
],
"timings": {
- "gateMs": 1162.860374999931,
+ "gateMs": 319.008833000029,
"gateSkipped": false,
- "ttftMs": 7545.211208000081,
- "modelMs": 8299.588374999934,
- "toolMs": 0.35895799996796995,
- "totalMs": 8304.802375000087,
- "judgeMs": 1779.2250829999102,
+ "ttftMs": 11761.898417000019,
+ "modelMs": 11713.877665999986,
+ "toolMs": 0.34891699999570847,
+ "totalMs": 12041.062666999991,
+ "judgeMs": 539.3162920000032,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1161.9898329999996,
+ "roundMs": 1689.5204999999842,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 3531.1959169999463,
+ "roundMs": 2259.5719999999856,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.35895799996796995
+ "name": "listen_audio",
+ "execMs": 0.009041999990586191
}
]
},
{
- "roundMs": 3606.402624999988,
- "firstDeltaMs": 2847.1645410000347,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "generate_scene",
- "round": 1,
- "atMs": 5925.647832999937,
- "execMs": 0.01324999995995313
- }
- ],
- "timings": {
- "gateMs": 893.6855000000214,
- "gateSkipped": false,
- "ttftMs": 16793.49883299996,
- "modelMs": 17455.906167000066,
- "toolMs": 0.01324999995995313,
- "totalMs": 17461.872875,
- "judgeMs": 1277.7486250000075,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 893.1205000000773,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 5028.361959000002,
+ "roundMs": 2769.332875000022,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.01324999995995313
+ "execMs": 0.3398750000051223
}
]
},
{
- "roundMs": 11534.423707999988,
- "firstDeltaMs": 10866.617749999976,
+ "roundMs": 4995.452290999994,
+ "firstDeltaMs": 4716.576457999996,
"calls": []
}
]
@@ -5241,46 +4917,94 @@
{
"pass": true,
"intent": "complex",
+ "skill": "scene-productions",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 120,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "listen_audio",
"round": 1,
- "atMs": 4456.326165999984,
- "execMs": 0.010833000065758824
+ "atMs": 6614.068084000028,
+ "execMs": 0.019958999997470528,
+ "args": {
+ "asset_id": "a-au1"
+ }
+ },
+ {
+ "name": "generate_scene",
+ "round": 2,
+ "atMs": 8982.310875000025,
+ "execMs": 0.013042000005953014,
+ "args": {
+ "style": "smooth 2D cartoon animation, clean line art, vibrant colors, fluid character motion",
+ "from_audio_asset_id": "a-au1",
+ "brief": "A smooth 2D cartoon animation depicting the story narrated in the audio, featuring expressive characters, clean linework, vibrant colors, and fluid motion.",
+ "audio_language": "ko-KR"
+ }
}
],
"timings": {
- "gateMs": 1472.5850840000203,
+ "gateMs": 319.92545899999095,
"gateSkipped": false,
- "ttftMs": 8098.270583000034,
- "modelMs": 8742.547458000015,
- "toolMs": 0.010833000065758824,
- "totalMs": 8748.81820800004,
- "judgeMs": 917.0955829999875,
+ "ttftMs": 12573.294542000047,
+ "modelMs": 12580.106417999952,
+ "toolMs": 0.03300100000342354,
+ "totalMs": 12907.581750000012,
+ "judgeMs": 569.1341249999823,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1473.7681249999441,
+ "roundMs": 1661.9575839999598,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 2977.2100830000127,
+ "roundMs": 4627.9814590000315,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.019958999997470528
+ }
+ ]
+ },
+ {
+ "roundMs": 2366.469333999965,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.010833000065758824
+ "execMs": 0.013042000005953014
}
]
},
{
- "roundMs": 4291.569250000059,
- "firstDeltaMs": 3641.26095799997,
+ "roundMs": 3923.698040999996,
+ "firstDeltaMs": 3589.924874999968,
"calls": []
}
]
@@ -5295,122 +5019,129 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
+ "p50": 252,
+ "p95": 260,
+ "mean": 256,
+ "n": 2
},
"ttftMs": {
- "p50": 6892,
- "p95": 16081,
- "mean": 9198,
- "n": 3
+ "p50": 5897,
+ "p95": 8931,
+ "mean": 7414,
+ "n": 2
},
"totalMs": {
- "p50": 7136,
- "p95": 16433,
- "mean": 9464,
- "n": 3
+ "p50": 6194,
+ "p95": 9198,
+ "mean": 7696,
+ "n": 2
},
"modelMs": {
- "p50": 7127,
- "p95": 16427,
- "mean": 9457,
- "n": 3
+ "p50": 5928,
+ "p95": 8937,
+ "mean": 7432,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 3
}
},
"runs": [
{
"pass": true,
"intent": "complex",
+ "skill": "scene-productions",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 127,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "listen_audio",
"round": 0,
- "atMs": 4710.818583999993,
- "execMs": 0.22883300005923957
+ "atMs": 3098.703125,
+ "execMs": 0.04220800002804026,
+ "args": {
+ "asset_id": "a-au1"
+ }
+ },
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 6774.4283339999965,
+ "execMs": 0.1091249999590218,
+ "args": {
+ "style": "flat hand-drawn 2D cartoon animation, colorful storybook illustration style, clean vibrant outlines, lively fluid motion",
+ "from_audio_asset_id": "a-au1",
+ "reference_asset_ids": [
+ "a-i1"
+ ]
+ }
}
],
"timings": {
- "gateMs": 0.2376660000300035,
+ "gateMs": 251.5820839999942,
"gateSkipped": true,
- "ttftMs": 6892.419208999956,
- "modelMs": 7127.323667000164,
- "toolMs": 0.22883300005923957,
- "totalMs": 7136.454499999993,
- "judgeMs": 1145.267790999962,
+ "ttftMs": 8930.976375000027,
+ "modelMs": 8936.575916999951,
+ "toolMs": 0.15133299998706207,
+ "totalMs": 9198.011667000013,
+ "judgeMs": 587.8315410000505,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4706.737000000081,
+ "roundMs": 2842.6163329999545,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.22883300005923957
+ "name": "listen_audio",
+ "execMs": 0.04220800002804026
}
]
},
{
- "roundMs": 2420.5866670000833,
- "firstDeltaMs": 2177.1685419999994,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "generate_scene",
- "round": 0,
- "atMs": 6332.369415999972,
- "execMs": 0.01875000004656613
- }
- ],
- "timings": {
- "gateMs": 0.4041669999714941,
- "gateSkipped": true,
- "ttftMs": 16080.593915999983,
- "modelMs": 16426.877125,
- "toolMs": 0.01875000004656613,
- "totalMs": 16432.639707999886,
- "judgeMs": 1386.7852500000736,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 6328.900290999911,
+ "roundMs": 3672.175458999991,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.01875000004656613
+ "execMs": 0.1091249999590218
}
]
},
{
- "roundMs": 10097.976834000088,
- "firstDeltaMs": 9746.562667000107,
+ "roundMs": 2421.7841250000056,
+ "firstDeltaMs": 2154.885249999992,
"calls": []
}
]
@@ -5419,41 +5150,91 @@
{
"pass": true,
"intent": "complex",
+ "skill": "scene-productions",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 127,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "listen_audio",
"round": 0,
- "atMs": 2745.4742919999408,
- "execMs": 0.013000000035390258
+ "atMs": 1975.7431670000078,
+ "execMs": 0.006290999997872859,
+ "args": {
+ "asset_id": "a-au1"
+ }
+ },
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 4181.843500000017,
+ "execMs": 0.011375000001862645,
+ "args": {
+ "style": "vibrant 2D cartoon animation style matching the attached illustration",
+ "reference_asset_ids": [
+ "a-i1"
+ ],
+ "from_audio_asset_id": "a-au1"
+ }
}
],
"timings": {
- "gateMs": 0.34895799995865673,
+ "gateMs": 260.4637910000165,
"gateSkipped": true,
- "ttftMs": 4621.055333000026,
- "modelMs": 4816.262623999966,
- "toolMs": 0.013000000035390258,
- "totalMs": 4822.712167000049,
- "judgeMs": 1995.059917000006,
+ "ttftMs": 5897.35820800002,
+ "modelMs": 5928.399665999983,
+ "toolMs": 0.017665999999735504,
+ "totalMs": 6194.27816700004,
+ "judgeMs": 574.2261669999571,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2740.9132080000127,
+ "roundMs": 1712.2283329999773,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.006290999997872859
+ }
+ ]
+ },
+ {
+ "roundMs": 2205.2085829999996,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.013000000035390258
+ "execMs": 0.011375000001862645
}
]
},
{
- "roundMs": 2075.3494159999536,
- "firstDeltaMs": 1873.7792909999844,
+ "roundMs": 2010.962750000006,
+ "firstDeltaMs": 1714.131457999989,
"calls": []
}
]
@@ -5468,34 +5249,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1116,
- "p95": 1766,
- "mean": 1325,
- "n": 3
+ "p50": 210,
+ "p95": 246,
+ "mean": 228,
+ "n": 2
},
"ttftMs": {
- "p50": 3323,
- "p95": 4501,
- "mean": 3648,
- "n": 3
+ "p50": 3343,
+ "p95": 6764,
+ "mean": 5053,
+ "n": 2
},
"totalMs": {
- "p50": 4116,
- "p95": 6309,
- "mean": 4817,
- "n": 3
+ "p50": 4215,
+ "p95": 8147,
+ "mean": 6181,
+ "n": 2
},
"modelMs": {
- "p50": 4111,
- "p95": 6304,
- "mean": 4811,
- "n": 3
+ "p50": 4001,
+ "p95": 7895,
+ "mean": 5948,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -5506,29 +5287,52 @@
{
"pass": true,
"intent": "complex",
+ "skill": "scene-productions",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 120,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": null,
+ "judgeNote": "concise factual 0.46; long 2.0",
"trace": [],
"timings": {
- "gateMs": 1115.9496670000954,
+ "gateMs": 246.42304099997273,
"gateSkipped": false,
- "ttftMs": 3323.354583999957,
- "modelMs": 4020.126667000004,
+ "ttftMs": 6763.740624999977,
+ "modelMs": 7895.323125000054,
"toolMs": 0,
- "totalMs": 4025.639124999987,
- "judgeMs": 1217.7737079999642,
+ "totalMs": 8147.037375000014,
+ "judgeMs": 486.8101669999887,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1114.0416250000708,
+ "roundMs": 4289.968167000043,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 2906.0850419999333,
- "firstDeltaMs": 2204.075124999974,
+ "roundMs": 3605.354958000011,
+ "firstDeltaMs": 2222.187957999995,
"calls": []
}
]
@@ -5537,60 +5341,52 @@
{
"pass": true,
"intent": "complex",
+ "skill": "scene-productions",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "removal",
+ "elements",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 120,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1765.5236659999937,
+ "gateMs": 210.49154099996667,
"gateSkipped": false,
- "ttftMs": 4501.251999999979,
- "modelMs": 6303.563374999911,
+ "ttftMs": 3342.778667000006,
+ "modelMs": 4000.9623339999234,
"toolMs": 0,
- "totalMs": 6308.628125000047,
- "judgeMs": 1630.3350420000497,
+ "totalMs": 4215.245208000008,
+ "judgeMs": 598.4915000000037,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1763.324499999988,
+ "roundMs": 1541.7603749999544,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 4540.238874999923,
- "firstDeltaMs": 2733.175249999971,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": "self-praise (\"stunning\" implied in descriptive flow) and includes an unrequested, wordy explanation of the tool's process.",
- "trace": [],
- "timings": {
- "gateMs": 1093.0182499999646,
- "gateSkipped": false,
- "ttftMs": 3118.7219999999506,
- "modelMs": 4110.685000000056,
- "toolMs": 0,
- "totalMs": 4115.643791999901,
- "judgeMs": 1247.2138330000453,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1091.218375000055,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 3019.466625000001,
- "firstDeltaMs": 2022.7906669999938,
+ "roundMs": 2459.201958999969,
+ "firstDeltaMs": 1587.3116670000018,
"calls": []
}
]
@@ -5605,44 +5401,61 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
+ "p50": 245,
+ "p95": 256,
+ "mean": 251,
+ "n": 2
},
"ttftMs": {
- "p50": 11012,
- "p95": 19160,
- "mean": 12910,
- "n": 3
+ "p50": 19034,
+ "p95": 26135,
+ "mean": 22584,
+ "n": 2
},
"totalMs": {
- "p50": 11340,
- "p95": 19843,
- "mean": 13662,
- "n": 3
+ "p50": 19271,
+ "p95": 26690,
+ "mean": 22980,
+ "n": 2
},
"modelMs": {
- "p50": 11317,
- "p95": 19824,
- "mean": 13645,
- "n": 3
+ "p50": 19020,
+ "p95": 26415,
+ "mean": 22717,
+ "n": 2
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 4.7,
- "max": 5
+ "mean": 7.5,
+ "max": 10
}
},
"runs": [
{
"pass": true,
"intent": "complex",
+ "skill": "audio-and-subtitles",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "transitions",
+ "subtitles",
+ "audio",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "editor"
+ ],
+ "declaredTools": 106,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
@@ -5650,243 +5463,148 @@
{
"name": "listen_audio",
"round": 0,
- "atMs": 2326.3417500000214,
- "execMs": 0.44516700005624443
+ "atMs": 2614.5971659999923,
+ "execMs": 0.10375000000931323,
+ "args": {
+ "to": 6,
+ "clip_id": "au-n",
+ "from": 0
+ }
},
{
"name": "watch_video",
- "round": 1,
- "atMs": 4285.465250000008,
- "execMs": 0.029792000073939562
+ "round": 0,
+ "atMs": 2614.805083000043,
+ "execMs": 0.002582999994046986,
+ "args": {
+ "clip_id": "sc-1"
+ }
},
{
"name": "watch_video",
- "round": 1,
- "atMs": 4285.989458000055,
- "execMs": 0.00712500000372529
- },
- {
- "name": "note_source",
- "round": 2,
- "atMs": 6627.878000000026,
- "execMs": 0.013916000025346875
+ "round": 0,
+ "atMs": 2614.8421250000247,
+ "execMs": 0.0005830000154674053,
+ "args": {
+ "clip_id": "sc-2"
+ }
},
{
"name": "note_source",
- "round": 2,
- "atMs": 6628.18529100006,
- "execMs": 0.003874999936670065
- },
- {
- "name": "regenerate_shot",
- "round": 3,
- "atMs": 9115.618875000044,
- "execMs": 0.02212500001769513
- },
- {
- "name": "regenerate_shot",
- "round": 3,
- "atMs": 9116.135666000075,
- "execMs": 0.010083999950438738
- }
- ],
- "timings": {
- "gateMs": 0.2022919999435544,
- "gateSkipped": true,
- "ttftMs": 11012.474916000036,
- "modelMs": 11317.323916000081,
- "toolMs": 0.5320840000640601,
- "totalMs": 11339.858958000084,
- "judgeMs": 1887.7754999999888,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2321.46562500007,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.44516700005624443
- }
- ]
- },
- {
- "roundMs": 1956.2378749999916,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.029792000073939562
- },
- {
- "name": "watch_video",
- "execMs": 0.00712500000372529
- }
- ]
- },
- {
- "roundMs": 2338.9965000000084,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "note_source",
- "execMs": 0.013916000025346875
- },
- {
- "name": "note_source",
- "execMs": 0.003874999936670065
- }
- ]
- },
- {
- "roundMs": 2479.031707999995,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "regenerate_shot",
- "execMs": 0.02212500001769513
- },
+ "round": 1,
+ "atMs": 5099.87737500004,
+ "execMs": 0.0039160000160336494,
+ "args": {
+ "clip_id": "sc-1",
+ "notes": [
{
- "name": "regenerate_shot",
- "execMs": 0.010083999950438738
+ "text": "Shot 1 of generated scene: dog running in park/field",
+ "to": 3,
+ "from": 0
}
]
- },
- {
- "roundMs": 2221.5922080000164,
- "firstDeltaMs": 1894.3229999999749,
- "calls": []
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 2718.871916999924,
- "execMs": 0.013666999991983175
- },
- {
- "name": "watch_video",
- "round": 1,
- "atMs": 4655.962416999973,
- "execMs": 0.014041999937035143
- },
- {
- "name": "watch_video",
- "round": 1,
- "atMs": 4656.1257500000065,
- "execMs": 0.00254100002348423
- },
- {
- "name": "note_source",
- "round": 2,
- "atMs": 14936.513791999896,
- "execMs": 0.018124999944120646
},
{
"name": "note_source",
- "round": 2,
- "atMs": 14936.845666999929,
- "execMs": 0.005292000016197562
+ "round": 1,
+ "atMs": 5099.935500000021,
+ "execMs": 0.0006249999860301614,
+ "args": {
+ "clip_id": "sc-2",
+ "notes": [
+ {
+ "to": 3,
+ "text": "Shot 2 of generated scene: very similar angle of dog running in park/field",
+ "from": 0
+ }
+ ]
+ }
},
{
- "name": "read_skill",
+ "name": "get_state",
"round": 2,
- "atMs": 14939.280416999944,
- "execMs": 0.016959000029601157
- },
- {
- "name": "regenerate_shot",
- "round": 3,
- "atMs": 17546.57899999991,
- "execMs": 0.027750000008381903
+ "atMs": 7577.166541000013,
+ "execMs": 0.008750000037252903,
+ "args": {}
},
{
"name": "regenerate_shot",
"round": 3,
- "atMs": 17547.048124999972,
- "execMs": 0.004667000030167401
+ "atMs": 10322.591208000027,
+ "execMs": 0.009959000046364963,
+ "args": {
+ "note": "Close-up reaction shot that matches the narration",
+ "n": 2
+ }
}
],
"timings": {
- "gateMs": 0.3606669999426231,
+ "gateMs": 245.41316700004973,
"gateSkipped": true,
- "ttftMs": 19159.965124999988,
- "modelMs": 19824.339418000076,
- "toolMs": 0.10304299998097122,
- "totalMs": 19843.275334000005,
- "judgeMs": 965.2413749999832,
+ "ttftMs": 19034.05487500003,
+ "modelMs": 19020.037291000015,
+ "toolMs": 0.1301660001045093,
+ "totalMs": 19271.390000000014,
+ "judgeMs": 577.3008750000154,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2714.844875000068,
+ "roundMs": 2368.2152079999796,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.013666999991983175
- }
- ]
- },
- {
- "roundMs": 1934.1415839999681,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.10375000000931323
+ },
{
"name": "watch_video",
- "execMs": 0.014041999937035143
+ "execMs": 0.002582999994046986
},
{
"name": "watch_video",
- "execMs": 0.00254100002348423
+ "execMs": 0.0005830000154674053
}
]
},
{
- "roundMs": 10278.842749999953,
+ "roundMs": 2484.524792000011,
"firstDeltaMs": null,
"calls": [
{
"name": "note_source",
- "execMs": 0.018124999944120646
+ "execMs": 0.0039160000160336494
},
{
"name": "note_source",
- "execMs": 0.005292000016197562
- },
- {
- "name": "read_skill",
- "execMs": 0.016959000029601157
+ "execMs": 0.0006249999860301614
}
]
},
{
- "roundMs": 2603.5681670000777,
+ "roundMs": 2476.674291000003,
"firstDeltaMs": null,
"calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.027750000008381903
- },
+ "name": "get_state",
+ "execMs": 0.008750000037252903
+ }
+ ]
+ },
+ {
+ "roundMs": 2743.2271670000046,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "regenerate_shot",
- "execMs": 0.004667000030167401
+ "execMs": 0.009959000046364963
}
]
},
{
- "roundMs": 2292.9420420000097,
- "firstDeltaMs": 1609.9613750000717,
+ "roundMs": 8947.395833000017,
+ "firstDeltaMs": 8710.480333000014,
"calls": []
}
]
@@ -5895,6 +5613,24 @@
{
"pass": true,
"intent": "complex",
+ "skill": "audio-and-subtitles",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "transitions",
+ "subtitles",
+ "audio",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "editor"
+ ],
+ "declaredTools": 107,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
@@ -5902,106 +5638,207 @@
{
"name": "listen_audio",
"round": 0,
- "atMs": 2224.036375000025,
- "execMs": 0.008000000030733645
+ "atMs": 7032.2287079999805,
+ "execMs": 0.01087499997811392,
+ "args": {
+ "from": 0,
+ "asset_id": "a-au1",
+ "to": 6
+ }
},
{
"name": "watch_video",
- "round": 0,
- "atMs": 2224.119041000027,
- "execMs": 0.0007919999770820141
+ "round": 1,
+ "atMs": 9419.263541999971,
+ "execMs": 0.010249999992083758,
+ "args": {
+ "clip_id": "sc-1"
+ }
+ },
+ {
+ "name": "note_source",
+ "round": 2,
+ "atMs": 12498.851542000019,
+ "execMs": 0.016582999960519373,
+ "args": {
+ "clip_id": "sc-1",
+ "notes": [
+ {
+ "from": 0,
+ "to": 3,
+ "text": "Close-up shot of a golden retriever puppy looking at the camera on grass"
+ }
+ ]
+ }
},
{
"name": "watch_video",
- "round": 0,
- "atMs": 2224.136458000052,
- "execMs": 0.0004160000244155526
+ "round": 3,
+ "atMs": 14469.211249999993,
+ "execMs": 0.016583000018727034,
+ "args": {
+ "clip_id": "sc-2"
+ }
},
{
"name": "note_source",
- "round": 1,
- "atMs": 4695.270083000069,
- "execMs": 0.017374999937601388
+ "round": 4,
+ "atMs": 16545.08166700002,
+ "execMs": 0.01554200000828132,
+ "args": {
+ "notes": [
+ {
+ "to": 3,
+ "text": "Close-up shot of the same golden retriever puppy sitting in the grass looking at the camera",
+ "from": 0
+ }
+ ],
+ "clip_id": "sc-2"
+ }
},
{
- "name": "note_source",
- "round": 1,
- "atMs": 4695.574000000022,
- "execMs": 0.003459000028669834
+ "name": "get_state",
+ "round": 5,
+ "atMs": 18451.339666999993,
+ "execMs": 0.0050830000545829535,
+ "args": {}
+ },
+ {
+ "name": "detect_silence",
+ "round": 6,
+ "atMs": 20426.178917000012,
+ "execMs": 0.02245799999218434,
+ "args": {
+ "asset_id": "a-au1",
+ "from": 0,
+ "to": 9
+ }
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 6921.36491600005,
- "execMs": 0.014583999989554286
+ "round": 7,
+ "atMs": 22340.80920800002,
+ "execMs": 0.028541000036057085,
+ "args": {
+ "note": "Golden retriever puppy running through the park playing fetch",
+ "n": 1
+ }
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 6921.718790999963,
- "execMs": 0.006958000012673438
+ "round": 8,
+ "atMs": 24551.809917000006,
+ "execMs": 0.025666999979875982,
+ "args": {
+ "note": "Golden retriever puppy running along the beach splashing in the shallow water",
+ "n": 2
+ }
}
],
"timings": {
- "gateMs": 0.24262499995529652,
+ "gateMs": 255.9873329999973,
"gateSkipped": true,
- "ttftMs": 8558.923375000013,
- "modelMs": 9792.947293000063,
- "toolMs": 0.05158400000073016,
- "totalMs": 9802.158458000049,
- "judgeMs": 1079.816208000062,
+ "ttftMs": 26134.942541999975,
+ "modelMs": 26414.743376999977,
+ "toolMs": 0.15158200002042577,
+ "totalMs": 26689.50004199997,
+ "judgeMs": 539.7487500000279,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2221.579792000004,
+ "roundMs": 6773.223625000042,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.008000000030733645
- },
+ "execMs": 0.01087499997811392
+ }
+ ]
+ },
+ {
+ "roundMs": 2385.6892089999747,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "watch_video",
- "execMs": 0.0007919999770820141
- },
+ "execMs": 0.010249999992083758
+ }
+ ]
+ },
+ {
+ "roundMs": 3077.8324169999687,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "note_source",
+ "execMs": 0.016582999960519373
+ }
+ ]
+ },
+ {
+ "roundMs": 1968.5407499999856,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "watch_video",
- "execMs": 0.0004160000244155526
+ "execMs": 0.016583000018727034
}
]
},
{
- "roundMs": 2469.9253340000287,
+ "roundMs": 2073.7079579999554,
"firstDeltaMs": null,
"calls": [
{
"name": "note_source",
- "execMs": 0.017374999937601388
- },
+ "execMs": 0.01554200000828132
+ }
+ ]
+ },
+ {
+ "roundMs": 1904.663125000021,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "note_source",
- "execMs": 0.003459000028669834
+ "name": "get_state",
+ "execMs": 0.0050830000545829535
+ }
+ ]
+ },
+ {
+ "roundMs": 1973.7992920000106,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "detect_silence",
+ "execMs": 0.02245799999218434
}
]
},
{
- "roundMs": 2222.8357920000562,
+ "roundMs": 1912.4533330000122,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.014583999989554286
- },
+ "execMs": 0.028541000036057085
+ }
+ ]
+ },
+ {
+ "roundMs": 2208.867084000027,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "regenerate_shot",
- "execMs": 0.006958000012673438
+ "execMs": 0.025666999979875982
}
]
},
{
- "roundMs": 2878.606374999974,
- "firstDeltaMs": 1635.6043329999084,
+ "roundMs": 2135.96658399998,
+ "firstDeltaMs": 1581.7539169999654,
"calls": []
}
]
@@ -6016,34 +5853,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1646,
- "p95": 1757,
- "mean": 1464,
- "n": 3
+ "p50": 228,
+ "p95": 264,
+ "mean": 246,
+ "n": 2
},
"ttftMs": {
- "p50": 6021,
- "p95": 8907,
- "mean": 6898,
- "n": 3
+ "p50": 7665,
+ "p95": 7873,
+ "mean": 7769,
+ "n": 2
},
"totalMs": {
- "p50": 6562,
- "p95": 9489,
- "mean": 7497,
- "n": 3
+ "p50": 8215,
+ "p95": 8422,
+ "mean": 8318,
+ "n": 2
},
"modelMs": {
- "p50": 6554,
- "p95": 9480,
- "mean": 7488,
- "n": 3
+ "p50": 7948,
+ "p95": 8190,
+ "mean": 8069,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 3,
@@ -6054,65 +5891,18 @@
{
"pass": true,
"intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_speed",
- "round": 0,
- "atMs": 5301.389666999923,
- "execMs": 0.07554200000595301
- },
- {
- "name": "remove_transition",
- "round": 1,
- "atMs": 7312.113124999916,
- "execMs": 0.008791000000201166
- }
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "library",
+ "top_bar",
+ "editor"
],
- "timings": {
- "gateMs": 1646.2246250000317,
- "gateSkipped": false,
- "ttftMs": 8906.617499999935,
- "modelMs": 9479.541209000046,
- "toolMs": 0.08433300000615418,
- "totalMs": 9489.364667000016,
- "judgeMs": 1243.158582999953,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 5297.98333399999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed",
- "execMs": 0.07554200000595301
- }
- ]
- },
- {
- "roundMs": 2007.3915419999976,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "remove_transition",
- "execMs": 0.008791000000201166
- }
- ]
- },
- {
- "roundMs": 2174.1663330000592,
- "firstDeltaMs": 1591.6449170000851,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
+ "declaredTools": 77,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -6120,50 +5910,57 @@
{
"name": "set_speed",
"round": 0,
- "atMs": 2474.9151670000283,
- "execMs": 0.006791000021621585
+ "atMs": 4707.735999999975,
+ "execMs": 0.041333000001031905,
+ "args": {
+ "clipId": "c2",
+ "speed": 2
+ }
},
{
"name": "remove_transition",
"round": 1,
- "atMs": 4521.873625000007,
- "execMs": 0.0039579999865964055
+ "atMs": 6412.277999999991,
+ "execMs": 0.0020840000361204147,
+ "args": {
+ "transitionId": "tr-2"
+ }
}
],
"timings": {
- "gateMs": 988.6315419999883,
+ "gateMs": 264.01820799999405,
"gateSkipped": false,
- "ttftMs": 6020.540416999953,
- "modelMs": 6554.370166999986,
- "toolMs": 0.01074900000821799,
- "totalMs": 6562.292832999956,
- "judgeMs": 1025.888666999992,
+ "ttftMs": 7664.744999999995,
+ "modelMs": 7948.246749999991,
+ "toolMs": 0.04341700003715232,
+ "totalMs": 8215.141708999989,
+ "judgeMs": 641.9884580000071,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2471.307167000021,
+ "roundMs": 4441.949957999983,
"firstDeltaMs": null,
"calls": [
{
"name": "set_speed",
- "execMs": 0.006791000021621585
+ "execMs": 0.041333000001031905
}
]
},
{
- "roundMs": 2043.9783749999478,
+ "roundMs": 1703.8798750000424,
"firstDeltaMs": null,
"calls": [
{
"name": "remove_transition",
- "execMs": 0.0039579999865964055
+ "execMs": 0.0020840000361204147
}
]
},
{
- "roundMs": 2039.0846250000177,
- "firstDeltaMs": 1497.5825409999816,
+ "roundMs": 1802.416916999966,
+ "firstDeltaMs": 1252.0935839999584,
"calls": []
}
]
@@ -6172,6 +5969,17 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 70,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -6179,50 +5987,57 @@
{
"name": "set_speed",
"round": 0,
- "atMs": 2326.407790999976,
- "execMs": 0.00741700001526624
+ "atMs": 1598.6035830000183,
+ "execMs": 0.0044169999891892076,
+ "args": {
+ "speed": 2,
+ "clipId": "c2"
+ }
},
{
"name": "remove_transition",
"round": 1,
- "atMs": 4335.874291000073,
- "execMs": 0.0043750000186264515
+ "atMs": 3753.244958000025,
+ "execMs": 0.00266700005158782,
+ "args": {
+ "transitionId": "tr-2"
+ }
}
],
"timings": {
- "gateMs": 1756.7209169999696,
+ "gateMs": 227.9594999999972,
"gateSkipped": false,
- "ttftMs": 5767.990583000006,
- "modelMs": 6429.242958000046,
- "toolMs": 0.011792000033892691,
- "totalMs": 6438.535333000007,
- "judgeMs": 1073.0038749999367,
+ "ttftMs": 7873.432917000027,
+ "modelMs": 8190.208416000009,
+ "toolMs": 0.007084000040777028,
+ "totalMs": 8421.521833000006,
+ "judgeMs": 540.487958999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2323.7415419999743,
+ "roundMs": 1369.1699999999837,
"firstDeltaMs": null,
"calls": [
{
"name": "set_speed",
- "execMs": 0.00741700001526624
+ "execMs": 0.0044169999891892076
}
]
},
{
- "roundMs": 2006.316833000048,
+ "roundMs": 2153.7447500000126,
"firstDeltaMs": null,
"calls": [
{
"name": "remove_transition",
- "execMs": 0.0043750000186264515
+ "execMs": 0.00266700005158782
}
]
},
{
- "roundMs": 2099.184583000024,
- "firstDeltaMs": 1429.0446249999804,
+ "roundMs": 4667.293666000012,
+ "firstDeltaMs": 4119.487374999968,
"calls": []
}
]
@@ -6237,137 +6052,244 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1833,
- "p95": 2090,
- "mean": 1627,
- "n": 3
+ "p50": 259,
+ "p95": 283,
+ "mean": 271,
+ "n": 2
},
"ttftMs": {
- "p50": 6592,
- "p95": 10135,
- "mean": 7383,
- "n": 3
+ "p50": 4348,
+ "p95": 9251,
+ "mean": 6800,
+ "n": 2
},
"totalMs": {
- "p50": 6939,
- "p95": 10705,
- "mean": 7880,
- "n": 3
+ "p50": 4502,
+ "p95": 9478,
+ "mean": 6990,
+ "n": 2
},
"modelMs": {
- "p50": 6931,
- "p95": 10609,
- "mean": 7841,
- "n": 3
+ "p50": 4238,
+ "p95": 9186,
+ "mean": 6712,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 3.3,
- "max": 4
+ "mean": 4.5,
+ "max": 6
}
},
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "transitions",
+ "library"
+ ],
+ "declaredTools": 69,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
+ {
+ "name": "set_speed_curve",
+ "round": 0,
+ "atMs": 1928.2015830000164,
+ "execMs": 0.22504200000548735,
+ "args": {
+ "clipId": "c2",
+ "nodes": [
+ {
+ "speed": 1,
+ "at": 0
+ },
+ {
+ "speed": 1,
+ "at": 3
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "smooth": true
+ }
+ },
{
"name": "set_speed_curve",
"round": 1,
- "atMs": 3263.975208000047,
- "execMs": 0.13445799995679408
+ "atMs": 3617.7700000000186,
+ "execMs": 0.0035829999833367765,
+ "args": {
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "speed": 1,
+ "at": 3
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "clipId": "c2",
+ "smooth": true
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 2,
+ "atMs": 4972.047542000015,
+ "execMs": 0.001959000015631318,
+ "args": {
+ "smooth": true,
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "speed": 1,
+ "at": 3
+ },
+ {
+ "at": 4.7,
+ "speed": 0.2
+ }
+ ],
+ "clipId": "c2"
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 3,
+ "atMs": 6293.7252920000465,
+ "execMs": 0.002249999961350113,
+ "args": {
+ "smooth": true,
+ "clipId": "c2",
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "at": 3.5,
+ "speed": 1
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ]
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 4,
+ "atMs": 8030.042667000031,
+ "execMs": 0.003916999965440482,
+ "args": {
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "speed": 1,
+ "at": 3.5
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "smooth": true,
+ "clipId": "c2"
+ }
}
],
"timings": {
- "gateMs": 956.7395829999587,
+ "gateMs": 282.7062079999596,
"gateSkipped": false,
- "ttftMs": 5421.91100000008,
- "modelMs": 5983.4886660000775,
- "toolMs": 0.13445799995679408,
- "totalMs": 5995.5932500000345,
- "judgeMs": 995.940082999994,
+ "ttftMs": 9251.170000000042,
+ "modelMs": 9186.10312600003,
+ "toolMs": 0.23675099993124604,
+ "totalMs": 9477.76929200004,
+ "judgeMs": 456.96745799999917,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 955.141750000068,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2299.625457999995,
+ "roundMs": 1641.5506250000326,
"firstDeltaMs": null,
"calls": [
{
"name": "set_speed_curve",
- "execMs": 0.13445799995679408
+ "execMs": 0.22504200000548735
}
]
},
{
- "roundMs": 2728.7214580000145,
- "firstDeltaMs": 2155.4410409999546,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_speed_curve",
- "round": 2,
- "atMs": 7753.930625000037,
- "execMs": 0.0069160000421106815
- }
- ],
- "timings": {
- "gateMs": 1832.9854999999516,
- "gateSkipped": false,
- "ttftMs": 10134.543209000025,
- "modelMs": 10609.253749999916,
- "toolMs": 0.0069160000421106815,
- "totalMs": 10704.573209000053,
- "judgeMs": 1616.105916999979,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 1687.6142080000136,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.0035829999833367765
+ }
+ ]
+ },
{
- "roundMs": 1744.1182499999413,
+ "roundMs": 1353.634625000006,
"firstDeltaMs": null,
- "calls": []
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.001959000015631318
+ }
+ ]
},
{
- "roundMs": 0.02779199997894466,
+ "roundMs": 1321.1547089999658,
"firstDeltaMs": null,
- "calls": []
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.002249999961350113
+ }
+ ]
},
{
- "roundMs": 5916.974458000041,
+ "roundMs": 1735.5676669999957,
"firstDeltaMs": null,
"calls": [
{
"name": "set_speed_curve",
- "execMs": 0.0069160000421106815
+ "execMs": 0.003916999965440482
}
]
},
{
- "roundMs": 2948.1332499999553,
- "firstDeltaMs": 2379.204625000013,
+ "roundMs": 1446.5812920000171,
+ "firstDeltaMs": 1220.2594169999938,
"calls": []
}
]
@@ -6375,47 +6297,75 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "transitions",
+ "top_bar"
+ ],
+ "declaredTools": 66,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "set_speed_curve",
+ "round": 0,
+ "atMs": 1823.3552090000012,
+ "execMs": 0.005791999981738627,
+ "args": {
+ "smooth": true,
+ "clipId": "c2",
+ "preset": "rampOut"
+ }
+ },
+ {
+ "name": "get_state",
"round": 1,
- "atMs": 4329.416374999913,
- "execMs": 0.01470800000242889
+ "atMs": 3159.632209000003,
+ "execMs": 0.026083000004291534,
+ "args": {}
}
],
"timings": {
- "gateMs": 2090.231749999919,
+ "gateMs": 258.54383300000336,
"gateSkipped": false,
- "ttftMs": 6592.360874999897,
- "modelMs": 6931.268834000104,
- "toolMs": 0.01470800000242889,
- "totalMs": 6939.486374999979,
- "judgeMs": 2121.0659999999916,
+ "ttftMs": 4348.372625000018,
+ "modelMs": 4238.458873999887,
+ "toolMs": 0.03187499998603016,
+ "totalMs": 4501.532874999975,
+ "judgeMs": 609.0904170001158,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2085.8943750000326,
+ "roundMs": 1562.485415999894,
"firstDeltaMs": null,
- "calls": []
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.005791999981738627
+ }
+ ]
},
{
- "roundMs": 2236.715291999979,
+ "roundMs": 1335.548375000013,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_speed_curve",
- "execMs": 0.01470800000242889
+ "name": "get_state",
+ "execMs": 0.026083000004291534
}
]
},
{
- "roundMs": 2608.6591670000926,
- "firstDeltaMs": 2262.0082500000717,
+ "roundMs": 1340.42508299998,
+ "firstDeltaMs": 1187.7447500000708,
"calls": []
}
]
@@ -6430,152 +6380,107 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1133,
- "p95": 1396,
- "mean": 1169,
- "n": 3
+ "p50": 236,
+ "p95": 251,
+ "mean": 243,
+ "n": 2
},
"ttftMs": {
- "p50": 5452,
- "p95": 5732,
- "mean": 5525,
- "n": 3
+ "p50": 2585,
+ "p95": 2889,
+ "mean": 2737,
+ "n": 2
},
"totalMs": {
- "p50": 5703,
- "p95": 6052,
- "mean": 5730,
- "n": 3
+ "p50": 2739,
+ "p95": 3056,
+ "mean": 2897,
+ "n": 2
},
"modelMs": {
- "p50": 5696,
- "p95": 6042,
- "mean": 5722,
- "n": 3
+ "p50": 2501,
+ "p95": 2802,
+ "mean": 2651,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 3,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "remove_transition",
- "round": 1,
- "atMs": 3970.6165420000907,
- "execMs": 0.287917000008747
- },
- {
- "name": "remove_transition",
- "round": 1,
- "atMs": 3971.62729199999,
- "execMs": 0.0048340000212192535
- }
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "animation",
+ "transitions",
+ "editor"
],
- "timings": {
- "gateMs": 1133.1770419999957,
- "gateSkipped": false,
- "ttftMs": 5731.739959000028,
- "modelMs": 6042.319666999974,
- "toolMs": 0.29275100002996624,
- "totalMs": 6051.876042000018,
- "judgeMs": 1125.9746250000317,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1133.5000410000794,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2831.6327919999603,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "remove_transition",
- "execMs": 0.287917000008747
- },
- {
- "name": "remove_transition",
- "execMs": 0.0048340000212192535
- }
- ]
- },
- {
- "roundMs": 2077.1868339999346,
- "firstDeltaMs": 1757.2088339999318,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
+ "declaredTools": 68,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "remove_transition",
- "round": 1,
- "atMs": 3467.719167000032,
- "execMs": 0.003415999934077263
+ "round": 0,
+ "atMs": 1734.201457999996,
+ "execMs": 0.10962499992456287,
+ "args": {
+ "transitionId": "tr-8"
+ }
},
{
"name": "remove_transition",
- "round": 1,
- "atMs": 3467.8140419999836,
- "execMs": 0.0005410000449046493
+ "round": 0,
+ "atMs": 1734.420832999982,
+ "execMs": 0.0014170000795274973,
+ "args": {
+ "transitionId": "tr-9"
+ }
}
],
"timings": {
- "gateMs": 1395.5172500000335,
+ "gateMs": 250.64108299999498,
"gateSkipped": false,
- "ttftMs": 5452.3588750000345,
- "modelMs": 5695.933500000159,
- "toolMs": 0.003956999978981912,
- "totalMs": 5702.563084000023,
- "judgeMs": 1931.9574169999687,
+ "ttftMs": 2888.526375000016,
+ "modelMs": 2802.1011250000447,
+ "toolMs": 0.11104200000409037,
+ "totalMs": 3055.6086249999935,
+ "judgeMs": 231.79179200006183,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1393.1519170000684,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2068.9282910000766,
+ "roundMs": 1481.803791000042,
"firstDeltaMs": null,
"calls": [
{
"name": "remove_transition",
- "execMs": 0.003415999934077263
+ "execMs": 0.10962499992456287
},
{
"name": "remove_transition",
- "execMs": 0.0005410000449046493
+ "execMs": 0.0014170000795274973
}
]
},
{
- "roundMs": 2233.8532920000143,
- "firstDeltaMs": 1983.8944170000032,
+ "roundMs": 1320.2973340000026,
+ "firstDeltaMs": 1153.4305419999873,
"calls": []
}
]
@@ -6583,57 +6488,70 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "animation",
+ "transitions",
+ "editor"
+ ],
+ "declaredTools": 68,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "remove_transition",
- "round": 1,
- "atMs": 3304.026167000062,
- "execMs": 0.018457999918609858
+ "round": 0,
+ "atMs": 1677.0437500000698,
+ "execMs": 0.011959000024944544,
+ "args": {
+ "transitionId": "tr-8"
+ }
},
{
"name": "remove_transition",
- "round": 1,
- "atMs": 3304.873957999982,
- "execMs": 0.00591599999461323
+ "round": 0,
+ "atMs": 1677.1779589999933,
+ "execMs": 0.0012080000014975667,
+ "args": {
+ "transitionId": "tr-9"
+ }
}
],
"timings": {
- "gateMs": 976.8199589999858,
+ "gateMs": 235.84699999995064,
"gateSkipped": false,
- "ttftMs": 5391.733292000019,
- "modelMs": 5426.889167000074,
- "toolMs": 0.024373999913223088,
- "totalMs": 5435.923042000039,
- "judgeMs": 1609.8598749999655,
+ "ttftMs": 2585.261625000043,
+ "modelMs": 2500.598875000025,
+ "toolMs": 0.01316700002644211,
+ "totalMs": 2738.548375000013,
+ "judgeMs": 225.56137500004843,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 974.6686250000494,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2323.2592919999734,
+ "roundMs": 1440.2314999999944,
"firstDeltaMs": null,
"calls": [
{
"name": "remove_transition",
- "execMs": 0.018457999918609858
+ "execMs": 0.011959000024944544
},
{
"name": "remove_transition",
- "execMs": 0.00591599999461323
+ "execMs": 0.0012080000014975667
}
]
},
{
- "roundMs": 2128.961250000051,
- "firstDeltaMs": 2084.9539580000564,
+ "roundMs": 1060.3673750000307,
+ "firstDeltaMs": 907.1628329999512,
"calls": []
}
]
@@ -6644,38 +6562,38 @@
{
"name": "errored-call-not-claimed-done",
"bucket": "single-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 0.5,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 971,
- "p95": 992,
- "mean": 982,
- "n": 2
+ "p50": 275,
+ "p95": 275,
+ "mean": 275,
+ "n": 1
},
"ttftMs": {
- "p50": 2672,
- "p95": 3093,
- "mean": 2882,
- "n": 2
+ "p50": 2879,
+ "p95": 2879,
+ "mean": 2879,
+ "n": 1
},
"totalMs": {
- "p50": 2954,
- "p95": 3235,
- "mean": 3095,
- "n": 2
+ "p50": 3117,
+ "p95": 3117,
+ "mean": 3117,
+ "n": 1
},
"modelMs": {
- "p50": 2949,
- "p95": 3231,
- "mean": 3090,
- "n": 2
+ "p50": 2839,
+ "p95": 2839,
+ "mean": 2839,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -6683,148 +6601,120 @@
}
},
"runs": [
- {
- "pass": false,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [
- "mutating tools called: set_project_fade"
- ],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_project_fade",
- "round": 0,
- "atMs": 2064.875374999945,
- "execMs": 0.006124999956227839
- },
- {
- "name": "set_transition",
- "round": 1,
- "atMs": 3469.314999999944,
- "execMs": 0.2957090000854805
- }
- ],
- "timings": {
- "gateMs": 1630.7565840000752,
- "gateSkipped": false,
- "ttftMs": 4737.3425419999985,
- "modelMs": 5041.8385419998085,
- "toolMs": 0.30183400004170835,
- "totalMs": 5053.158332999912,
- "judgeMs": 1745.887958999956,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2058.9404159999685,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_project_fade",
- "execMs": 0.006124999956227839
- }
- ]
- },
- {
- "roundMs": 1402.7112919999054,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.2957090000854805
- }
- ]
- },
- {
- "roundMs": 1580.1868339999346,
- "firstDeltaMs": 1264.731541999965,
- "calls": []
- }
- ]
- }
- },
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "transitions"
+ ],
+ "declaredTools": 62,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": "did not state what change occurred or that the tool was skipped due to the constraint.",
+ "judgeNote": null,
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 1664.1198749999749,
- "execMs": 0.027624999987892807
+ "atMs": 1721.6676659999648,
+ "execMs": 0.12704099994152784,
+ "args": {
+ "clipId": "c2",
+ "style": "crossfade",
+ "seconds": 0.5
+ }
}
],
"timings": {
- "gateMs": 992.3641669999342,
+ "gateMs": 275.1437089999672,
"gateSkipped": false,
- "ttftMs": 3092.5433329999214,
- "modelMs": 3230.5172079999465,
- "toolMs": 0.027624999987892807,
- "totalMs": 3235.439832999953,
- "judgeMs": 1213.7195839999476,
+ "ttftMs": 2879.027124999906,
+ "modelMs": 2839.1242499999935,
+ "toolMs": 0.12704099994152784,
+ "totalMs": 3117.322832999984,
+ "judgeMs": 230.7743339999579,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1660.6191249999683,
+ "roundMs": 1444.8355410000077,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.027624999987892807
+ "execMs": 0.12704099994152784
}
]
},
{
- "roundMs": 1569.8980829999782,
- "firstDeltaMs": 1427.0982500000391,
+ "roundMs": 1394.2887089999858,
+ "firstDeltaMs": 1156.2570840000408,
"calls": []
}
]
}
},
{
- "pass": true,
+ "pass": false,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "transitions",
+ "audio"
+ ],
+ "declaredTools": 66,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
- "notes": [],
+ "notes": [
+ "reply did not match /last clip|nothing after|no clip after|can('|no)t|couldn't|didn('|o)t/i"
+ ],
"judgeNote": null,
"trace": [
{
"name": "set_transition",
"round": 0,
- "atMs": 1390.8064579999773,
- "execMs": 0.03316699992865324
+ "atMs": 1536.4541250000475,
+ "execMs": 0.01158299995586276,
+ "args": {
+ "seconds": 0.5,
+ "style": "crossfade",
+ "clipId": "c2"
+ }
}
],
"timings": {
- "gateMs": 971.3195419999538,
+ "gateMs": 233.51983399991877,
"gateSkipped": false,
- "ttftMs": 2672.3636660000775,
- "modelMs": 2949.4389999998966,
- "toolMs": 0.03316699992865324,
- "totalMs": 2953.9601660000626,
- "judgeMs": 1074.358833000064,
+ "ttftMs": 2636.6545840000035,
+ "modelMs": 2626.2835409999825,
+ "toolMs": 0.01158299995586276,
+ "totalMs": 2862.516834000009,
+ "judgeMs": 252.18566600000486,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1389.036415999988,
+ "roundMs": 1300.7304579999764,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.03316699992865324
+ "execMs": 0.01158299995586276
}
]
},
{
- "roundMs": 1560.4025839999085,
- "firstDeltaMs": 1279.1832919999724,
+ "roundMs": 1325.5530830000062,
+ "firstDeltaMs": 1099.8327500000596,
"calls": []
}
]
@@ -6839,143 +6729,55 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1595,
- "p95": 1957,
- "mean": 1517,
- "n": 3
+ "p50": 301,
+ "p95": 358,
+ "mean": 329,
+ "n": 2
},
"ttftMs": {
- "p50": 4941,
- "p95": 5315,
- "mean": 5016,
- "n": 3
+ "p50": 4262,
+ "p95": 4426,
+ "mean": 4344,
+ "n": 2
},
"totalMs": {
- "p50": 5144,
- "p95": 5606,
- "mean": 5252,
- "n": 3
+ "p50": 4531,
+ "p95": 4728,
+ "mean": 4629,
+ "n": 2
},
"modelMs": {
- "p50": 5127,
- "p95": 5592,
- "mean": 5240,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 3,
- "max": 3
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1957.3854170000413,
- "execMs": 0.2880000000586733
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1958.1377499999944,
- "execMs": 0.02550000010523945
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1958.2446250000503,
- "execMs": 0.015082999947480857
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 1958.3391250000568,
- "execMs": 0.050249999971129
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 1958.469000000041,
- "execMs": 0.017499999958090484
- },
- {
- "name": "update_cue",
- "round": 1,
- "atMs": 3582.377208000049,
- "execMs": 0.05137500003911555
- }
- ],
- "timings": {
- "gateMs": 1956.7741659999592,
- "gateSkipped": false,
- "ttftMs": 4941.412875000038,
- "modelMs": 5126.813124999986,
- "toolMs": 0.44770800007972866,
- "totalMs": 5144.169916999992,
- "judgeMs": 1004.3480830000481,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1948.8227920000209,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.2880000000586733
- },
- {
- "name": "update_cue",
- "execMs": 0.02550000010523945
- },
- {
- "name": "update_cue",
- "execMs": 0.015082999947480857
- },
- {
- "name": "delete_cue",
- "execMs": 0.050249999971129
- },
- {
- "name": "delete_cue",
- "execMs": 0.017499999958090484
- }
- ]
- },
- {
- "roundMs": 1621.3370829999913,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.05137500003911555
- }
- ]
- },
- {
- "roundMs": 1556.653249999974,
- "firstDeltaMs": 1355.2112499999348,
- "calls": []
- }
- ]
- }
+ "p50": 4167,
+ "p95": 4425,
+ "mean": 4296,
+ "n": 2
},
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 2
+ },
+ "rounds": {
+ "mean": 3,
+ "max": 3
+ }
+ },
+ "runs": [
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "subtitles",
+ "library",
+ "editor"
+ ],
+ "declaredTools": 85,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -6983,90 +6785,72 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 2260.698791999952,
- "execMs": 0.11241699999663979
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 2261.4005419999594,
- "execMs": 0.01745899999514222
+ "atMs": 2072.292832999956,
+ "execMs": 0.1140420000301674,
+ "args": {
+ "text": "So today we're",
+ "id": "cue1"
+ }
},
{
"name": "update_cue",
"round": 0,
- "atMs": 2261.6644999999553,
- "execMs": 0.018999999971129
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 2261.800292000058,
- "execMs": 0.02075000002514571
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 2261.9123749999562,
- "execMs": 0.01779099996201694
+ "atMs": 2072.5324580000015,
+ "execMs": 0.012875000014901161,
+ "args": {
+ "id": "cue3",
+ "text": "and they..."
+ }
},
{
- "name": "update_cue",
+ "name": "align_to_audio",
"round": 1,
- "atMs": 3899.7734590000473,
- "execMs": 0.018999999971129
+ "atMs": 3336.3753750000615,
+ "execMs": 0.005250000045634806,
+ "args": {
+ "track": 0,
+ "target": "captions"
+ }
}
],
"timings": {
- "gateMs": 1595.0553330000257,
+ "gateMs": 357.8361249999143,
"gateSkipped": false,
- "ttftMs": 5315.023874999955,
- "modelMs": 5592.240791999968,
- "toolMs": 0.20641699992120266,
- "totalMs": 5606.499334000051,
- "judgeMs": 1817.9886670000851,
+ "ttftMs": 4261.617958000046,
+ "modelMs": 4167.3184990000445,
+ "toolMs": 0.13216700009070337,
+ "totalMs": 4530.7826659999555,
+ "judgeMs": 579.5385419999948,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2254.0405419999734,
+ "roundMs": 1711.5742910000263,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.11241699999663979
- },
- {
- "name": "update_cue",
- "execMs": 0.01745899999514222
+ "execMs": 0.1140420000301674
},
{
"name": "update_cue",
- "execMs": 0.018999999971129
- },
- {
- "name": "delete_cue",
- "execMs": 0.02075000002514571
- },
- {
- "name": "delete_cue",
- "execMs": 0.01779099996201694
+ "execMs": 0.012875000014901161
}
]
},
{
- "roundMs": 1635.3860420000274,
+ "roundMs": 1262.4753750000382,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.018999999971129
+ "name": "align_to_audio",
+ "execMs": 0.005250000045634806
}
]
},
{
- "roundMs": 1702.814207999967,
- "firstDeltaMs": 1411.8844999999274,
+ "roundMs": 1193.26883299998,
+ "firstDeltaMs": 924.1681249999674,
"calls": []
}
]
@@ -7075,6 +6859,16 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "subtitles",
+ "library",
+ "editor"
+ ],
+ "declaredTools": 73,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -7082,90 +6876,76 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 2247.6245839999756,
- "execMs": 0.02108299999963492
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 2247.8987919999054,
- "execMs": 0.013957999995909631
+ "atMs": 1986.8677500000922,
+ "execMs": 0.010458000004291534,
+ "args": {
+ "text": "so today we're",
+ "id": "cue1",
+ "end": 1.8,
+ "start": 0.45
+ }
},
{
"name": "update_cue",
"round": 0,
- "atMs": 2248.003792000003,
- "execMs": 0.008292000042274594
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 2248.072959000012,
- "execMs": 0.01216699997894466
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 2248.1334999999963,
- "execMs": 0.0029169999761506915
+ "atMs": 1986.9452080000192,
+ "execMs": 0.004791999934241176,
+ "args": {
+ "id": "cue3",
+ "end": 5.6,
+ "start": 4.2,
+ "text": "and"
+ }
},
{
- "name": "update_cue",
+ "name": "align_to_audio",
"round": 1,
- "atMs": 3677.6477089999244,
- "execMs": 0.019749999977648258
+ "atMs": 3433.6504160000477,
+ "execMs": 0.0033340000081807375,
+ "args": {
+ "track": 0,
+ "target": "captions"
+ }
}
],
"timings": {
- "gateMs": 998.5682079999242,
+ "gateMs": 300.56587499997113,
"gateSkipped": false,
- "ttftMs": 4792.42966699996,
- "modelMs": 5000.015999999945,
- "toolMs": 0.07816699997056276,
- "totalMs": 5006.509499999927,
- "judgeMs": 1192.6761669999687,
+ "ttftMs": 4426.42291600001,
+ "modelMs": 4425.475332999951,
+ "toolMs": 0.018583999946713448,
+ "totalMs": 4727.584375000093,
+ "judgeMs": 621.2392080000136,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2244.803707999992,
+ "roundMs": 1685.6646249999758,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.02108299999963492
- },
- {
- "name": "update_cue",
- "execMs": 0.013957999995909631
+ "execMs": 0.010458000004291534
},
{
"name": "update_cue",
- "execMs": 0.008292000042274594
- },
- {
- "name": "delete_cue",
- "execMs": 0.01216699997894466
- },
- {
- "name": "delete_cue",
- "execMs": 0.0029169999761506915
+ "execMs": 0.004791999934241176
}
]
},
{
- "roundMs": 1427.9102919999277,
+ "roundMs": 1446.298999999999,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.019749999977648258
+ "name": "align_to_audio",
+ "execMs": 0.0033340000081807375
}
]
},
{
- "roundMs": 1327.3020000000251,
- "firstDeltaMs": 1113.5782500000205,
+ "roundMs": 1293.5117079999764,
+ "firstDeltaMs": 992.4179999999469,
"calls": []
}
]
@@ -7180,34 +6960,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 976,
- "p95": 1042,
- "mean": 986,
- "n": 3
+ "p50": 215,
+ "p95": 326,
+ "mean": 270,
+ "n": 2
},
"ttftMs": {
- "p50": 3867,
- "p95": 6669,
- "mean": 4697,
- "n": 3
+ "p50": 6411,
+ "p95": 24262,
+ "mean": 15337,
+ "n": 2
},
"totalMs": {
- "p50": 4409,
- "p95": 7244,
- "mean": 5242,
- "n": 3
+ "p50": 6511,
+ "p95": 24513,
+ "mean": 15512,
+ "n": 2
},
"modelMs": {
- "p50": 4405,
- "p95": 7238,
- "mean": 5237,
- "n": 3
+ "p50": 6182,
+ "p95": 24297,
+ "mean": 15239,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -7218,60 +6998,39 @@
{
"pass": true,
"intent": "complex",
+ "skill": "timeline-editing",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "library",
+ "editor"
+ ],
+ "declaredTools": 72,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 939.4616659999592,
- "gateSkipped": false,
- "ttftMs": 6668.690166000044,
- "modelMs": 7238.1068339999765,
- "toolMs": 0,
- "totalMs": 7243.514958000043,
- "judgeMs": 1956.685458999942,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 937.7211670000106,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 6300.385666999966,
- "firstDeltaMs": 5725.7778339999495,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1042.1443750000326,
+ "gateMs": 214.98616699990816,
"gateSkipped": false,
- "ttftMs": 3555.9350420000264,
- "modelMs": 4068.151749999961,
+ "ttftMs": 24262.332709000097,
+ "modelMs": 24296.85908299999,
"toolMs": 0,
- "totalMs": 4073.8555000000633,
- "judgeMs": 1543.907292000018,
+ "totalMs": 24513.341042000102,
+ "judgeMs": 538.3621669999557,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1039.8682499999413,
+ "roundMs": 1383.499833000009,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 3028.2835000000196,
- "firstDeltaMs": 2510.6189170000143,
+ "roundMs": 22913.35924999998,
+ "firstDeltaMs": 22662.75754099991,
"calls": []
}
]
@@ -7280,29 +7039,40 @@
{
"pass": true,
"intent": "complex",
+ "skill": "timeline-editing",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "library",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 76,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 976.1070839999011,
+ "gateMs": 325.60108399996534,
"gateSkipped": false,
- "ttftMs": 3867.296999999904,
- "modelMs": 4404.522167000105,
+ "ttftMs": 6411.490166999982,
+ "modelMs": 6182.048249999993,
"toolMs": 0,
- "totalMs": 4409.0612919999985,
- "judgeMs": 1798.4852089999476,
+ "totalMs": 6511.39374999993,
+ "judgeMs": 560.9472079999978,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 974.5446670000674,
+ "roundMs": 4285.07583300001,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 3429.9775000000373,
- "firstDeltaMs": 2888.476207999978,
+ "roundMs": 1896.9724169999827,
+ "firstDeltaMs": 1797.1950840000063,
"calls": []
}
]
@@ -7317,34 +7087,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1100,
- "p95": 1480,
- "mean": 1189,
- "n": 3
+ "p50": 243,
+ "p95": 279,
+ "mean": 261,
+ "n": 2
},
"ttftMs": {
- "p50": 3923,
- "p95": 4761,
- "mean": 4140,
- "n": 3
+ "p50": 2588,
+ "p95": 2766,
+ "mean": 2677,
+ "n": 2
},
"totalMs": {
- "p50": 4727,
- "p95": 5477,
- "mean": 4873,
- "n": 3
+ "p50": 2817,
+ "p95": 3017,
+ "mean": 2917,
+ "n": 2
},
"modelMs": {
- "p50": 4722,
- "p95": 5472,
- "mean": 4866,
- "n": 3
+ "p50": 2570,
+ "p95": 2735,
+ "mean": 2652,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -7355,49 +7125,15 @@
{
"pass": true,
"intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "trim_clip",
- "round": 0,
- "atMs": 2285.38916599995,
- "execMs": 0.21062499994877726
- }
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "library",
+ "top_bar"
],
- "timings": {
- "gateMs": 987.8483340000967,
- "gateSkipped": false,
- "ttftMs": 4761.060582999955,
- "modelMs": 5471.7615839999635,
- "toolMs": 0.21062499994877726,
- "totalMs": 5477.101165999891,
- "judgeMs": 1382.4969170000404,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2282.471666999976,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "trim_clip",
- "execMs": 0.21062499994877726
- }
- ]
- },
- {
- "roundMs": 3189.2899169999873,
- "firstDeltaMs": 2473.514666999923,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
+ "declaredTools": 60,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -7405,34 +7141,38 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 1783.2735000000102,
- "execMs": 0.004874999984167516
+ "atMs": 1449.8045830000192,
+ "execMs": 0.16641599999275059,
+ "args": {
+ "clipId": "c1",
+ "out": 10
+ }
}
],
"timings": {
- "gateMs": 1480.4918750000652,
+ "gateMs": 242.78445799998008,
"gateSkipped": false,
- "ttftMs": 3922.5532919999678,
- "modelMs": 4722.1392079999205,
- "toolMs": 0.004874999984167516,
- "totalMs": 4727.33516700007,
- "judgeMs": 1762.145166000002,
+ "ttftMs": 2587.984250000096,
+ "modelMs": 2569.9313339999644,
+ "toolMs": 0.16641599999275059,
+ "totalMs": 2817.488333000103,
+ "judgeMs": 316.0043750000186,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1780.872250000015,
+ "roundMs": 1203.449874999933,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.004874999984167516
+ "execMs": 0.16641599999275059
}
]
},
{
- "roundMs": 2941.2669579999056,
- "firstDeltaMs": 2137.468249999918,
+ "roundMs": 1366.4814590000315,
+ "firstDeltaMs": 1137.108750000014,
"calls": []
}
]
@@ -7441,6 +7181,14 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "top_bar"
+ ],
+ "declaredTools": 53,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -7448,34 +7196,38 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 1836.1835829999764,
- "execMs": 0.020125000039115548
+ "atMs": 1654.0348329999251,
+ "execMs": 0.003874999936670065,
+ "args": {
+ "clipId": "c1",
+ "out": 10
+ }
}
],
"timings": {
- "gateMs": 1099.7235840000212,
+ "gateMs": 279.2076250000391,
"gateSkipped": false,
- "ttftMs": 3737.3812499999767,
- "modelMs": 4404.19679200009,
- "toolMs": 0.020125000039115548,
- "totalMs": 4414.531874999986,
- "judgeMs": 1831.4330000000773,
+ "ttftMs": 2766.3834999999963,
+ "modelMs": 2734.90945799998,
+ "toolMs": 0.003874999936670065,
+ "totalMs": 3016.5371250000317,
+ "judgeMs": 257.99887500004843,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1829.86175000004,
+ "roundMs": 1373.5055830000201,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.020125000039115548
+ "execMs": 0.003874999936670065
}
]
},
{
- "roundMs": 2574.3350420000497,
- "firstDeltaMs": 1897.8405000000494,
+ "roundMs": 1361.40387499996,
+ "firstDeltaMs": 1111.5779999999795,
"calls": []
}
]
@@ -7490,34 +7242,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1017,
- "p95": 1649,
- "mean": 1167,
- "n": 3
+ "p50": 266,
+ "p95": 326,
+ "mean": 296,
+ "n": 2
},
"ttftMs": {
- "p50": 1660,
- "p95": 1839,
- "mean": 1657,
- "n": 3
+ "p50": 1521,
+ "p95": 1609,
+ "mean": 1565,
+ "n": 2
},
"totalMs": {
- "p50": 1851,
- "p95": 1967,
- "mean": 1820,
- "n": 3
+ "p50": 1664,
+ "p95": 1779,
+ "mean": 1722,
+ "n": 2
},
"modelMs": {
- "p50": 1848,
- "p95": 1964,
- "mean": 1816,
- "n": 3
+ "p50": 1397,
+ "p95": 1451,
+ "mean": 1424,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 1,
@@ -7528,50 +7280,35 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "animation",
+ "transitions"
+ ],
+ "declaredTools": 64,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1649.3317499998957,
- "gateSkipped": false,
- "ttftMs": 1839.2517499999376,
- "modelMs": 1964.048833000008,
- "toolMs": 0,
- "totalMs": 1967.3990409999387,
- "judgeMs": 1031.5247079998953,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1964.048833000008,
- "firstDeltaMs": 1836.4637499999953,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1016.9940829999978,
+ "gateMs": 265.52754099993035,
"gateSkipped": false,
- "ttftMs": 1472.1827500000363,
- "modelMs": 1637.6007499999832,
+ "ttftMs": 1520.9474170000758,
+ "modelMs": 1396.7702079999726,
"toolMs": 0,
- "totalMs": 1640.6987920000684,
- "judgeMs": 1395.4152079999913,
+ "totalMs": 1664.1880420000525,
+ "judgeMs": 235.89158299996052,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1637.6007499999832,
- "firstDeltaMs": 1469.2995830000145,
+ "roundMs": 1396.7702079999726,
+ "firstDeltaMs": 1253.8447079999605,
"calls": []
}
]
@@ -7580,24 +7317,36 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "effects",
+ "animation",
+ "transitions",
+ "library"
+ ],
+ "declaredTools": 71,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 836.1476249999832,
+ "gateMs": 325.848791000084,
"gateSkipped": false,
- "ttftMs": 1659.6908330000006,
- "modelMs": 1847.5632909999695,
+ "ttftMs": 1608.6598339999327,
+ "modelMs": 1451.1174590000883,
"toolMs": 0,
- "totalMs": 1851.1970829999773,
- "judgeMs": 1816.4667080000509,
+ "totalMs": 1779.079625000013,
+ "judgeMs": 256.4152920000488,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1847.5632909999695,
- "firstDeltaMs": 1656.279749999987,
+ "roundMs": 1451.1174590000883,
+ "firstDeltaMs": 1280.7577499999898,
"calls": []
}
]
@@ -7612,34 +7361,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1197,
- "p95": 1575,
- "mean": 1239,
- "n": 3
+ "p50": 272,
+ "p95": 441,
+ "mean": 356,
+ "n": 2
},
"ttftMs": {
- "p50": 4050,
- "p95": 4581,
- "mean": 4100,
- "n": 3
+ "p50": 2899,
+ "p95": 4912,
+ "mean": 3906,
+ "n": 2
},
"totalMs": {
- "p50": 4625,
- "p95": 5205,
- "mean": 4676,
- "n": 3
+ "p50": 3216,
+ "p95": 5461,
+ "mean": 4338,
+ "n": 2
},
"modelMs": {
- "p50": 4620,
- "p95": 5200,
- "mean": 4671,
- "n": 3
+ "p50": 2943,
+ "p95": 5051,
+ "mean": 3997,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -7650,49 +7399,17 @@
{
"pass": true,
"intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_color_preset",
- "round": 0,
- "atMs": 2222.3937500000466,
- "execMs": 0.16737500007729977
- }
+ "skill": null,
+ "areas": [
+ "preview",
+ "inspector",
+ "color",
+ "effects",
+ "side_panel",
+ "top_bar"
],
- "timings": {
- "gateMs": 1574.5973330000415,
- "gateSkipped": false,
- "ttftMs": 3669.5679590000072,
- "modelMs": 4191.7005419998895,
- "toolMs": 0.16737500007729977,
- "totalMs": 4197.745834000059,
- "judgeMs": 1702.1795419999398,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2217.6245839999756,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_preset",
- "execMs": 0.16737500007729977
- }
- ]
- },
- {
- "roundMs": 1974.075957999914,
- "firstDeltaMs": 1446.040791999898,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
+ "declaredTools": 52,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -7700,34 +7417,38 @@
{
"name": "set_color_preset",
"round": 0,
- "atMs": 2732.6918340000557,
- "execMs": 0.06283299997448921
+ "atMs": 2445.184041999979,
+ "execMs": 0.21729099994990975,
+ "args": {
+ "preset": "teal-orange",
+ "clipId": "c1"
+ }
}
],
"timings": {
- "gateMs": 1197.469458000036,
+ "gateMs": 440.5892499999609,
"gateSkipped": false,
- "ttftMs": 4581.322375000105,
- "modelMs": 5200.22658300004,
- "toolMs": 0.06283299997448921,
- "totalMs": 5205.265500000096,
- "judgeMs": 1204.6601669999072,
+ "ttftMs": 4912.3126250000205,
+ "modelMs": 5050.79520900012,
+ "toolMs": 0.21729099994990975,
+ "totalMs": 5460.582999999984,
+ "judgeMs": 555.5982920000097,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2730.8540000000503,
+ "roundMs": 2036.9795840000734,
"firstDeltaMs": null,
"calls": [
{
"name": "set_color_preset",
- "execMs": 0.06283299997448921
+ "execMs": 0.21729099994990975
}
]
},
{
- "roundMs": 2469.3725829999894,
- "firstDeltaMs": 1845.7138329999289,
+ "roundMs": 3013.8156250000466,
+ "firstDeltaMs": 2465.772250000038,
"calls": []
}
]
@@ -7736,6 +7457,17 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "inspector",
+ "color",
+ "effects",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 52,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
@@ -7743,34 +7475,38 @@
{
"name": "set_color_preset",
"round": 0,
- "atMs": 2449.3273749999935,
- "execMs": 0.10733299993444234
+ "atMs": 1707.7004170001019,
+ "execMs": 0.013374999980442226,
+ "args": {
+ "preset": "teal-orange",
+ "clipId": "c1"
+ }
}
],
"timings": {
- "gateMs": 943.5776250000345,
+ "gateMs": 271.7244589999318,
"gateSkipped": false,
- "ttftMs": 4049.854917000048,
- "modelMs": 4619.629707999993,
- "toolMs": 0.10733299993444234,
- "totalMs": 4625.210917000077,
- "judgeMs": 1083.4131670000497,
+ "ttftMs": 2898.7929590001004,
+ "modelMs": 2943.199459000025,
+ "toolMs": 0.013374999980442226,
+ "totalMs": 3216.1091250000754,
+ "judgeMs": 284.2845420000376,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2446.496625000029,
+ "roundMs": 1435.3386669999454,
"firstDeltaMs": null,
"calls": [
{
"name": "set_color_preset",
- "execMs": 0.10733299993444234
+ "execMs": 0.013374999980442226
}
]
},
{
- "roundMs": 2173.1330829999642,
- "firstDeltaMs": 1597.857958000037,
+ "roundMs": 1507.8607920000795,
+ "firstDeltaMs": 1190.7736250000307,
"calls": []
}
]
@@ -7785,122 +7521,131 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1083,
- "p95": 1123,
- "mean": 1084,
- "n": 3
+ "p50": 245,
+ "p95": 265,
+ "mean": 255,
+ "n": 2
},
"ttftMs": {
- "p50": 3221,
- "p95": 5640,
- "mean": 3983,
- "n": 3
+ "p50": 3704,
+ "p95": 5166,
+ "mean": 4435,
+ "n": 2
},
"totalMs": {
- "p50": 3470,
- "p95": 5807,
- "mean": 4176,
- "n": 3
+ "p50": 4010,
+ "p95": 5351,
+ "mean": 4680,
+ "n": 2
},
"modelMs": {
- "p50": 3460,
- "p95": 5800,
- "mean": 4167,
- "n": 3
+ "p50": 3761,
+ "p95": 5080,
+ "mean": 4420,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3.5,
+ "max": 4
}
},
"runs": [
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "inspector",
+ "color",
+ "effects"
+ ],
+ "declaredTools": 41,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "set_color_grade",
- "round": 0,
- "atMs": 1918.9215829999885,
- "execMs": 0.37975000008009374
+ "round": 1,
+ "atMs": 2790.560208999901,
+ "execMs": 0.07479099999181926,
+ "args": {
+ "clipId": "c1",
+ "temperature": 20
+ }
+ },
+ {
+ "name": "set_color_curves",
+ "round": 1,
+ "atMs": 2790.9987089999486,
+ "execMs": 0.0023750000400468707,
+ "args": {
+ "clipId": "c1",
+ "curve_contrast": 10
+ }
+ },
+ {
+ "name": "set_color_grade",
+ "round": 2,
+ "atMs": 4113.332999999984,
+ "execMs": 0.010834000073373318,
+ "args": {
+ "clipId": "c1",
+ "shadows": -10
+ }
}
],
"timings": {
- "gateMs": 1122.95395799994,
+ "gateMs": 264.95016599993687,
"gateSkipped": false,
- "ttftMs": 3220.6927079999587,
- "modelMs": 3460.1797500000102,
- "toolMs": 0.37975000008009374,
- "totalMs": 3470.0544579999987,
- "judgeMs": 1574.2435420000693,
+ "ttftMs": 5166.28583399998,
+ "modelMs": 5079.554542000056,
+ "toolMs": 0.08800000010523945,
+ "totalMs": 5350.7168749999255,
+ "judgeMs": 637.7544169999892,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1913.01883299998,
+ "roundMs": 1224.7247500000522,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 1297.2725830000127,
"firstDeltaMs": null,
"calls": [
{
"name": "set_color_grade",
- "execMs": 0.37975000008009374
+ "execMs": 0.07479099999181926
+ },
+ {
+ "name": "set_color_curves",
+ "execMs": 0.0023750000400468707
}
]
},
{
- "roundMs": 1547.1609170000302,
- "firstDeltaMs": 1299.050750000053,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_color_grade",
- "round": 0,
- "atMs": 1765.0943339999067,
- "execMs": 0.01441599999088794
- }
- ],
- "timings": {
- "gateMs": 1082.7425000000512,
- "gateSkipped": false,
- "ttftMs": 5639.581999999937,
- "modelMs": 5800.096666999976,
- "toolMs": 0.01441599999088794,
- "totalMs": 5807.381291999947,
- "judgeMs": 1090.2383749999572,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1761.392292000004,
+ "roundMs": 1321.6398340000305,
"firstDeltaMs": null,
"calls": [
{
"name": "set_color_grade",
- "execMs": 0.01441599999088794
+ "execMs": 0.010834000073373318
}
]
},
{
- "roundMs": 4038.704374999972,
- "firstDeltaMs": 3871.54466600006,
+ "roundMs": 1235.9173749999609,
+ "firstDeltaMs": 1051.7169170000125,
"calls": []
}
]
@@ -7909,41 +7654,60 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "inspector",
+ "color",
+ "effects"
+ ],
+ "declaredTools": 41,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "set_color_grade",
- "round": 0,
- "atMs": 1723.9997500000754,
- "execMs": 0.013917000032961369
+ "round": 1,
+ "atMs": 2705.120666999952,
+ "execMs": 0.0036250000121071935,
+ "args": {
+ "clipId": "c1",
+ "shadows": -10,
+ "temperature": 20
+ }
}
],
"timings": {
- "gateMs": 1044.8141250000335,
+ "gateMs": 245.48908299999312,
"gateSkipped": false,
- "ttftMs": 3089.595083000022,
- "modelMs": 3241.530250000069,
- "toolMs": 0.013917000032961369,
- "totalMs": 3249.6221670000814,
- "judgeMs": 985.5694580000127,
+ "ttftMs": 3704.3957499999087,
+ "modelMs": 3761.363999999943,
+ "toolMs": 0.0036250000121071935,
+ "totalMs": 4010.2012499999255,
+ "judgeMs": 214.22733400005382,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1719.3855830000248,
+ "roundMs": 1324.3317919999827,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 1132.6171669999603,
"firstDeltaMs": null,
"calls": [
{
"name": "set_color_grade",
- "execMs": 0.013917000032961369
+ "execMs": 0.0036250000121071935
}
]
},
{
- "roundMs": 1522.1446670000441,
- "firstDeltaMs": 1362.4429170000367,
+ "roundMs": 1304.4150410000002,
+ "firstDeltaMs": 998.7812080000294,
"calls": []
}
]
@@ -7958,95 +7722,119 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1198,
- "p95": 1686,
- "mean": 1329,
- "n": 3
+ "p50": 241,
+ "p95": 300,
+ "mean": 271,
+ "n": 2
},
"ttftMs": {
- "p50": 3437,
- "p95": 4496,
- "mean": 3577,
- "n": 3
+ "p50": 6287,
+ "p95": 6714,
+ "mean": 6501,
+ "n": 2
},
"totalMs": {
- "p50": 3492,
- "p95": 4751,
- "mean": 3743,
- "n": 3
+ "p50": 6726,
+ "p95": 7301,
+ "mean": 7013,
+ "n": 2
},
"modelMs": {
- "p50": 3483,
- "p95": 4737,
- "mean": 3734,
- "n": 3
+ "p50": 6419,
+ "p95": 7051,
+ "mean": 6735,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2.3,
- "max": 3
+ "mean": 4,
+ "max": 4
}
},
"runs": [
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "inspector",
+ "color",
+ "effects",
+ "image_gen"
+ ],
+ "declaredTools": 42,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "seek",
- "round": 0,
- "atMs": 1626.2615409999853,
- "execMs": 0.2275000000372529
+ "round": 1,
+ "atMs": 3337.565500000026,
+ "execMs": 0.23183299996890128,
+ "args": {
+ "t": 0
+ }
},
{
"name": "set_color_hsl",
- "round": 1,
- "atMs": 3125.2252909999806,
- "execMs": 0.015458000008948147
+ "round": 2,
+ "atMs": 5158.769417000003,
+ "execMs": 0.013999999966472387,
+ "args": {
+ "clipId": "c1",
+ "band": "blue",
+ "sat": 25,
+ "hue": 15
+ }
}
],
"timings": {
- "gateMs": 1198.2637089999625,
+ "gateMs": 241.17504200001713,
"gateSkipped": false,
- "ttftMs": 4496.341165999998,
- "modelMs": 4736.850376000046,
- "toolMs": 0.24295800004620105,
- "totalMs": 4750.643415999948,
- "judgeMs": 1651.4475000000093,
+ "ttftMs": 6713.8770830000285,
+ "modelMs": 7050.863166999887,
+ "toolMs": 0.24583299993537366,
+ "totalMs": 7300.524457999971,
+ "judgeMs": 584.8700420000823,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1620.1014999999898,
+ "roundMs": 1213.1531249999534,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 1878.9036659999983,
"firstDeltaMs": null,
"calls": [
{
"name": "seek",
- "execMs": 0.2275000000372529
+ "execMs": 0.23183299996890128
}
]
},
{
- "roundMs": 1494.313792000059,
+ "roundMs": 1818.3687919999938,
"firstDeltaMs": null,
"calls": [
{
"name": "set_color_hsl",
- "execMs": 0.015458000008948147
+ "execMs": 0.013999999966472387
}
]
},
{
- "roundMs": 1622.435083999997,
- "firstDeltaMs": 1369.4329999999609,
+ "roundMs": 2140.437583999941,
+ "firstDeltaMs": 1554.0435419999994,
"calls": []
}
]
@@ -8055,104 +7843,81 @@
{
"pass": true,
"intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "inspector",
+ "color",
+ "effects",
+ "image_gen"
+ ],
+ "declaredTools": 42,
+ "routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "seek",
- "round": 0,
- "atMs": 1876.1551660000114,
- "execMs": 0.01716599997598678
+ "round": 1,
+ "atMs": 3156.8108749999665,
+ "execMs": 0.014333999948576093,
+ "args": {
+ "t": 0
+ }
},
{
"name": "set_color_hsl",
- "round": 0,
- "atMs": 1876.5840409999946,
- "execMs": 0.007624999969266355
+ "round": 2,
+ "atMs": 5049.040666999994,
+ "execMs": 0.01362500002142042,
+ "args": {
+ "hue": 20,
+ "sat": 25,
+ "band": "blue",
+ "clipId": "c1"
+ }
}
],
"timings": {
- "gateMs": 1685.6962500000373,
+ "gateMs": 300.2105000000447,
"gateSkipped": false,
- "ttftMs": 3437.4552910000784,
- "modelMs": 3482.741874999949,
- "toolMs": 0.024790999945253134,
- "totalMs": 3491.8295830000425,
- "judgeMs": 993.7518339999951,
+ "ttftMs": 6287.196249999921,
+ "modelMs": 6418.930500999908,
+ "toolMs": 0.027958999969996512,
+ "totalMs": 6725.693250000011,
+ "judgeMs": 541.2030830000294,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1870.9716669999762,
+ "roundMs": 1158.9759589999449,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 1693.9996249999385,
"firstDeltaMs": null,
"calls": [
{
"name": "seek",
- "execMs": 0.01716599997598678
- },
- {
- "name": "set_color_hsl",
- "execMs": 0.007624999969266355
+ "execMs": 0.014333999948576093
}
]
},
{
- "roundMs": 1611.7702079999726,
- "firstDeltaMs": 1558.1464169999817,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "seek",
- "round": 0,
- "atMs": 1570.2511249999516,
- "execMs": 0.031208000029437244
- },
- {
- "name": "set_color_hsl",
- "round": 0,
- "atMs": 1570.6037500000093,
- "execMs": 0.006874999962747097
- }
- ],
- "timings": {
- "gateMs": 1102.9493329999968,
- "gateSkipped": false,
- "ttftMs": 2798.27329199994,
- "modelMs": 2981.6024579999503,
- "toolMs": 0.03808299999218434,
- "totalMs": 2988.01791699999,
- "judgeMs": 983.8525000000373,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1566.9772080000257,
+ "roundMs": 1890.6579589999747,
"firstDeltaMs": null,
"calls": [
- {
- "name": "seek",
- "execMs": 0.031208000029437244
- },
{
"name": "set_color_hsl",
- "execMs": 0.006874999962747097
+ "execMs": 0.01362500002142042
}
]
},
{
- "roundMs": 1414.6252499999246,
- "firstDeltaMs": 1225.16949999996,
+ "roundMs": 1675.29695800005,
+ "firstDeltaMs": 1236.9017080001067,
"calls": []
}
]
@@ -8167,44 +7932,55 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
+ "p50": 255,
+ "p95": 264,
+ "mean": 259,
+ "n": 2
},
"ttftMs": {
- "p50": 5515,
- "p95": 28804,
- "mean": 12816,
- "n": 3
+ "p50": 10077,
+ "p95": 20151,
+ "mean": 15114,
+ "n": 2
},
"totalMs": {
- "p50": 5760,
- "p95": 28953,
- "mean": 12985,
- "n": 3
+ "p50": 10240,
+ "p95": 20436,
+ "mean": 15338,
+ "n": 2
},
"modelMs": {
- "p50": 5751,
- "p95": 28944,
- "mean": 12977,
- "n": 3
+ "p50": 9969,
+ "p95": 20176,
+ "mean": 15072,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 3
}
},
"runs": [
{
"pass": true,
"intent": "complex",
+ "skill": null,
+ "areas": [
+ "preview",
+ "inspector",
+ "color",
+ "effects",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 52,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
@@ -8212,34 +7988,57 @@
{
"name": "match_color_grade",
"round": 0,
- "atMs": 26791.38537500007,
- "execMs": 0.13295799994375557
+ "atMs": 5722.50245800009,
+ "execMs": 0.23270799999590963,
+ "args": {
+ "ref_asset_id": "a-i1",
+ "clipId": "c1"
+ }
+ },
+ {
+ "name": "read_color_stats",
+ "round": 1,
+ "atMs": 8448.422292000032,
+ "execMs": 0.053958000033162534,
+ "args": {
+ "clipId": "c1"
+ }
}
],
"timings": {
- "gateMs": 0.23025000002235174,
+ "gateMs": 263.6963749999413,
"gateSkipped": true,
- "ttftMs": 28804.41766600008,
- "modelMs": 28943.838249999913,
- "toolMs": 0.13295799994375557,
- "totalMs": 28953.002541000023,
- "judgeMs": 1152.6302910000086,
+ "ttftMs": 10076.608708000043,
+ "modelMs": 9968.938291000086,
+ "toolMs": 0.28666600002907217,
+ "totalMs": 10239.683917000075,
+ "judgeMs": 499.6667080000043,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 26785.220708000008,
+ "roundMs": 5455.68979199999,
"firstDeltaMs": null,
"calls": [
{
"name": "match_color_grade",
- "execMs": 0.13295799994375557
+ "execMs": 0.23270799999590963
+ }
+ ]
+ },
+ {
+ "roundMs": 2723.8698330000043,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "read_color_stats",
+ "execMs": 0.053958000033162534
}
]
},
{
- "roundMs": 2158.6175419999054,
- "firstDeltaMs": 2010.3386249999749,
+ "roundMs": 1789.3786660000915,
+ "firstDeltaMs": 1626.7485410000663,
"calls": []
}
]
@@ -8248,6 +8047,17 @@
{
"pass": true,
"intent": "complex",
+ "skill": null,
+ "areas": [
+ "preview",
+ "inspector",
+ "color",
+ "effects",
+ "side_panel",
+ "top_bar"
+ ],
+ "declaredTools": 52,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
"judgeNote": null,
@@ -8255,34 +8065,233 @@
{
"name": "match_color_grade",
"round": 0,
- "atMs": 3402.7044590000296,
- "execMs": 0.03708299994468689
+ "atMs": 16618.874959000037,
+ "execMs": 0.015000000013969839,
+ "args": {
+ "ref_asset_id": "a-i1",
+ "clipId": "c1"
+ }
+ },
+ {
+ "name": "read_color_stats",
+ "round": 1,
+ "atMs": 18550.126249999972,
+ "execMs": 0.008917000028304756,
+ "args": {
+ "clipId": "c1"
+ }
}
],
"timings": {
- "gateMs": 0.41433299996424466,
+ "gateMs": 255.05629099998623,
"gateSkipped": true,
- "ttftMs": 5515.280000000028,
- "modelMs": 5751.399750000099,
- "toolMs": 0.03708299994468689,
- "totalMs": 5760.000209000078,
- "judgeMs": 1124.0259589999914,
+ "ttftMs": 20151.05587499996,
+ "modelMs": 20175.66770799982,
+ "toolMs": 0.023917000042274594,
+ "totalMs": 20436.48333399999,
+ "judgeMs": 523.6447499999776,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3397.4130000000587,
+ "roundMs": 16361.533707999974,
"firstDeltaMs": null,
"calls": [
{
"name": "match_color_grade",
- "execMs": 0.03708299994468689
+ "execMs": 0.015000000013969839
+ }
+ ]
+ },
+ {
+ "roundMs": 1929.4221249999246,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "read_color_stats",
+ "execMs": 0.008917000028304756
+ }
+ ]
+ },
+ {
+ "roundMs": 1884.7118749999208,
+ "firstDeltaMs": 1599.4977499999804,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "replicate-link-reads-then-replicates",
+ "bucket": "multi-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 256,
+ "p95": 275,
+ "mean": 265,
+ "n": 2
+ },
+ "ttftMs": {
+ "p50": 13186,
+ "p95": 13852,
+ "mean": 13519,
+ "n": 2
+ },
+ "totalMs": {
+ "p50": 13680,
+ "p95": 14212,
+ "mean": 13946,
+ "n": 2
+ },
+ "modelMs": {
+ "p50": 13386,
+ "p95": 13935,
+ "mean": 13660,
+ "n": 2
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 1,
+ "mean": 1,
+ "n": 2
+ },
+ "rounds": {
+ "mean": 5.5,
+ "max": 6
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "complex",
+ "skill": "replicating-a-project",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "removal",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "video_gen",
+ "scene",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 121,
+ "routingMisses": [],
+ "roundModel": "gemini-3.8-flash",
+ "notes": [],
+ "judgeNote": "upsell 0.54",
+ "trace": [
+ {
+ "name": "read_project",
+ "round": 1,
+ "atMs": 7373.890207999968,
+ "execMs": 0.689790999982506,
+ "args": {
+ "link": "https://donkeycut.com/app/p/ref-123"
+ }
+ },
+ {
+ "name": "replicate_project",
+ "round": 2,
+ "atMs": 9963.025707999943,
+ "execMs": 0.1279169999761507,
+ "args": {
+ "link": "https://donkeycut.com/app/p/ref-123",
+ "media": [
+ {
+ "source_asset_id": "r-v1",
+ "asset_id": "a-v1"
+ },
+ {
+ "asset_id": "a-v2",
+ "source_asset_id": "r-v2"
+ }
+ ]
+ }
+ },
+ {
+ "name": "update_overlay",
+ "round": 3,
+ "atMs": 11836.166540999897,
+ "execMs": 0.03116699995007366,
+ "args": {
+ "id": "t-n1",
+ "text": "MY DESK SETUP"
+ }
+ },
+ {
+ "name": "subtitles_generate",
+ "round": 3,
+ "atMs": 11836.586374999955,
+ "execMs": 0.03554099996108562,
+ "args": {}
+ }
+ ],
+ "timings": {
+ "gateMs": 255.5660420000786,
+ "gateSkipped": false,
+ "ttftMs": 13851.970332999947,
+ "modelMs": 13934.569001000025,
+ "toolMs": 0.884415999869816,
+ "totalMs": 14211.674124999903,
+ "judgeMs": 628.9623330000322,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 4637.172499999986,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 2474.225375000038,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "read_project",
+ "execMs": 0.689790999982506
+ }
+ ]
+ },
+ {
+ "roundMs": 2583.4949999999953,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "replicate_project",
+ "execMs": 0.1279169999761507
+ }
+ ]
+ },
+ {
+ "roundMs": 1867.8827089999104,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_overlay",
+ "execMs": 0.03116699995007366
+ },
+ {
+ "name": "subtitles_generate",
+ "execMs": 0.03554099996108562
}
]
},
{
- "roundMs": 2353.98675000004,
- "firstDeltaMs": 2109.750791000086,
+ "roundMs": 2371.7934170000954,
+ "firstDeltaMs": 2013.3481250000186,
"calls": []
}
]
@@ -8291,41 +8300,135 @@
{
"pass": true,
"intent": "complex",
+ "skill": "replicating-a-project",
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "effects",
+ "animation",
+ "transitions",
+ "subtitles",
+ "audio",
+ "video_gen",
+ "scene",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
+ ],
+ "declaredTools": 118,
+ "routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": null,
+ "judgeNote": "upsell 0.57",
"trace": [
{
- "name": "match_color_grade",
- "round": 0,
- "atMs": 2234.2474580000853,
- "execMs": 0.007416999898850918
+ "name": "read_project",
+ "round": 1,
+ "atMs": 4354.051875000005,
+ "execMs": 0.0339169999351725,
+ "args": {
+ "link": "https://donkeycut.com/app/p/ref-123"
+ }
+ },
+ {
+ "name": "replicate_project",
+ "round": 2,
+ "atMs": 7220.381125000073,
+ "execMs": 0.11149999999906868,
+ "args": {
+ "media": [
+ {
+ "source_asset_id": "r-v1",
+ "asset_id": "a-v1"
+ },
+ {
+ "asset_id": "a-v2",
+ "source_asset_id": "r-v2"
+ }
+ ],
+ "link": "https://donkeycut.com/app/p/ref-123"
+ }
+ },
+ {
+ "name": "update_overlay",
+ "round": 3,
+ "atMs": 9275.000750000007,
+ "execMs": 0.033500000019557774,
+ "args": {
+ "text": "MY DESK SETUP",
+ "id": "t-n1"
+ }
+ },
+ {
+ "name": "subtitles_generate",
+ "round": 4,
+ "atMs": 11213.23350000009,
+ "execMs": 0.03175000008195639,
+ "args": {}
}
],
"timings": {
- "gateMs": 0.33587499998975545,
- "gateSkipped": true,
- "ttftMs": 4128.475250000018,
- "modelMs": 4235.808000000077,
- "toolMs": 0.007416999898850918,
- "totalMs": 4241.48458300007,
- "judgeMs": 932.2050410000375,
+ "gateMs": 275.12804199999664,
+ "gateSkipped": false,
+ "ttftMs": 13185.857250000001,
+ "modelMs": 13385.525876000058,
+ "toolMs": 0.21066700003575534,
+ "totalMs": 13680.230250000022,
+ "judgeMs": 590.8087079999968,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2230.3242500000633,
+ "roundMs": 1735.5736670000479,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 2335.4039169999305,
"firstDeltaMs": null,
"calls": [
{
- "name": "match_color_grade",
- "execMs": 0.007416999898850918
+ "name": "read_project",
+ "execMs": 0.0339169999351725
+ }
+ ]
+ },
+ {
+ "roundMs": 2863.430708000087,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "replicate_project",
+ "execMs": 0.11149999999906868
+ }
+ ]
+ },
+ {
+ "roundMs": 2051.527374999947,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_overlay",
+ "execMs": 0.033500000019557774
+ }
+ ]
+ },
+ {
+ "roundMs": 1935.511792000034,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "subtitles_generate",
+ "execMs": 0.03175000008195639
}
]
},
{
- "roundMs": 2005.483750000014,
- "firstDeltaMs": 1892.9491670000134,
+ "roundMs": 2464.0784170000115,
+ "firstDeltaMs": 1970.1614590000827,
"calls": []
}
]