Skip to content

Open source CapCut alternative.

TypeScript

chore(evals): refresh the Cut chat latency baseline on the reconciled cases

David Le62a2ee3e0121
diff --git a/evals/cut-chat-latency.latest-report.json b/evals/cut-chat-latency.latest-report.json
index 3d4c9fdf..e70e56e1 100644
--- a/evals/cut-chat-latency.latest-report.json
+++ b/evals/cut-chat-latency.latest-report.json
@@ -1,6 +1,6 @@
{
"schema": "cut-chat-latency/v1",
- "generatedAt": "2026-08-04T06:28:10.472Z",
+ "generatedAt": "2026-08-04T12:45:08.327Z",
"base": "http://localhost:3000",
"runsPerCase": 3,
"configs": [
@@ -13,70 +13,70 @@
"cases": 7,
"passRate": 1,
"gateMs": {
- "p50": 1283,
- "p95": 1778,
- "mean": 976,
+ "p50": 1160,
+ "p95": 1509,
+ "mean": 910,
"n": 7
},
"ttftMs": {
- "p50": 4896,
- "p95": 6967,
- "mean": 5020,
+ "p50": 4978,
+ "p95": 17580,
+ "mean": 6277,
"n": 7
},
"totalMs": {
- "p50": 6272,
- "p95": 7449,
- "mean": 5517,
+ "p50": 5453,
+ "p95": 14566,
+ "mean": 6154,
"n": 7
},
- "meanRounds": 1.1
+ "meanRounds": 1.4
},
"multi-tool": {
"cases": 4,
- "passRate": 0.5,
+ "passRate": 0.83,
"gateMs": {
"p50": 0,
- "p95": 1880,
- "mean": 940,
- "n": 2
+ "p95": 1391,
+ "mean": 625,
+ "n": 4
},
"ttftMs": {
- "p50": 10835,
- "p95": 26244,
- "mean": 18540,
- "n": 2
+ "p50": 15413,
+ "p95": 67131,
+ "mean": 28984,
+ "n": 4
},
"totalMs": {
- "p50": 11153,
- "p95": 26737,
- "mean": 18945,
- "n": 2
+ "p50": 15725,
+ "p95": 67381,
+ "mean": 29249,
+ "n": 4
},
- "meanRounds": 4.5
+ "meanRounds": 7.8
},
"single-tool": {
"cases": 15,
- "passRate": 0.78,
+ "passRate": 0.93,
"gateMs": {
- "p50": 1250,
- "p95": 1836,
- "mean": 1206,
- "n": 12
+ "p50": 1259,
+ "p95": 1822,
+ "mean": 1233,
+ "n": 15
},
"ttftMs": {
- "p50": 7194,
- "p95": 11263,
- "mean": 7926,
- "n": 12
+ "p50": 8238,
+ "p95": 11456,
+ "mean": 8346,
+ "n": 15
},
"totalMs": {
- "p50": 7376,
- "p95": 11637,
- "mean": 8270,
- "n": 12
+ "p50": 8410,
+ "p95": 11649,
+ "mean": 8651,
+ "n": 15
},
- "meanRounds": 2.3
+ "meanRounds": 2.2
}
},
"cases": [
@@ -93,21 +93,21 @@
"n": 3
},
"ttftMs": {
- "p50": 6967,
- "p95": 7318,
- "mean": 5894,
+ "p50": 6149,
+ "p95": 8577,
+ "mean": 6393,
"n": 3
},
"totalMs": {
- "p50": 7299,
- "p95": 7562,
- "mean": 6185,
+ "p50": 6748,
+ "p95": 8789,
+ "mean": 6698,
"n": 3
},
"modelMs": {
- "p50": 7298,
- "p95": 7562,
- "mean": 6185,
+ "p50": 6748,
+ "p95": 8789,
+ "mean": 6698,
"n": 3
},
"toolMs": {
@@ -130,16 +130,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 7317.646374999999,
- "modelMs": 7561.652625,
+ "ttftMs": 8576.98725,
+ "modelMs": 8789.334125000001,
"toolMs": 0,
- "totalMs": 7561.652625,
+ "totalMs": 8789.334125000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7561.652625,
- "firstDeltaMs": 7316.949917000001,
+ "roundMs": 8789.334125000001,
+ "firstDeltaMs": 8576.560417,
"calls": []
}
]
@@ -149,20 +149,37 @@
"pass": true,
"intent": "work",
"notes": [],
- "trace": [],
+ "trace": [
+ {
+ "name": "listen_audio",
+ "round": 0,
+ "atMs": 4096.202207999999,
+ "execMs": 0.3234169999996084
+ }
+ ],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3396.2884160000012,
- "modelMs": 3694.4401670000007,
- "toolMs": 0,
- "totalMs": 3694.4401670000007,
+ "ttftMs": 6148.970374999999,
+ "modelMs": 6747.897542000001,
+ "toolMs": 0.3234169999996084,
+ "totalMs": 6748.220959,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3694.4401670000007,
- "firstDeltaMs": 3396.2415839999994,
+ "roundMs": 4095.9574580000008,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.3234169999996084
+ }
+ ]
+ },
+ {
+ "roundMs": 2651.940084,
+ "firstDeltaMs": 2052.3677499999994,
"calls": []
}
]
@@ -172,37 +189,20 @@
"pass": true,
"intent": "work",
"notes": [],
- "trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 4223.3645830000005,
- "execMs": 0.0865830000002461
- }
- ],
+ "trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6966.797832999999,
- "modelMs": 7298.473457999999,
- "toolMs": 0.0865830000002461,
- "totalMs": 7298.560040999999,
+ "ttftMs": 4452.217624999999,
+ "modelMs": 4557.717500000001,
+ "toolMs": 0,
+ "totalMs": 4557.717500000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4221.189082999999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.0865830000002461
- }
- ]
- },
- {
- "roundMs": 3077.2843749999993,
- "firstDeltaMs": 2742.5914999999986,
+ "roundMs": 4557.717500000001,
+ "firstDeltaMs": 4450.335540999999,
"calls": []
}
]
@@ -217,27 +217,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1157,
- "p95": 1417,
- "mean": 1209,
+ "p50": 1509,
+ "p95": 1838,
+ "mean": 1545,
"n": 3
},
"ttftMs": {
- "p50": 2964,
- "p95": 3515,
- "mean": 3052,
+ "p50": 3463,
+ "p95": 3642,
+ "mean": 3430,
"n": 3
},
"totalMs": {
- "p50": 3124,
- "p95": 3540,
- "mean": 3122,
+ "p50": 3668,
+ "p95": 3915,
+ "mean": 3655,
"n": 3
},
"modelMs": {
- "p50": 1967,
- "p95": 2123,
- "mean": 1913,
+ "p50": 2378,
+ "p95": 2407,
+ "mean": 2109,
"n": 3
},
"toolMs": {
@@ -258,18 +258,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1417.362041999997,
+ "gateMs": 1837.6699999999983,
"gateSkipped": false,
- "ttftMs": 3514.5605829999986,
- "modelMs": 2122.709749999998,
+ "ttftMs": 3185.6539159999993,
+ "modelMs": 1542.4246669999993,
"toolMs": 0,
- "totalMs": 3540.071791999995,
+ "totalMs": 3380.0946669999976,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2122.709749999998,
- "firstDeltaMs": 2096.1275419999984,
+ "roundMs": 1542.4246669999993,
+ "firstDeltaMs": 1347.8809999999976,
"calls": []
}
]
@@ -281,18 +281,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1156.7389169999988,
+ "gateMs": 1290.1574160000018,
"gateSkipped": false,
- "ttftMs": 2963.965083000003,
- "modelMs": 1966.931875000002,
+ "ttftMs": 3462.997625,
+ "modelMs": 2377.9761670000007,
"toolMs": 0,
- "totalMs": 3123.670792000001,
+ "totalMs": 3668.1335830000025,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1966.931875000002,
- "firstDeltaMs": 1807.130459,
+ "roundMs": 2377.9761670000007,
+ "firstDeltaMs": 2172.6522920000025,
"calls": []
}
]
@@ -304,18 +304,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1053.4215000000004,
+ "gateMs": 1508.6002919999992,
"gateSkipped": false,
- "ttftMs": 2676.7100420000024,
- "modelMs": 1648.9532910000016,
+ "ttftMs": 3641.7858749999978,
+ "modelMs": 2406.833541,
"toolMs": 0,
- "totalMs": 2702.374791000002,
+ "totalMs": 3915.433832999999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1648.9532910000016,
- "firstDeltaMs": 1623.111541000002,
+ "roundMs": 2406.833541,
+ "firstDeltaMs": 2132.990249999999,
"calls": []
}
]
@@ -330,27 +330,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1302,
- "p95": 1397,
- "mean": 1270,
+ "p50": 1076,
+ "p95": 1094,
+ "mean": 1049,
"n": 3
},
"ttftMs": {
- "p50": 3283,
- "p95": 3292,
- "mean": 3152,
+ "p50": 2885,
+ "p95": 4510,
+ "mean": 3288,
"n": 3
},
"totalMs": {
- "p50": 3461,
- "p95": 3496,
- "mean": 3358,
+ "p50": 2951,
+ "p95": 4867,
+ "mean": 3526,
"n": 3
},
"modelMs": {
- "p50": 2064,
- "p95": 2385,
- "mean": 2088,
+ "p50": 1857,
+ "p95": 3890,
+ "mean": 2477,
"n": 3
},
"toolMs": {
@@ -371,18 +371,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1302.431457999999,
+ "gateMs": 977.4157909999994,
"gateSkipped": false,
- "ttftMs": 2882.048708000002,
- "modelMs": 1815.6380419999987,
+ "ttftMs": 4510.370917000004,
+ "modelMs": 3889.548999999999,
"toolMs": 0,
- "totalMs": 3118.069499999998,
+ "totalMs": 4866.9647909999985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1815.6380419999987,
- "firstDeltaMs": 1579.4611670000013,
+ "roundMs": 3889.548999999999,
+ "firstDeltaMs": 3532.399666999998,
"calls": []
}
]
@@ -394,18 +394,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1396.850750000005,
+ "gateMs": 1093.682667000001,
"gateSkipped": false,
- "ttftMs": 3282.873709000003,
- "modelMs": 2063.7577919999967,
+ "ttftMs": 2884.7979169999962,
+ "modelMs": 1856.9127500000031,
"toolMs": 0,
- "totalMs": 3460.6085420000018,
+ "totalMs": 2950.595417000004,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2063.7577919999967,
- "firstDeltaMs": 1885.7852090000015,
+ "roundMs": 1856.9127500000031,
+ "firstDeltaMs": 1790.9594169999982,
"calls": []
}
]
@@ -417,18 +417,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1111.103750000002,
+ "gateMs": 1076.224749999994,
"gateSkipped": false,
- "ttftMs": 3292.4117499999993,
- "modelMs": 2385.2409999999945,
+ "ttftMs": 2469.644207999998,
+ "modelMs": 1683.3724999999977,
"toolMs": 0,
- "totalMs": 3496.3447499999966,
+ "totalMs": 2759.5972499999916,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2385.2409999999945,
- "firstDeltaMs": 2181.222750000001,
+ "roundMs": 1683.3724999999977,
+ "firstDeltaMs": 1393.2937079999974,
"calls": []
}
]
@@ -443,32 +443,32 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1880,
- "p95": 1931,
- "mean": 1618,
+ "p50": 1107,
+ "p95": 1212,
+ "mean": 1110,
"n": 3
},
"ttftMs": {
- "p50": 10835,
- "p95": 12235,
- "mean": 10444,
+ "p50": 9768,
+ "p95": 11022,
+ "mean": 9704,
"n": 3
},
"totalMs": {
- "p50": 11153,
- "p95": 12579,
- "mean": 10846,
+ "p50": 10002,
+ "p95": 11357,
+ "mean": 10076,
"n": 3
},
"modelMs": {
- "p50": 10109,
- "p95": 10699,
- "mean": 9227,
+ "p50": 8789,
+ "p95": 10250,
+ "mean": 8966,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 2,
"mean": 1,
"n": 3
},
@@ -486,43 +486,43 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 8550.342958000001,
- "execMs": 1.4473749999961
+ "atMs": 8153.245041000002,
+ "execMs": 1.6797500000029686
},
{
"name": "update_cue",
"round": 0,
- "atMs": 8551.833208000004,
- "execMs": 0.019500000002153683
+ "atMs": 8154.971040999997,
+ "execMs": 0.02033300000039162
}
],
"timings": {
- "gateMs": 1042.5413340000014,
+ "gateMs": 1211.6982919999937,
"gateSkipped": false,
- "ttftMs": 10834.595625000002,
- "modelMs": 10109.382084000004,
- "toolMs": 1.4668749999982538,
- "totalMs": 11153.390293000004,
+ "ttftMs": 9767.803,
+ "modelMs": 8788.785415999992,
+ "toolMs": 1.7000830000033602,
+ "totalMs": 10002.183790999989,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7506.233375000003,
+ "roundMs": 6940.052624999997,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 1.4473749999961
+ "execMs": 1.6797500000029686
},
{
"name": "update_cue",
- "execMs": 0.019500000002153683
+ "execMs": 0.02033300000039162
}
]
},
{
- "roundMs": 2603.148709000001,
- "firstDeltaMs": 2282.694792000002,
+ "roundMs": 1848.732790999995,
+ "firstDeltaMs": 1612.7021249999962,
"calls": []
}
]
@@ -536,43 +536,43 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 6873.603999999999,
- "execMs": 0.010541000003286172
+ "atMs": 6914.115749999997,
+ "execMs": 0.008416000004217494
},
{
"name": "update_cue",
"round": 0,
- "atMs": 6873.6163339999985,
- "execMs": 0.003125000002910383
+ "atMs": 6914.130624999998,
+ "execMs": 0.0030830000032437965
}
],
"timings": {
- "gateMs": 1930.8417910000062,
+ "gateMs": 1009.8917920000022,
"gateSkipped": false,
- "ttftMs": 8263.247625000004,
- "modelMs": 6874.199374000003,
- "toolMs": 0.013666000006196555,
- "totalMs": 8805.054831000016,
+ "ttftMs": 8323.100833999997,
+ "modelMs": 7857.6472090000025,
+ "toolMs": 0.01149900000746129,
+ "totalMs": 8867.550500000012,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4942.393291,
+ "roundMs": 5903.6918339999975,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.010541000003286172
+ "execMs": 0.008416000004217494
},
{
"name": "update_cue",
- "execMs": 0.003125000002910383
+ "execMs": 0.0030830000032437965
}
]
},
{
- "roundMs": 1931.8060830000031,
- "firstDeltaMs": 1389.6195420000004,
+ "roundMs": 1953.955375000005,
+ "firstDeltaMs": 1408.9313750000001,
"calls": []
}
]
@@ -586,43 +586,43 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 7491.459458000005,
- "execMs": 0.05662499999743886
+ "atMs": 9448.869582999992,
+ "execMs": 0.06337499999790452
},
{
"name": "update_cue",
"round": 0,
- "atMs": 7491.529457999997,
- "execMs": 0.010750000001280569
+ "atMs": 9448.936583000002,
+ "execMs": 0.003875000009429641
}
],
"timings": {
- "gateMs": 1880.2253750000018,
+ "gateMs": 1107.188457999997,
"gateSkipped": false,
- "ttftMs": 12234.916541000006,
- "modelMs": 10698.84599999999,
- "toolMs": 0.06737499999871943,
- "totalMs": 12579.138749999991,
+ "ttftMs": 11021.766833000001,
+ "modelMs": 10250.153165999996,
+ "toolMs": 0.06725000000733417,
+ "totalMs": 11357.408874,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5610.801332999996,
+ "roundMs": 8341.394458000002,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.05662499999743886
+ "execMs": 0.06337499999790452
},
{
"name": "update_cue",
- "execMs": 0.010750000001280569
+ "execMs": 0.003875000009429641
}
]
},
{
- "roundMs": 5088.044666999995,
- "firstDeltaMs": 4743.32704199999,
+ "roundMs": 1908.758707999994,
+ "firstDeltaMs": 1572.7940830000007,
"calls": []
}
]
@@ -637,27 +637,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1283,
- "p95": 1823,
- "mean": 1443,
+ "p50": 1191,
+ "p95": 1706,
+ "mean": 1352,
"n": 3
},
"ttftMs": {
- "p50": 3897,
- "p95": 4374,
- "mean": 3852,
+ "p50": 3868,
+ "p95": 4119,
+ "mean": 3836,
"n": 3
},
"totalMs": {
- "p50": 4296,
- "p95": 4909,
- "mean": 4263,
+ "p50": 4078,
+ "p95": 4614,
+ "mean": 4114,
"n": 3
},
"modelMs": {
- "p50": 3012,
- "p95": 3086,
- "mean": 2820,
+ "p50": 2908,
+ "p95": 2920,
+ "mean": 2762,
"n": 3
},
"toolMs": {
@@ -678,18 +678,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1223.9884170000005,
+ "gateMs": 1705.7851249999949,
"gateSkipped": false,
- "ttftMs": 3284.3948329999985,
- "modelMs": 2361.609792000003,
+ "ttftMs": 4118.664791999996,
+ "modelMs": 2907.916041999997,
"toolMs": 0,
- "totalMs": 3585.5982090000034,
+ "totalMs": 4613.701166999992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2361.609792000003,
- "firstDeltaMs": 2059.8249170000054,
+ "roundMs": 2907.916041999997,
+ "firstDeltaMs": 2412.4086249999964,
"calls": []
}
]
@@ -701,18 +701,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1822.8158339999936,
+ "gateMs": 1157.9814999999944,
"gateSkipped": false,
- "ttftMs": 4373.996333999996,
- "modelMs": 3086.1554580000084,
+ "ttftMs": 3868.436167000007,
+ "modelMs": 2919.821916999994,
"toolMs": 0,
- "totalMs": 4908.971292000002,
+ "totalMs": 4077.8034169999883,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3086.1554580000084,
- "firstDeltaMs": 2550.989625000002,
+ "roundMs": 2919.821916999994,
+ "firstDeltaMs": 2710.1762080000044,
"calls": []
}
]
@@ -724,18 +724,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1283.1000000000058,
+ "gateMs": 1191.4955419999897,
"gateSkipped": false,
- "ttftMs": 3897.253874999995,
- "modelMs": 3012.4396669999987,
+ "ttftMs": 3520.0412919999944,
+ "modelMs": 2458.8083749999932,
"toolMs": 0,
- "totalMs": 4295.539667000005,
+ "totalMs": 3650.303916999983,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3012.4396669999987,
- "firstDeltaMs": 2613.824708,
+ "roundMs": 2458.8083749999932,
+ "firstDeltaMs": 2328.29849999999,
"calls": []
}
]
@@ -756,21 +756,21 @@
"n": 3
},
"ttftMs": {
- "p50": 26244,
- "p95": 28291,
- "mean": 24255,
+ "p50": 15413,
+ "p95": 17166,
+ "mean": 15380,
"n": 3
},
"totalMs": {
- "p50": 26737,
- "p95": 28864,
- "mean": 24724,
+ "p50": 15725,
+ "p95": 17706,
+ "mean": 15810,
"n": 3
},
"modelMs": {
- "p50": 26737,
- "p95": 28864,
- "mean": 24724,
+ "p50": 15725,
+ "p95": 17706,
+ "mean": 15810,
"n": 3
},
"toolMs": {
@@ -780,8 +780,8 @@
"n": 3
},
"rounds": {
- "mean": 7,
- "max": 9
+ "mean": 3,
+ "max": 4
}
},
"runs": [
@@ -793,75 +793,75 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 6369.322457999995,
- "execMs": 0.002542000001994893
+ "atMs": 5399.164292000001,
+ "execMs": 0.008541999995941296
},
{
"name": "add_clip",
"round": 0,
- "atMs": 6369.334415999998,
- "execMs": 0.0044999999954598024
+ "atMs": 5399.222542000003,
+ "execMs": 0.014292000007117167
},
{
"name": "get_state",
"round": 1,
- "atMs": 8814.291165999995,
- "execMs": 0.02412499999627471
+ "atMs": 8805.871375000002,
+ "execMs": 0.033624999996391125
},
{
"name": "get_state",
"round": 2,
- "atMs": 12863.168250000002,
- "execMs": 0.011916999996174127
+ "atMs": 12627.986542000013,
+ "execMs": 0.05674999998882413
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 18231.080291000006,
- "modelMs": 18572.188916999992,
- "toolMs": 0.04308399998990353,
- "totalMs": 18572.232000999982,
+ "ttftMs": 15413.465208000009,
+ "modelMs": 15725.282208000004,
+ "toolMs": 0.11320899998827372,
+ "totalMs": 15725.395416999992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6369.018249999994,
+ "roundMs": 5398.719041999997,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.002542000001994893
+ "execMs": 0.008541999995941296
},
{
"name": "add_clip",
- "execMs": 0.0044999999954598024
+ "execMs": 0.014292000007117167
}
]
},
{
- "roundMs": 2444.9239579999994,
+ "roundMs": 3406.529290999999,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.02412499999627471
+ "execMs": 0.033624999996391125
}
]
},
{
- "roundMs": 4048.8317499999976,
+ "roundMs": 3821.9903750000085,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.011916999996174127
+ "execMs": 0.05674999998882413
}
]
},
{
- "roundMs": 5709.4149590000015,
- "firstDeltaMs": 5367.890417000002,
+ "roundMs": 3098.0434999999998,
+ "firstDeltaMs": 2785.387042000002,
"calls": []
}
]
@@ -875,139 +875,43 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 6046.518291999993,
- "execMs": 0.0029999999969732016
+ "atMs": 6072.533167000001,
+ "execMs": 0.0020419999927980825
},
{
"name": "add_clip",
"round": 0,
- "atMs": 6046.535334,
- "execMs": 8.400001388508826e-05
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 8381.825500000006,
- "execMs": 0.013915999996243045
- },
- {
- "name": "get_state",
- "round": 2,
- "atMs": 12420.984125000003,
- "execMs": 0.00779099999635946
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 16006.751999999993,
- "execMs": 0.008000000001629815
- },
- {
- "name": "get_state",
- "round": 4,
- "atMs": 17800.151625,
- "execMs": 0.01216700000804849
- },
- {
- "name": "add_clip",
- "round": 5,
- "atMs": 20630.7595,
- "execMs": 0.02195900000515394
- },
- {
- "name": "get_state",
- "round": 6,
- "atMs": 22788.909125000006,
- "execMs": 0.03695799999695737
+ "atMs": 6072.547166999997,
+ "execMs": 0.00408400000014808
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 26243.687334000002,
- "modelMs": 26736.910291000007,
- "toolMs": 0.10387500001525041,
- "totalMs": 26737.014166000023,
+ "ttftMs": 13562.378583999991,
+ "modelMs": 13999.371750000006,
+ "toolMs": 0.006125999992946163,
+ "totalMs": 13999.377875999999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6046.177916999994,
+ "roundMs": 6072.236667000005,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.0029999999969732016
+ "execMs": 0.0020419999927980825
},
{
"name": "add_clip",
- "execMs": 8.400001388508826e-05
- }
- ]
- },
- {
- "roundMs": 2335.1770830000023,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.013915999996243045
- }
- ]
- },
- {
- "roundMs": 4039.067750000002,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.00779099999635946
- }
- ]
- },
- {
- "roundMs": 3585.5293749999983,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.008000000001629815
- }
- ]
- },
- {
- "roundMs": 1793.2787910000043,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.01216700000804849
- }
- ]
- },
- {
- "roundMs": 2830.5346250000002,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.02195900000515394
- }
- ]
- },
- {
- "roundMs": 2158.058917000002,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.03695799999695737
+ "execMs": 0.00408400000014808
}
]
},
{
- "roundMs": 3949.0858330000046,
- "firstDeltaMs": 3454.704291000002,
+ "roundMs": 7927.135083000001,
+ "firstDeltaMs": 7489.80745800001,
"calls": []
}
]
@@ -1021,155 +925,59 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 8206.494416999994,
- "execMs": 0.010250000021187589
+ "atMs": 6175.882874999996,
+ "execMs": 0.0012080000014975667
},
{
"name": "add_clip",
"round": 0,
- "atMs": 8206.55858299999,
- "execMs": 0.017041999992216006
+ "atMs": 6175.888124999998,
+ "execMs": 0.00012499999138526618
},
{
"name": "get_state",
"round": 1,
- "atMs": 10786.695041999992,
- "execMs": 0.02704099999391474
- },
- {
- "name": "get_state",
- "round": 2,
- "atMs": 14556.301166999983,
- "execMs": 0.021999999997206032
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 16613.332582999996,
- "execMs": 0.003333999979076907
- },
- {
- "name": "add_clip",
- "round": 4,
- "atMs": 18780.120791999987,
- "execMs": 0.003208999987691641
- },
- {
- "name": "get_state",
- "round": 5,
- "atMs": 20698.226833000008,
- "execMs": 0.02374999999301508
- },
- {
- "name": "add_clip",
- "round": 6,
- "atMs": 23335.387499999997,
- "execMs": 0.00108299998100847
- },
- {
- "name": "get_state",
- "round": 7,
- "atMs": 24980.632792000004,
- "execMs": 0.021958000026643276
+ "atMs": 9873.047166999997,
+ "execMs": 0.06841599999461323
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 28290.836874999994,
- "modelMs": 28863.655414000023,
- "toolMs": 0.12966699997195974,
- "totalMs": 28863.785080999995,
+ "ttftMs": 17165.5665,
+ "modelMs": 17706.077375000008,
+ "toolMs": 0.06974899998749606,
+ "totalMs": 17706.147123999996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8206.179541000005,
+ "roundMs": 6175.771083,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.010250000021187589
+ "execMs": 0.0012080000014975667
},
{
"name": "add_clip",
- "execMs": 0.017041999992216006
- }
- ]
- },
- {
- "roundMs": 2579.939291000017,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.02704099999391474
- }
- ]
- },
- {
- "roundMs": 3769.4990830000024,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.021999999997206032
- }
- ]
- },
- {
- "roundMs": 2056.9160830000183,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.003333999979076907
- }
- ]
- },
- {
- "roundMs": 2166.6990409999853,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.003208999987691641
- }
- ]
- },
- {
- "roundMs": 1917.98545800001,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.02374999999301508
- }
- ]
- },
- {
- "roundMs": 2637.072749999992,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.00108299998100847
+ "execMs": 0.00012499999138526618
}
]
},
{
- "roundMs": 1645.1913749999949,
+ "roundMs": 3697.1059169999935,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.021958000026643276
+ "execMs": 0.06841599999461323
}
]
},
{
- "roundMs": 3884.1727919999976,
- "firstDeltaMs": 3310.022833999974,
+ "roundMs": 7833.200375000015,
+ "firstDeltaMs": 7292.414749999996,
"calls": []
}
]
@@ -1190,21 +998,21 @@
"n": 3
},
"ttftMs": {
- "p50": 4896,
- "p95": 8122,
- "mean": 5673,
+ "p50": 5018,
+ "p95": 5102,
+ "mean": 4777,
"n": 3
},
"totalMs": {
- "p50": 6272,
- "p95": 9475,
- "mean": 6912,
+ "p50": 5453,
+ "p95": 5985,
+ "mean": 5610,
"n": 3
},
"modelMs": {
- "p50": 6272,
- "p95": 9475,
- "mean": 6912,
+ "p50": 5453,
+ "p95": 5985,
+ "mean": 5610,
"n": 3
},
"toolMs": {
@@ -1214,8 +1022,8 @@
"n": 3
},
"rounds": {
- "mean": 1.3,
- "max": 2
+ "mean": 1,
+ "max": 1
}
},
"runs": [
@@ -1227,16 +1035,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4896.08679099998,
- "modelMs": 6272.093250000005,
+ "ttftMs": 5102.391457999998,
+ "modelMs": 5453.402333999984,
"toolMs": 0,
- "totalMs": 6272.093250000005,
+ "totalMs": 5453.402333999984,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6272.093250000005,
- "firstDeltaMs": 4895.672374999995,
+ "roundMs": 5453.402333999984,
+ "firstDeltaMs": 5102.1967920000025,
"calls": []
}
]
@@ -1246,47 +1054,20 @@
"pass": true,
"intent": "work",
"notes": [],
- "trace": [
- {
- "name": "watch_video",
- "round": 0,
- "atMs": 4645.401290999987,
- "execMs": 0.04320799998822622
- },
- {
- "name": "watch_video",
- "round": 0,
- "atMs": 4645.464582999994,
- "execMs": 0.0016250000044237822
- }
- ],
+ "trace": [],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 8121.71854099998,
- "modelMs": 9475.248500000016,
- "toolMs": 0.04483299999265,
- "totalMs": 9475.293333000009,
+ "ttftMs": 4209.426999999996,
+ "modelMs": 5391.779332999984,
+ "toolMs": 0,
+ "totalMs": 5391.779332999984,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4645.252333000011,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.04320799998822622
- },
- {
- "name": "watch_video",
- "execMs": 0.0016250000044237822
- }
- ]
- },
- {
- "roundMs": 4829.996167000005,
- "firstDeltaMs": 3476.2197089999972,
+ "roundMs": 5391.779332999984,
+ "firstDeltaMs": 4209.119540999993,
"calls": []
}
]
@@ -1300,16 +1081,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4000.800042000017,
- "modelMs": 4987.1807090000075,
+ "ttftMs": 5017.784875000012,
+ "modelMs": 5984.525208000006,
"toolMs": 0,
- "totalMs": 4987.1807090000075,
+ "totalMs": 5984.525208000006,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4987.1807090000075,
- "firstDeltaMs": 4000.6248750000086,
+ "roundMs": 5984.525208000006,
+ "firstDeltaMs": 5017.720375000004,
"calls": []
}
]
@@ -1324,27 +1105,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1312,
- "p95": 1847,
- "mean": 1481,
+ "p50": 1160,
+ "p95": 1206,
+ "mean": 1107,
"n": 3
},
"ttftMs": {
- "p50": 6274,
- "p95": 10129,
- "mean": 7208,
+ "p50": 4978,
+ "p95": 5257,
+ "mean": 5017,
"n": 3
},
"totalMs": {
- "p50": 6715,
- "p95": 10601,
- "mean": 7712,
+ "p50": 5612,
+ "p95": 5894,
+ "mean": 5696,
"n": 3
},
"modelMs": {
- "p50": 5431,
- "p95": 8754,
- "mean": 6230,
+ "p50": 4659,
+ "p95": 4733,
+ "mean": 4590,
"n": 3
},
"toolMs": {
@@ -1365,18 +1146,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1284.2171669999952,
+ "gateMs": 1205.902333999984,
"gateSkipped": false,
- "ttftMs": 6273.943583000015,
- "modelMs": 5431.056582999998,
+ "ttftMs": 4977.504665999993,
+ "modelMs": 4376.710125000012,
"toolMs": 0,
- "totalMs": 6715.273749999993,
+ "totalMs": 5582.612458999996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5431.056582999998,
- "firstDeltaMs": 4989.362542000017,
+ "roundMs": 4376.710125000012,
+ "firstDeltaMs": 3771.0252080000064,
"calls": []
}
]
@@ -1388,18 +1169,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1312.3858330000076,
+ "gateMs": 953.5840000000026,
"gateSkipped": false,
- "ttftMs": 5220.442958,
- "modelMs": 4506.357250000001,
+ "ttftMs": 4817.078207999992,
+ "modelMs": 4658.516584000026,
"toolMs": 0,
- "totalMs": 5818.7430830000085,
+ "totalMs": 5612.100584000029,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4506.357250000001,
- "firstDeltaMs": 3907.693250000011,
+ "roundMs": 4658.516584000026,
+ "firstDeltaMs": 3863.340959000023,
"calls": []
}
]
@@ -1411,18 +1192,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1847.3271669999813,
+ "gateMs": 1160.252124999999,
"gateSkipped": false,
- "ttftMs": 10129.191374999995,
- "modelMs": 8753.937124999997,
+ "ttftMs": 5256.684749999986,
+ "modelMs": 4733.449541000009,
"toolMs": 0,
- "totalMs": 10601.264291999978,
+ "totalMs": 5893.701666000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8753.937124999997,
- "firstDeltaMs": 8281.66750000001,
+ "roundMs": 4733.449541000009,
+ "firstDeltaMs": 4096.169708000001,
"calls": []
}
]
@@ -1437,27 +1218,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1417,
- "p95": 1785,
- "mean": 1468,
+ "p50": 1272,
+ "p95": 1703,
+ "mean": 1330,
"n": 3
},
"ttftMs": {
- "p50": 8773,
- "p95": 8787,
- "mean": 8373,
+ "p50": 9003,
+ "p95": 12494,
+ "mean": 10100,
"n": 3
},
"totalMs": {
- "p50": 9296,
- "p95": 9579,
- "mean": 9020,
+ "p50": 9489,
+ "p95": 12843,
+ "mean": 10515,
"n": 3
},
"modelMs": {
- "p50": 7511,
- "p95": 8376,
- "mean": 7551,
+ "p50": 7943,
+ "p95": 11826,
+ "mean": 9185,
"n": 3
},
"toolMs": {
@@ -1480,33 +1261,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 5819.81708399998,
- "execMs": 0.05316700000548735
+ "atMs": 6709.2962919999845,
+ "execMs": 0.011499999993247911
}
],
"timings": {
- "gateMs": 1417.0378329999803,
+ "gateMs": 1271.8447499999893,
"gateSkipped": false,
- "ttftMs": 7559.620124999987,
- "modelMs": 6767.07033399996,
- "toolMs": 0.05316700000548735,
- "totalMs": 8184.161333999946,
+ "ttftMs": 8802.53566699999,
+ "modelMs": 7942.633458000026,
+ "toolMs": 0.011499999993247911,
+ "totalMs": 9214.489708000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4402.213583999983,
+ "roundMs": 5437.112875000021,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.05316700000548735
+ "execMs": 0.011499999993247911
}
]
},
{
- "roundMs": 2364.856749999977,
- "firstDeltaMs": 1739.71424999999,
+ "roundMs": 2505.520583000005,
+ "firstDeltaMs": 2093.1843749999825,
"calls": []
}
]
@@ -1520,33 +1301,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 6817.497666999989,
- "execMs": 0.010500000003958121
+ "atMs": 7074.677708000003,
+ "execMs": 0.023249999998370185
}
],
"timings": {
- "gateMs": 1202.928917000012,
+ "gateMs": 1702.866458000004,
"gateSkipped": false,
- "ttftMs": 8787.35483299999,
- "modelMs": 8375.692374999984,
- "toolMs": 0.010500000003958121,
- "totalMs": 9578.631792,
+ "ttftMs": 9003.127875000006,
+ "modelMs": 7785.776334000024,
+ "toolMs": 0.023249999998370185,
+ "totalMs": 9488.666042000026,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5614.409874999983,
+ "roundMs": 5371.4401670000225,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.010500000003958121
+ "execMs": 0.023249999998370185
}
]
},
{
- "roundMs": 2761.282500000001,
- "firstDeltaMs": 1969.8107080000045,
+ "roundMs": 2414.3361670000013,
+ "firstDeltaMs": 1928.379041999986,
"calls": []
}
]
@@ -1560,33 +1341,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 6900.531667000003,
- "execMs": 0.012042000016663224
+ "atMs": 10436.578915999999,
+ "execMs": 0.0034580000210553408
}
],
"timings": {
- "gateMs": 1785.08924999999,
+ "gateMs": 1016.3358339999977,
"gateSkipped": false,
- "ttftMs": 8772.587584000023,
- "modelMs": 7511.276333999995,
- "toolMs": 0.012042000016663224,
- "totalMs": 9296.377626000001,
+ "ttftMs": 12493.659500000009,
+ "modelMs": 11826.393541999976,
+ "toolMs": 0.0034580000210553408,
+ "totalMs": 12842.732833999995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5114.984541999991,
+ "roundMs": 9420.036791999999,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.012042000016663224
+ "execMs": 0.0034580000210553408
}
]
},
{
- "roundMs": 2396.2917920000036,
- "firstDeltaMs": 1871.972833000007,
+ "roundMs": 2406.356749999977,
+ "firstDeltaMs": 2057.066124999983,
"calls": []
}
]
@@ -1601,27 +1382,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1104,
- "p95": 1353,
- "mean": 1229,
+ "p50": 1110,
+ "p95": 1111,
+ "mean": 1111,
"n": 2
},
"ttftMs": {
- "p50": 9425,
- "p95": 15956,
- "mean": 12691,
+ "p50": 9317,
+ "p95": 11714,
+ "mean": 10516,
"n": 2
},
"totalMs": {
- "p50": 9976,
- "p95": 16263,
- "mean": 13119,
+ "p50": 9994,
+ "p95": 12089,
+ "mean": 11041,
"n": 2
},
"modelMs": {
- "p50": 8623,
- "p95": 15159,
- "mean": 11891,
+ "p50": 8883,
+ "p95": 10977,
+ "mean": 9930,
"n": 2
},
"toolMs": {
@@ -1631,120 +1412,120 @@
"n": 2
},
"rounds": {
- "mean": 2.5,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: set_color_grade"
- ],
+ "notes": [],
"trace": [
- {
- "name": "set_color_grade",
- "round": 0,
- "atMs": 10724.541457999992,
- "execMs": 0.26504100000602193
- },
{
"name": "generate_video",
- "round": 1,
- "atMs": 16833.578999999998,
- "execMs": 0.027750000008381903
+ "round": 0,
+ "atMs": 9793.092833000002,
+ "execMs": 0.15454200000385754
}
],
"timings": {
- "gateMs": 2218.938334000006,
+ "gateMs": 1110.8876670000027,
"gateSkipped": false,
- "ttftMs": 18940.05575,
- "modelMs": 17175.843291999976,
- "toolMs": 0.29279100001440383,
- "totalMs": 19395.074416999996,
+ "ttftMs": 11714.384250000003,
+ "modelMs": 10977.486208000017,
+ "toolMs": 0.15454200000385754,
+ "totalMs": 12088.528417000023,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8505.170624999999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.26504100000602193
- }
- ]
- },
- {
- "roundMs": 6108.625958999997,
+ "roundMs": 8681.98550000001,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.027750000008381903
+ "execMs": 0.15454200000385754
}
]
},
{
- "roundMs": 2562.04670799998,
- "firstDeltaMs": 2106.404749999987,
+ "roundMs": 2295.500708000007,
+ "firstDeltaMs": 1921.113458000007,
"calls": []
}
]
}
},
{
- "pass": true,
+ "pass": false,
"intent": "work",
- "notes": [],
+ "notes": [
+ "mutating tools called: set_color_grade"
+ ],
"trace": [
{
- "name": "watch_video",
+ "name": "stock_search",
"round": 0,
- "atMs": 7350.879082999978,
- "execMs": 0.06574999997974373
+ "atMs": 8475.122957999993,
+ "execMs": 0.015625
},
{
- "name": "generate_video",
+ "name": "set_color_grade",
"round": 1,
- "atMs": 13634.937750000012,
- "execMs": 0.05058300000382587
+ "atMs": 13092.420624999999,
+ "execMs": 0.013582999992650002
+ },
+ {
+ "name": "generate_video",
+ "round": 2,
+ "atMs": 16768.91391599999,
+ "execMs": 0.006833000021288171
}
],
"timings": {
- "gateMs": 1104.1562080000003,
+ "gateMs": 1207.7262920000067,
"gateSkipped": false,
- "ttftMs": 15956.37479100001,
- "modelMs": 15158.517332999938,
- "toolMs": 0.11633299998356961,
- "totalMs": 16262.789873999922,
+ "ttftMs": 18754.59925,
+ "modelMs": 17953.298833999987,
+ "toolMs": 0.036041000013938174,
+ "totalMs": 19161.061167000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6246.326165999984,
+ "roundMs": 7267.261541999993,
"firstDeltaMs": null,
"calls": [
{
- "name": "watch_video",
- "execMs": 0.06574999997974373
+ "name": "stock_search",
+ "execMs": 0.015625
+ }
+ ]
+ },
+ {
+ "roundMs": 4617.233917000005,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_color_grade",
+ "execMs": 0.013582999992650002
}
]
},
{
- "roundMs": 6283.335041999962,
+ "roundMs": 3676.446499999991,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.05058300000382587
+ "execMs": 0.006833000021288171
}
]
},
{
- "roundMs": 2628.856124999991,
- "firstDeltaMs": 2321.0421659999993,
+ "roundMs": 2392.3568749999977,
+ "firstDeltaMs": 1985.6671250000072,
"calls": []
}
]
@@ -1758,33 +1539,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 7204.547250000003,
- "execMs": 0.02466699999058619
+ "atMs": 7371.604292000004,
+ "execMs": 0.027291999984299764
}
],
"timings": {
- "gateMs": 1352.9717919999966,
+ "gateMs": 1110.1656249999942,
"gateSkipped": false,
- "ttftMs": 9424.770791999996,
- "modelMs": 8622.81791599997,
- "toolMs": 0.02466699999058619,
- "totalMs": 9975.814374999958,
+ "ttftMs": 9316.821916999994,
+ "modelMs": 8883.443875000026,
+ "toolMs": 0.027291999984299764,
+ "totalMs": 9993.636792000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5851.186624999973,
+ "roundMs": 6260.990833000018,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.02466699999058619
+ "execMs": 0.027291999984299764
}
]
},
{
- "roundMs": 2771.631290999998,
- "firstDeltaMs": 2220.131875000021,
+ "roundMs": 2622.4530420000083,
+ "firstDeltaMs": 1945.0718750000233,
"calls": []
}
]
@@ -1795,242 +1576,160 @@
{
"name": "mute-all-does-not-redo-prior-turn",
"bucket": "single-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1437,
+ "p95": 2027,
+ "mean": 1555,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 6715,
+ "p95": 13182,
+ "mean": 8509,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 6826,
+ "p95": 13463,
+ "mean": 8765,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 5389,
+ "p95": 11435,
+ "mean": 7210,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "slow: 4 tool calls (cap 3)",
- "mutating tools called: set_track_muted"
- ],
+ "notes": [],
"trace": [
{
"name": "set_track_muted",
"round": 0,
- "atMs": 6623.0621670000255,
- "execMs": 0.16370800003642216
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 8472.292584000039,
- "execMs": 0.03004099999088794
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 8472.364417000033,
- "execMs": 0.00591599999461323
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 8472.372875000001,
- "execMs": 0.0009999999892897904
+ "atMs": 8173.891625000018,
+ "execMs": 0.06483299998217262
}
],
"timings": {
- "gateMs": 2121.007082999975,
+ "gateMs": 2027.1493750000081,
"gateSkipped": false,
- "ttftMs": 10134.554584000027,
- "modelMs": 8338.392165999976,
- "toolMs": 0.20066500001121312,
- "totalMs": 10459.599913999962,
+ "ttftMs": 13181.601916000014,
+ "modelMs": 11435.438792,
+ "toolMs": 0.06483299998217262,
+ "totalMs": 13462.652999999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4500.434750000015,
+ "roundMs": 6146.34262499999,
"firstDeltaMs": null,
"calls": [
{
"name": "set_track_muted",
- "execMs": 0.16370800003642216
+ "execMs": 0.06483299998217262
}
]
},
{
- "roundMs": 1848.9092079999973,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_clip_muted",
- "execMs": 0.03004099999088794
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.00591599999461323
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0009999999892897904
- }
- ]
- },
- {
- "roundMs": 1989.0482079999638,
- "firstDeltaMs": 1662.140999999945,
+ "roundMs": 5289.096167000011,
+ "firstDeltaMs": 5007.625042,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "slow: 4 tool calls (cap 3)",
- "mutating tools called: set_track_muted"
- ],
+ "notes": [],
"trace": [
{
"name": "set_track_muted",
"round": 0,
- "atMs": 5585.682999999961,
- "execMs": 0.031291999970562756
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 7385.556958000001,
- "execMs": 0.006874999962747097
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 7385.574250000005,
- "execMs": 0.001500000013038516
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 7385.576583000016,
- "execMs": 0.00024999998277053237
+ "atMs": 4689.030708000006,
+ "execMs": 0.013542000000597909
}
],
"timings": {
- "gateMs": 1125.3159579999628,
+ "gateMs": 1436.7740420000046,
"gateSkipped": false,
- "ttftMs": 9028.546374999976,
- "modelMs": 8025.366709000024,
- "toolMs": 0.0399169999291189,
- "totalMs": 9150.722583999916,
+ "ttftMs": 6714.651792000019,
+ "modelMs": 5389.012416999962,
+ "toolMs": 0.013542000000597909,
+ "totalMs": 6825.800000999967,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4459.936541999981,
+ "roundMs": 3252.0281249999825,
"firstDeltaMs": null,
"calls": [
{
"name": "set_track_muted",
- "execMs": 0.031291999970562756
- }
- ]
- },
- {
- "roundMs": 1799.763208999997,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_clip_muted",
- "execMs": 0.006874999962747097
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.001500000013038516
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.00024999998277053237
+ "execMs": 0.013542000000597909
}
]
},
{
- "roundMs": 1765.6669580000453,
- "firstDeltaMs": 1642.9540830000187,
+ "roundMs": 2136.984291999979,
+ "firstDeltaMs": 2025.5842089999933,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "slow: 4 tool calls (cap 3)",
- "mutating tools called: set_track_muted"
- ],
+ "notes": [],
"trace": [
{
"name": "set_track_muted",
"round": 0,
- "atMs": 5558.35170900001,
- "execMs": 0.022458999999798834
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 7743.768792000017,
- "execMs": 0.01862499996786937
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 7743.7952090000035,
- "execMs": 0.0008329999982379377
- },
- {
- "name": "set_clip_muted",
- "round": 1,
- "atMs": 7743.7971670000115,
- "execMs": 0.005541999998968095
+ "atMs": 4158.955250000028,
+ "execMs": 0.014582999981939793
}
],
"timings": {
- "gateMs": 1337.6305830000201,
+ "gateMs": 1201.2304170000134,
"gateSkipped": false,
- "ttftMs": 9560.071458999999,
- "modelMs": 8394.434458000003,
- "toolMs": 0.04745899996487424,
- "totalMs": 9732.112499999988,
+ "ttftMs": 5630.5582920000015,
+ "modelMs": 4804.557749000029,
+ "toolMs": 0.014582999981939793,
+ "totalMs": 6005.802749000024,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4220.355125000002,
+ "roundMs": 2957.5812910000095,
"firstDeltaMs": null,
"calls": [
{
"name": "set_track_muted",
- "execMs": 0.022458999999798834
- }
- ]
- },
- {
- "roundMs": 2185.346542000014,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_clip_muted",
- "execMs": 0.01862499996786937
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0008329999982379377
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.005541999998968095
+ "execMs": 0.014582999981939793
}
]
},
{
- "roundMs": 1988.7327909999876,
- "firstDeltaMs": 1816.2392910000053,
+ "roundMs": 1846.9764580000192,
+ "firstDeltaMs": 1471.5589170000167,
"calls": []
}
]
@@ -2045,32 +1744,32 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1342,
- "p95": 1752,
- "mean": 1453,
+ "p50": 1822,
+ "p95": 1911,
+ "mean": 1580,
"n": 3
},
"ttftMs": {
- "p50": 7031,
- "p95": 11360,
- "mean": 8461,
+ "p50": 7259,
+ "p95": 7698,
+ "mean": 7224,
"n": 3
},
"totalMs": {
- "p50": 7326,
- "p95": 11635,
- "mean": 8732,
+ "p50": 7542,
+ "p95": 7997,
+ "mean": 7522,
"n": 3
},
"modelMs": {
- "p50": 5984,
- "p95": 9883,
- "mean": 7279,
+ "p50": 6020,
+ "p95": 6175,
+ "mean": 5942,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
@@ -2088,33 +1787,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 5557.226792000001,
- "execMs": 0.5211250000284053
+ "atMs": 5192.144250000012,
+ "execMs": 0.41970900003798306
}
],
"timings": {
- "gateMs": 1341.5421670000069,
+ "gateMs": 1006.0502080000006,
"gateSkipped": false,
- "ttftMs": 6992.008458000026,
- "modelMs": 5983.646875000035,
- "toolMs": 0.5211250000284053,
- "totalMs": 7325.71016700007,
+ "ttftMs": 6715.104333000025,
+ "modelMs": 6020.141250999994,
+ "toolMs": 0.41970900003798306,
+ "totalMs": 7026.611168000032,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4214.763250000018,
+ "roundMs": 4185.736208999995,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.5211250000284053
+ "execMs": 0.41970900003798306
}
]
},
{
- "roundMs": 1768.8836250000168,
- "firstDeltaMs": 1434.2008750000386,
+ "roundMs": 1834.4050419999985,
+ "firstDeltaMs": 1522.493542000011,
"calls": []
}
]
@@ -2128,33 +1827,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 6533.677333,
- "execMs": 0.05441600002814084
+ "atMs": 5345.037791999988,
+ "execMs": 0.04845899995416403
}
],
"timings": {
- "gateMs": 1752.2976659999695,
+ "gateMs": 1821.9001250000438,
"gateSkipped": false,
- "ttftMs": 11359.620624999981,
- "modelMs": 9883.143207999936,
- "toolMs": 0.05441600002814084,
- "totalMs": 11635.495289999933,
+ "ttftMs": 7698.064041999984,
+ "modelMs": 6174.794082999986,
+ "toolMs": 0.04845899995416403,
+ "totalMs": 7996.742666999984,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4781.074832999962,
+ "roundMs": 3522.7392079999554,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.05441600002814084
+ "execMs": 0.04845899995416403
}
]
},
{
- "roundMs": 5102.0683749999735,
- "firstDeltaMs": 4825.771540999995,
+ "roundMs": 2652.0548750000307,
+ "firstDeltaMs": 2352.9298750000307,
"calls": []
}
]
@@ -2168,33 +1867,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 5485.368625000003,
- "execMs": 0.012667000002693385
+ "atMs": 5695.8819169999915,
+ "execMs": 0.022166999988257885
}
],
"timings": {
- "gateMs": 1264.292833000014,
+ "gateMs": 1910.5647079999908,
"gateSkipped": false,
- "ttftMs": 7030.665999999968,
- "modelMs": 5970.962209000019,
- "toolMs": 0.012667000002693385,
- "totalMs": 7235.267709000036,
+ "ttftMs": 7259.481750000035,
+ "modelMs": 5631.468125000014,
+ "toolMs": 0.022166999988257885,
+ "totalMs": 7542.054999999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4220.86712499999,
+ "roundMs": 3784.839957999997,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.012667000002693385
+ "execMs": 0.022166999988257885
}
]
},
{
- "roundMs": 1750.0950840000296,
- "firstDeltaMs": 1545.248459000024,
+ "roundMs": 1846.6281670000171,
+ "firstDeltaMs": 1563.510958000028,
"calls": []
}
]
@@ -2209,27 +1908,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1130,
- "p95": 1267,
- "mean": 1132,
+ "p50": 1065,
+ "p95": 2160,
+ "mean": 1377,
"n": 3
},
"ttftMs": {
- "p50": 7005,
- "p95": 7202,
- "mean": 6894,
+ "p50": 10008,
+ "p95": 10353,
+ "mean": 9346,
"n": 3
},
"totalMs": {
- "p50": 7245,
- "p95": 7497,
- "mean": 7190,
+ "p50": 10234,
+ "p95": 10630,
+ "mean": 9642,
"n": 3
},
"modelMs": {
- "p50": 6115,
- "p95": 6229,
- "mean": 6058,
+ "p50": 8469,
+ "p95": 9169,
+ "mean": 8265,
"n": 3
},
"toolMs": {
@@ -2252,33 +1951,33 @@
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 5305.76449999999,
- "execMs": 0.011333000031299889
+ "atMs": 5941.507541000028,
+ "execMs": 0.04733299999497831
}
],
"timings": {
- "gateMs": 1267.420832999982,
+ "gateMs": 905.2902089999989,
"gateSkipped": false,
- "ttftMs": 7201.594041999953,
- "modelMs": 6229.165791999956,
- "toolMs": 0.011333000031299889,
- "totalMs": 7496.597957999969,
+ "ttftMs": 7676.015916000004,
+ "modelMs": 7158.120374999999,
+ "toolMs": 0.04733299999497831,
+ "totalMs": 8063.457916999992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4037.615749999997,
+ "roundMs": 5036.027834000008,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.011333000031299889
+ "execMs": 0.04733299999497831
}
]
},
{
- "roundMs": 2191.550041999959,
- "firstDeltaMs": 1895.7253339999588,
+ "roundMs": 2122.092540999991,
+ "firstDeltaMs": 1734.4257909999578,
"calls": []
}
]
@@ -2292,33 +1991,33 @@
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 4756.152499999967,
- "execMs": 0.03662500000791624
+ "atMs": 8421.749416999985,
+ "execMs": 0.007083000033162534
}
],
"timings": {
- "gateMs": 997.7669579999638,
+ "gateMs": 2160.396375000011,
"gateSkipped": false,
- "ttftMs": 6476.255000000005,
- "modelMs": 5829.631666000001,
- "toolMs": 0.03662500000791624,
- "totalMs": 6827.435248999973,
+ "ttftMs": 10352.87725000002,
+ "modelMs": 8469.106291000033,
+ "toolMs": 0.007083000033162534,
+ "totalMs": 10629.509749000077,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3757.9845409999834,
+ "roundMs": 6261.116416000004,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.03662500000791624
+ "execMs": 0.007083000033162534
}
]
},
{
- "roundMs": 2071.6471250000177,
- "firstDeltaMs": 1719.9763750000275,
+ "roundMs": 2207.9898750000284,
+ "firstDeltaMs": 1931.0752920000232,
"calls": []
}
]
@@ -2332,33 +2031,33 @@
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 5197.447249999968,
- "execMs": 0.008833999978378415
+ "atMs": 7729.45866600005,
+ "execMs": 0.03279199998360127
}
],
"timings": {
- "gateMs": 1130.0697079999954,
+ "gateMs": 1065.0833749999874,
"gateSkipped": false,
- "ttftMs": 7005.06124999997,
- "modelMs": 6114.737208000035,
- "toolMs": 0.008833999978378415,
- "totalMs": 7244.815750000009,
+ "ttftMs": 10007.91729100002,
+ "modelMs": 9168.680292000063,
+ "toolMs": 0.03279199998360127,
+ "totalMs": 10233.796459000034,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4067.1918750000186,
+ "roundMs": 6664.182584000053,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.008833999978378415
+ "execMs": 0.03279199998360127
}
]
},
{
- "roundMs": 2047.5453330000164,
- "firstDeltaMs": 1807.552916000015,
+ "roundMs": 2504.49770800001,
+ "firstDeltaMs": 2278.3590829999885,
"calls": []
}
]
@@ -2369,179 +2068,160 @@
{
"name": "music-ask-single-generate",
"bucket": "single-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1259,
+ "p95": 2081,
+ "mean": 1448,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 7013,
+ "p95": 8741,
+ "mean": 7314,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 7414,
+ "p95": 8820,
+ "mean": 7583,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 5511,
+ "p95": 7561,
+ "mean": 6135,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: add_clip"
- ],
+ "notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 6397.593499999959,
- "execMs": 0.005834000010509044
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 9883.791999999958,
- "execMs": 0.03129099996294826
+ "atMs": 6438.912791999988,
+ "execMs": 0.08666599995922297
}
],
"timings": {
- "gateMs": 1744.0604590000003,
+ "gateMs": 1259.023583000002,
"gateSkipped": false,
- "ttftMs": 13676.348832999996,
- "modelMs": 12476.355583999946,
- "toolMs": 0.03712499997345731,
- "totalMs": 14220.45316799992,
+ "ttftMs": 8741.262625000032,
+ "modelMs": 7560.590957999986,
+ "toolMs": 0.08666599995922297,
+ "totalMs": 8819.701206999947,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4653.300999999978,
+ "roundMs": 5179.565833000001,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.005834000010509044
- }
- ]
- },
- {
- "roundMs": 3486.1480420000153,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.03129099996294826
+ "execMs": 0.08666599995922297
}
]
},
{
- "roundMs": 4336.906541999953,
- "firstDeltaMs": 3792.4770839999546,
+ "roundMs": 2381.0251249999856,
+ "firstDeltaMs": 2302.2430409999797,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: add_clip"
- ],
+ "notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 4395.817959000007,
- "execMs": 0.0062500000349245965
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 7597.321916999994,
- "execMs": 0.023416999960318208
+ "atMs": 4278.594582999998,
+ "execMs": 0.01333300000987947
}
],
"timings": {
- "gateMs": 962.8018339999835,
+ "gateMs": 1005.0069159999839,
"gateSkipped": false,
- "ttftMs": 11128.675500000012,
- "modelMs": 10605.511792000034,
- "toolMs": 0.029666999995242804,
- "totalMs": 11568.343293000013,
+ "ttftMs": 6188.406166000001,
+ "modelMs": 5511.011332999973,
+ "toolMs": 0.01333300000987947,
+ "totalMs": 6516.031581999967,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3432.546875,
+ "roundMs": 3273.310499999963,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.0062500000349245965
- }
- ]
- },
- {
- "roundMs": 3201.356792000006,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.023416999960318208
+ "execMs": 0.01333300000987947
}
]
},
{
- "roundMs": 3971.608125000028,
- "firstDeltaMs": 3531.2515419999836,
+ "roundMs": 2237.70083300001,
+ "firstDeltaMs": 1909.6614579999587,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: add_clip"
- ],
+ "notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 9365.28562499996,
- "execMs": 0.02374999999301508
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 12923.075374999957,
- "execMs": 0.014583000040147454
+ "atMs": 5327.312457999971,
+ "execMs": 0.012291999999433756
}
],
"timings": {
- "gateMs": 1657.066999999981,
+ "gateMs": 2080.738000000012,
"gateSkipped": false,
- "ttftMs": 16834.523791999964,
- "modelMs": 15652.907082999998,
- "toolMs": 0.038333000033162534,
- "totalMs": 17310.012416000012,
+ "ttftMs": 7013.472542000003,
+ "modelMs": 5333.22808400006,
+ "toolMs": 0.012291999999433756,
+ "totalMs": 7413.978376000072,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7707.938874999993,
+ "roundMs": 3246.235292000056,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.02374999999301508
- }
- ]
- },
- {
- "roundMs": 3557.6689999999944,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.014583000040147454
+ "execMs": 0.012291999999433756
}
]
},
{
- "roundMs": 4387.299208000011,
- "firstDeltaMs": 3911.414000000048,
+ "roundMs": 2086.9927920000046,
+ "firstDeltaMs": 1686.0714589999989,
"calls": []
}
]
@@ -2556,27 +2236,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1272,
- "p95": 2214,
- "mean": 1584,
+ "p50": 1166,
+ "p95": 1250,
+ "mean": 1127,
"n": 3
},
"ttftMs": {
- "p50": 6500,
- "p95": 8387,
- "mean": 6878,
+ "p50": 5950,
+ "p95": 6220,
+ "mean": 5998,
"n": 3
},
"totalMs": {
- "p50": 6745,
- "p95": 8695,
- "mean": 7182,
+ "p50": 6068,
+ "p95": 6449,
+ "mean": 6194,
"n": 3
},
"modelMs": {
- "p50": 5473,
- "p95": 6481,
- "mean": 5598,
+ "p50": 5100,
+ "p95": 5283,
+ "mean": 5067,
"n": 3
},
"toolMs": {
@@ -2599,33 +2279,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 4041.5212499999907,
- "execMs": 0.01829199999338016
+ "atMs": 4293.4655839999905,
+ "execMs": 0.014917000022251159
}
],
"timings": {
- "gateMs": 1265.1938749999972,
+ "gateMs": 1249.6147919999785,
"gateSkipped": false,
- "ttftMs": 5746.650040999986,
- "modelMs": 4840.957874000014,
- "toolMs": 0.01829199999338016,
- "totalMs": 6106.170041000005,
+ "ttftMs": 5949.80691699998,
+ "modelMs": 4818.79241699999,
+ "toolMs": 0.014917000022251159,
+ "totalMs": 6068.42212599999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2776.077958000009,
+ "roundMs": 3043.4214999999967,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.01829199999338016
+ "execMs": 0.014917000022251159
}
]
},
{
- "roundMs": 2064.8799160000053,
- "firstDeltaMs": 1705.0766660000081,
+ "roundMs": 1775.370916999993,
+ "firstDeltaMs": 1656.2577920000185,
"calls": []
}
]
@@ -2639,33 +2319,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 6486.319458000013,
- "execMs": 0.002582999994046986
+ "atMs": 4170.795999999973,
+ "execMs": 0.07366700004786253
}
],
"timings": {
- "gateMs": 2214.1327499999898,
+ "gateMs": 965.8940839999705,
"gateSkipped": false,
- "ttftMs": 8387.065166999993,
- "modelMs": 6480.738499000028,
- "toolMs": 0.002582999994046986,
- "totalMs": 8694.873832000012,
+ "ttftMs": 5824.283707999974,
+ "modelMs": 5099.601167000015,
+ "toolMs": 0.07366700004786253,
+ "totalMs": 6065.568918000034,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4271.954708000005,
+ "roundMs": 3204.676916999975,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.002582999994046986
+ "execMs": 0.07366700004786253
}
]
},
{
- "roundMs": 2208.7837910000235,
- "firstDeltaMs": 1900.7057500000228,
+ "roundMs": 1894.92425000004,
+ "firstDeltaMs": 1653.3652500000317,
"calls": []
}
]
@@ -2679,33 +2359,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 4694.141332999978,
- "execMs": 0.001166999980341643
+ "atMs": 3918.7810839999584,
+ "execMs": 0.0020000000367872417
}
],
"timings": {
- "gateMs": 1272.0871669999906,
+ "gateMs": 1165.9406249999884,
"gateSkipped": false,
- "ttftMs": 6499.640542000008,
- "modelMs": 5473.135207999963,
- "toolMs": 0.001166999980341643,
- "totalMs": 6745.223541999934,
+ "ttftMs": 6219.540333999961,
+ "modelMs": 5283.4743749999325,
+ "toolMs": 0.0020000000367872417,
+ "totalMs": 6449.416999999958,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3421.8870420000167,
+ "roundMs": 2752.6041249999544,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.001166999980341643
+ "execMs": 0.0020000000367872417
}
]
},
{
- "roundMs": 2051.2481659999467,
- "firstDeltaMs": 1805.4825409999467,
+ "roundMs": 2530.870249999978,
+ "firstDeltaMs": 2300.72879199998,
"calls": []
}
]
@@ -2720,27 +2400,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1521,
- "p95": 1737,
- "mean": 1536,
+ "p50": 1220,
+ "p95": 1698,
+ "mean": 1306,
"n": 3
},
"ttftMs": {
- "p50": 6338,
- "p95": 6860,
- "mean": 6451,
+ "p50": 8238,
+ "p95": 9094,
+ "mean": 8093,
"n": 3
},
"totalMs": {
- "p50": 6688,
- "p95": 7096,
- "mean": 6734,
+ "p50": 8410,
+ "p95": 9356,
+ "mean": 8323,
"n": 3
},
"modelMs": {
- "p50": 5066,
- "p95": 5575,
- "mean": 5197,
+ "p50": 7190,
+ "p95": 8358,
+ "mean": 7017,
"n": 3
},
"toolMs": {
@@ -2763,33 +2443,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 5017.68862500001,
- "execMs": 0.3066250000265427
+ "atMs": 6580.367499999993,
+ "execMs": 0.055958000011742115
}
],
"timings": {
- "gateMs": 1520.8685839999816,
+ "gateMs": 1220.4540419999976,
"gateSkipped": false,
- "ttftMs": 6860.3288750000065,
- "modelMs": 5574.965957999928,
- "toolMs": 0.3066250000265427,
- "totalMs": 7096.141166999936,
+ "ttftMs": 8238.116374999983,
+ "modelMs": 7189.800417000079,
+ "toolMs": 0.055958000011742115,
+ "totalMs": 8410.310417000088,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3496.3040409999667,
+ "roundMs": 5359.287709000055,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.3066250000265427
+ "execMs": 0.055958000011742115
}
]
},
{
- "roundMs": 2078.6619169999612,
- "firstDeltaMs": 1842.2913340000086,
+ "roundMs": 1830.5127080000238,
+ "firstDeltaMs": 1657.6754579999833,
"calls": []
}
]
@@ -2803,33 +2483,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 4532.0848750000005,
- "execMs": 0.02029199997195974
+ "atMs": 4653.387999999977,
+ "execMs": 0.010999999998603016
}
],
"timings": {
- "gateMs": 1351.067458000034,
+ "gateMs": 997.749375000014,
"gateSkipped": false,
- "ttftMs": 6152.820667000022,
- "modelMs": 5066.241750000045,
- "toolMs": 0.02029199997195974,
- "totalMs": 6417.329500000051,
+ "ttftMs": 9093.974166000029,
+ "modelMs": 8358.287125000032,
+ "toolMs": 0.010999999998603016,
+ "totalMs": 9356.047500000044,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3180.866207999992,
+ "roundMs": 3655.266708000039,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.02029199997195974
+ "execMs": 0.010999999998603016
}
]
},
{
- "roundMs": 1885.3755420000525,
- "firstDeltaMs": 1620.6782500000554,
+ "roundMs": 4703.020416999992,
+ "firstDeltaMs": 4440.501042000018,
"calls": []
}
]
@@ -2843,33 +2523,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 4817.474625000032,
- "execMs": 0.021208000020124018
+ "atMs": 5303.947540999972,
+ "execMs": 0.018249999964609742
}
],
"timings": {
- "gateMs": 1736.7110419999808,
+ "gateMs": 1698.298208000022,
"gateSkipped": false,
- "ttftMs": 6338.42516600003,
- "modelMs": 4951.152751000016,
- "toolMs": 0.021208000020124018,
- "totalMs": 6687.885001000017,
+ "ttftMs": 6945.728833000001,
+ "modelMs": 5503.714707999956,
+ "toolMs": 0.018249999964609742,
+ "totalMs": 7202.0311659999425,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3080.4755420000292,
+ "roundMs": 3605.2568749999627,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.021208000020124018
+ "execMs": 0.018249999964609742
}
]
},
{
- "roundMs": 1870.6772089999868,
- "firstDeltaMs": 1520.899167000025,
+ "roundMs": 1898.4578329999931,
+ "firstDeltaMs": 1641.7162910000188,
"calls": []
}
]
@@ -2884,27 +2564,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1836,
- "p95": 1925,
- "mean": 1653,
+ "p50": 1373,
+ "p95": 1389,
+ "mean": 1324,
"n": 3
},
"ttftMs": {
- "p50": 11263,
- "p95": 11879,
- "mean": 11385,
+ "p50": 11456,
+ "p95": 12054,
+ "mean": 10627,
"n": 3
},
"totalMs": {
- "p50": 11637,
- "p95": 12303,
- "mean": 11745,
+ "p50": 11649,
+ "p95": 12306,
+ "mean": 10876,
"n": 3
},
"modelMs": {
- "p50": 10378,
- "p95": 10440,
- "mean": 10092,
+ "p50": 10260,
+ "p95": 10933,
+ "mean": 9551,
"n": 3
},
"toolMs": {
@@ -2925,107 +2605,107 @@
"notes": [],
"trace": [
{
- "name": "set_animation",
+ "name": "list_skills",
"round": 0,
- "atMs": 6581.835709000006,
- "execMs": 0.26966700004413724
+ "atMs": 5810.473125000019,
+ "execMs": 0.1387090000207536
+ },
+ {
+ "name": "read_skill",
+ "round": 1,
+ "atMs": 7448.895874999987,
+ "execMs": 0.016583999968133867
+ },
+ {
+ "name": "set_animation",
+ "round": 2,
+ "atMs": 9865.527291000006,
+ "execMs": 0.013749999983701855
}
],
"timings": {
- "gateMs": 1836.1457919999957,
+ "gateMs": 1388.8772919999901,
"gateSkipped": false,
- "ttftMs": 11013.065042000031,
- "modelMs": 9457.764166000008,
- "toolMs": 0.26966700004413724,
- "totalMs": 11294.179625000048,
+ "ttftMs": 11455.86041600001,
+ "modelMs": 10260.113249999995,
+ "toolMs": 0.16904299997258931,
+ "totalMs": 11649.159584999958,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4745.331375000009,
+ "roundMs": 4421.327916999988,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.26966700004413724
+ "name": "list_skills",
+ "execMs": 0.1387090000207536
}
]
},
{
- "roundMs": 4712.432790999999,
- "firstDeltaMs": 4430.918540999992,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "work",
- "notes": [],
- "trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 5826.17624999996,
- "execMs": 0.08562500000698492
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 7467.488291999965,
- "execMs": 0.03187500004423782
- },
- {
- "name": "set_animation",
- "round": 2,
- "atMs": 9984.897082999989,
- "execMs": 0.018582999997306615
- }
- ],
- "timings": {
- "gateMs": 1925.0209589999868,
- "gateSkipped": false,
- "ttftMs": 11878.575207999966,
- "modelMs": 10377.629042999994,
- "toolMs": 0.13608300004852936,
- "totalMs": 12302.786085000029,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3900.8892089999863,
+ "roundMs": 1638.2129999999888,
"firstDeltaMs": null,
"calls": [
{
- "name": "list_skills",
- "execMs": 0.08562500000698492
+ "name": "read_skill",
+ "execMs": 0.016583999968133867
}
]
},
{
- "roundMs": 1641.1638340000063,
+ "roundMs": 2416.56020800001,
"firstDeltaMs": null,
"calls": [
{
- "name": "read_skill",
- "execMs": 0.03187500004423782
+ "name": "set_animation",
+ "execMs": 0.013749999983701855
}
]
},
{
- "roundMs": 2517.3102499999804,
+ "roundMs": 1784.0121250000084,
+ "firstDeltaMs": 1590.2963340000133,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "work",
+ "notes": [],
+ "trace": [
+ {
+ "name": "set_animation",
+ "round": 0,
+ "atMs": 6568.660749999981,
+ "execMs": 0.05358399997930974
+ }
+ ],
+ "timings": {
+ "gateMs": 1210.7979589999886,
+ "gateSkipped": false,
+ "ttftMs": 8370.474707999965,
+ "modelMs": 7461.026165999996,
+ "toolMs": 0.05358399997930974,
+ "totalMs": 8671.877708999964,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 5357.552000000025,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.018582999997306615
+ "execMs": 0.05358399997930974
}
]
},
{
- "roundMs": 2318.2657500000205,
- "firstDeltaMs": 1893.6262910000514,
+ "roundMs": 2103.474165999971,
+ "firstDeltaMs": 1801.677040999988,
"calls": []
}
]
@@ -3039,65 +2719,65 @@
{
"name": "list_skills",
"round": 0,
- "atMs": 5762.710000000021,
- "execMs": 0.054957999964244664
+ "atMs": 6750.958915999974,
+ "execMs": 0.043999999994412065
},
{
"name": "read_skill",
"round": 1,
- "atMs": 7414.59258300002,
- "execMs": 0.0530000000144355
+ "atMs": 8398.871207999997,
+ "execMs": 0.01208399998722598
},
{
"name": "set_animation",
"round": 2,
- "atMs": 10013.273749999993,
- "execMs": 0.006583000009413809
+ "atMs": 10522.651624999999,
+ "execMs": 0.005208000016864389
}
],
"timings": {
- "gateMs": 1197.418250000046,
+ "gateMs": 1373.1554169999436,
"gateSkipped": false,
- "ttftMs": 11263.004707999993,
- "modelMs": 10439.875208000012,
- "toolMs": 0.11454099998809397,
- "totalMs": 11637.407999000046,
+ "ttftMs": 12053.571415999962,
+ "modelMs": 10932.912998999993,
+ "toolMs": 0.06129199999850243,
+ "totalMs": 12306.129707999935,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4565.177832999965,
+ "roundMs": 5377.341791999992,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.054957999964244664
+ "execMs": 0.043999999994412065
}
]
},
{
- "roundMs": 1651.7470839999733,
+ "roundMs": 1647.7694580000243,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.0530000000144355
+ "execMs": 0.01208399998722598
}
]
},
{
- "roundMs": 2598.58050000004,
+ "roundMs": 2123.7285409999895,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.006583000009413809
+ "execMs": 0.005208000016864389
}
]
},
{
- "roundMs": 1624.3697910000337,
- "firstDeltaMs": 1249.7159580000443,
+ "roundMs": 1784.073207999987,
+ "firstDeltaMs": 1530.894666999986,
"calls": []
}
]
@@ -3108,182 +2788,160 @@
{
"name": "look-ask-single-tool",
"bucket": "single-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1267,
+ "p95": 1590,
+ "mean": 1331,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 6487,
+ "p95": 8774,
+ "mean": 7184,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 6710,
+ "p95": 9289,
+ "mean": 7504,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 5379,
+ "p95": 8022,
+ "mean": 6173,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, set_color_grade"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 5137.7635419999715,
- "execMs": 0.2654999999795109
- },
- {
- "name": "set_color_grade",
- "round": 1,
- "atMs": 9672.390542000008,
- "execMs": 0.02579200000036508
+ "atMs": 4698.273291999998,
+ "execMs": 0.10749999998370185
}
],
"timings": {
- "gateMs": 1199.774374999979,
+ "gateMs": 1134.8024999999907,
"gateSkipped": false,
- "ttftMs": 14228.714541999972,
- "modelMs": 13401.771707000036,
- "toolMs": 0.291291999979876,
- "totalMs": 14601.837373999995,
+ "ttftMs": 6290.164832999988,
+ "modelMs": 5379.344876000017,
+ "toolMs": 0.10749999998370185,
+ "totalMs": 6514.254875999992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3937.3804579999996,
+ "roundMs": 3563.1313340000343,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.2654999999795109
- }
- ]
- },
- {
- "roundMs": 4534.281791000045,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.02579200000036508
+ "execMs": 0.10749999998370185
}
]
},
{
- "roundMs": 4930.109457999992,
- "firstDeltaMs": 4556.244915999996,
+ "roundMs": 1816.2135419999831,
+ "firstDeltaMs": 1591.7688750000088,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, set_color_grade"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 4303.561791000015,
- "execMs": 0.011666999955195934
- },
- {
- "name": "set_color_grade",
- "round": 1,
- "atMs": 7946.62045799999,
- "execMs": 0.011082999990321696
+ "atMs": 4930.380124999967,
+ "execMs": 0.013958000054117292
}
],
"timings": {
- "gateMs": 1217.518207999994,
+ "gateMs": 1590.385249999992,
"gateSkipped": false,
- "ttftMs": 13145.375082999992,
- "modelMs": 12453.143749999988,
- "toolMs": 0.02274999994551763,
- "totalMs": 13670.684707999928,
+ "ttftMs": 6486.966458999959,
+ "modelMs": 5119.351334000006,
+ "toolMs": 0.013958000054117292,
+ "totalMs": 6709.7505420000525,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3085.7391670000507,
+ "roundMs": 3339.910666999989,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.011666999955195934
+ "execMs": 0.013958000054117292
}
]
},
{
- "roundMs": 3642.9844169999706,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.011082999990321696
- }
- ]
- },
- {
- "roundMs": 5724.420165999967,
- "firstDeltaMs": 5198.712583000015,
+ "roundMs": 1779.4406670000171,
+ "firstDeltaMs": 1556.5559999999823,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, set_color_grade"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 4724.51695900003,
- "execMs": 0.006999999983236194
- },
- {
- "name": "set_color_grade",
- "round": 1,
- "atMs": 13559.795291999995,
- "execMs": 0.015916000003926456
+ "atMs": 4080.5476250000065,
+ "execMs": 0.009750000026542693
}
],
"timings": {
- "gateMs": 1516.5394579999847,
+ "gateMs": 1266.9427499999874,
"gateSkipped": false,
- "ttftMs": 17481.25483400002,
- "modelMs": 16567.670165000018,
- "toolMs": 0.02291599998716265,
- "totalMs": 18084.23253899999,
+ "ttftMs": 8774.049124999961,
+ "modelMs": 8021.740000999998,
+ "toolMs": 0.009750000026542693,
+ "totalMs": 9288.692501000012,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3207.7665830000187,
+ "roundMs": 2813.4964590000454,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.006999999983236194
- }
- ]
- },
- {
- "roundMs": 8835.222790999978,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.015916000003926456
+ "execMs": 0.009750000026542693
}
]
},
{
- "roundMs": 4524.680791000021,
- "firstDeltaMs": 3921.410375000036,
+ "roundMs": 5208.243541999953,
+ "firstDeltaMs": 4693.477999999945,
"calls": []
}
]
@@ -3298,27 +2956,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1271,
- "p95": 1488,
- "mean": 1273,
+ "p50": 1802,
+ "p95": 2254,
+ "mean": 1747,
"n": 3
},
"ttftMs": {
- "p50": 8406,
- "p95": 8944,
- "mean": 7908,
+ "p50": 8820,
+ "p95": 9059,
+ "mean": 8509,
"n": 3
},
"totalMs": {
- "p50": 8738,
- "p95": 9277,
- "mean": 8277,
+ "p50": 9073,
+ "p95": 9263,
+ "mean": 8735,
"n": 3
},
"modelMs": {
- "p50": 7250,
- "p95": 8006,
- "mean": 7004,
+ "p50": 7271,
+ "p95": 8077,
+ "mean": 6988,
"n": 3
},
"toolMs": {
@@ -3341,43 +2999,43 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 6582.971207999974,
- "execMs": 0.10337499994784594
+ "atMs": 6073.476624999952,
+ "execMs": 0.41491699998732656
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6583.087999999989,
- "execMs": 0.003499999991618097
+ "atMs": 6073.92624999996,
+ "execMs": 0.00795899995137006
}
],
"timings": {
- "gateMs": 1270.6758749999572,
+ "gateMs": 2253.9949590000324,
"gateSkipped": false,
- "ttftMs": 8944.449207999976,
- "modelMs": 8006.174750000006,
- "toolMs": 0.10687499993946403,
- "totalMs": 9276.957499999902,
+ "ttftMs": 7649.155415999994,
+ "modelMs": 5615.490458999935,
+ "toolMs": 0.4228759999386966,
+ "totalMs": 7869.908293999906,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5311.879249999998,
+ "roundMs": 3818.6686669999617,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.10337499994784594
+ "execMs": 0.41491699998732656
},
{
"name": "set_animation",
- "execMs": 0.003499999991618097
+ "execMs": 0.00795899995137006
}
]
},
{
- "roundMs": 2694.2955000000075,
- "firstDeltaMs": 2361.3342499999562,
+ "roundMs": 1796.8217919999734,
+ "firstDeltaMs": 1575.1960419999668,
"calls": []
}
]
@@ -3391,43 +3049,43 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 6556.75920900004,
- "execMs": 0.06579199992120266
+ "atMs": 6694.024334000016,
+ "execMs": 0.006875000020954758
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6556.84375,
- "execMs": 0.009458000073209405
+ "atMs": 6694.036416999996,
+ "execMs": 0.0021670000278390944
}
],
"timings": {
- "gateMs": 1487.9176659999648,
+ "gateMs": 1801.9315840000054,
"gateSkipped": false,
- "ttftMs": 8405.68987500004,
- "modelMs": 7250.309292000136,
- "toolMs": 0.07524999999441206,
- "totalMs": 8738.302208000096,
+ "ttftMs": 8819.827374999993,
+ "modelMs": 7270.802790999936,
+ "toolMs": 0.009042000048793852,
+ "totalMs": 9072.74341699999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5068.414542000042,
+ "roundMs": 4891.957540999982,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.06579199992120266
+ "execMs": 0.006875000020954758
},
{
"name": "set_animation",
- "execMs": 0.009458000073209405
+ "execMs": 0.0021670000278390944
}
]
},
{
- "roundMs": 2181.894750000094,
- "firstDeltaMs": 1848.783417000086,
+ "roundMs": 2378.845249999955,
+ "firstDeltaMs": 2125.777624999988,
"calls": []
}
]
@@ -3441,43 +3099,43 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 4722.622042000061,
- "execMs": 0.023665999993681908
+ "atMs": 7289.976291999978,
+ "execMs": 0.026249999995343387
},
{
"name": "set_animation",
"round": 0,
- "atMs": 4722.654875000007,
- "execMs": 0.005917000002227724
+ "atMs": 7290.048542000004,
+ "execMs": 0.010041999979875982
}
],
"timings": {
- "gateMs": 1060.3868749999674,
+ "gateMs": 1185.259374999965,
"gateSkipped": false,
- "ttftMs": 6374.245624999981,
- "modelMs": 5755.61320900009,
- "toolMs": 0.02958299999590963,
- "totalMs": 6816.029667000053,
+ "ttftMs": 9058.721666999976,
+ "modelMs": 8077.2205829999875,
+ "toolMs": 0.03629199997521937,
+ "totalMs": 9262.516249999928,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3662.0965840000426,
+ "roundMs": 6104.293624999991,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.023665999993681908
+ "execMs": 0.026249999995343387
},
{
"name": "set_animation",
- "execMs": 0.005917000002227724
+ "execMs": 0.010041999979875982
}
]
},
{
- "roundMs": 2093.5166250000475,
- "firstDeltaMs": 1651.5425420000684,
+ "roundMs": 1972.9269579999964,
+ "firstDeltaMs": 1768.5934579999885,
"calls": []
}
]
@@ -3488,335 +3146,368 @@
{
"name": "filler-words-cut-with-editor",
"bucket": "multi-tool",
- "passRate": 0,
+ "passRate": 0.3333333333333333,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1391,
+ "p95": 1391,
+ "mean": 1391,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 67131,
+ "p95": 67131,
+ "mean": 67131,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 67381,
+ "p95": 67381,
+ "mean": 67381,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 65987,
+ "p95": 65987,
+ "mean": 65987,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 3,
+ "p95": 3,
+ "mean": 3,
+ "n": 1
+ },
+ "rounds": {
+ "mean": 20,
+ "max": 20
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: refine_speech_cuts"
- ],
+ "notes": [],
"trace": [
{
- "name": "list_skills",
+ "name": "detect_silence",
"round": 0,
- "atMs": 4727.645750000025,
- "execMs": 0.07866599992848933
+ "atMs": 7333.236832999974,
+ "execMs": 0.14279199996963143
},
{
- "name": "read_skill",
+ "name": "listen_audio",
"round": 1,
- "atMs": 6470.762833000044,
- "execMs": 0.01708300004247576
+ "atMs": 9874.058666999976,
+ "execMs": 0.06150000001071021
},
{
- "name": "detect_silence",
+ "name": "list_skills",
"round": 2,
- "atMs": 9311.962874999968,
- "execMs": 0.03608300001360476
+ "atMs": 12140.21537499997,
+ "execMs": 0.013000000035390258
},
{
- "name": "listen_audio",
+ "name": "read_skill",
"round": 3,
- "atMs": 12069.214666999993,
- "execMs": 0.0270419999724254
+ "atMs": 13771.351332999999,
+ "execMs": 0.018749999988358468
},
{
- "name": "detect_silence",
+ "name": "split_at",
"round": 4,
- "atMs": 16310.830874999985,
- "execMs": 0.026084000011906028
+ "atMs": 28939.77229200001,
+ "execMs": 1.0898750000051223
},
{
- "name": "read_skill",
+ "name": "split_at",
"round": 5,
- "atMs": 20689.93320800003,
- "execMs": 0.05520800000522286
+ "atMs": 33806.34895800002,
+ "execMs": 0.052499999990686774
},
{
"name": "split_at",
"round": 6,
- "atMs": 32925.99549999996,
- "execMs": 1.0909999998984858
+ "atMs": 35547.879625,
+ "execMs": 0.06954200001200661
},
{
- "name": "split_at",
+ "name": "get_state",
"round": 7,
- "atMs": 37976.69079200004,
- "execMs": 0.08787500008475035
+ "atMs": 37318.191500000015,
+ "execMs": 0.7523749999818392
},
{
- "name": "split_at",
+ "name": "delete_item",
"round": 8,
- "atMs": 39537.65258300002,
- "execMs": 0.05862499994691461
+ "atMs": 40801.46633299999,
+ "execMs": 0.3408330000238493
},
{
- "name": "get_state",
+ "name": "delete_item",
"round": 9,
- "atMs": 41417.228292000014,
- "execMs": 0.262250000028871
+ "atMs": 42450.066292,
+ "execMs": 0.021249999990686774
},
{
- "name": "delete_item",
+ "name": "get_state",
"round": 10,
- "atMs": 50940.33374999999,
- "execMs": 0.5496249999850988
+ "atMs": 43975.511583000014,
+ "execMs": 0.0359170000301674
},
{
- "name": "delete_item",
+ "name": "refine_speech_cuts",
"round": 11,
- "atMs": 53118.07591699995,
- "execMs": 0.023874999955296516
+ "atMs": 46250.26816699997,
+ "execMs": 0.11612500000046566
},
{
- "name": "delete_cue",
+ "name": "get_state",
"round": 12,
- "atMs": 55051.64375000005,
- "execMs": 0.3115420000394806
+ "atMs": 48320.21220800001,
+ "execMs": 0.01570799999171868
},
{
- "name": "update_cue",
+ "name": "delete_cue",
"round": 13,
- "atMs": 57576.81941700005,
- "execMs": 0.02812499995343387
+ "atMs": 54445.251708000025,
+ "execMs": 0.08254199998918921
},
{
"name": "update_cue",
"round": 14,
- "atMs": 59399.13291699998,
- "execMs": 0.04666600003838539
+ "atMs": 56415.621125000005,
+ "execMs": 0.04870799998752773
},
{
"name": "update_cue",
"round": 15,
- "atMs": 61347.61525000003,
- "execMs": 0.021666000015102327
+ "atMs": 58523.99674999999,
+ "execMs": 0.012042000016663224
},
{
"name": "update_cue",
"round": 16,
- "atMs": 62977.88354199997,
- "execMs": 0.030207999981939793
+ "atMs": 60634.95175000001,
+ "execMs": 0.009291999973356724
},
{
- "name": "get_state",
+ "name": "update_cue",
"round": 17,
- "atMs": 65021.40516700002,
- "execMs": 0.07620799995493144
+ "atMs": 63258.97337499994,
+ "execMs": 0.036416999995708466
},
{
- "name": "refine_speech_cuts",
+ "name": "get_state",
"round": 18,
- "atMs": 69454.97025000001,
- "execMs": 0.011584000079892576
+ "atMs": 65335.91304200003,
+ "execMs": 0.14449999993667006
}
],
"timings": {
- "gateMs": 1117.5984160000226,
+ "gateMs": 1391.3072919999831,
"gateSkipped": false,
- "ttftMs": 71887.62699999998,
- "modelMs": 71388.7636650001,
- "toolMs": 2.8394149999367073,
- "totalMs": 72509.20149600005,
+ "ttftMs": 67130.88370799995,
+ "modelMs": 65987.05029199994,
+ "toolMs": 3.0636679999297485,
+ "totalMs": 67381.42125199985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3609.7460829999764,
+ "roundMs": 5941.205750000023,
"firstDeltaMs": null,
"calls": [
{
- "name": "list_skills",
- "execMs": 0.07866599992848933
+ "name": "detect_silence",
+ "execMs": 0.14279199996963143
}
]
},
{
- "roundMs": 1742.9965420000954,
+ "roundMs": 2540.6026249999995,
"firstDeltaMs": null,
"calls": [
{
- "name": "read_skill",
- "execMs": 0.01708300004247576
+ "name": "listen_audio",
+ "execMs": 0.06150000001071021
}
]
},
{
- "roundMs": 2841.133416000055,
+ "roundMs": 2266.051291999989,
"firstDeltaMs": null,
"calls": [
{
- "name": "detect_silence",
- "execMs": 0.03608300001360476
+ "name": "list_skills",
+ "execMs": 0.013000000035390258
}
]
},
{
- "roundMs": 2757.158374999999,
+ "roundMs": 1631.0947499999893,
"firstDeltaMs": null,
"calls": [
{
- "name": "listen_audio",
- "execMs": 0.0270419999724254
+ "name": "read_skill",
+ "execMs": 0.018749999988358468
}
]
},
{
- "roundMs": 4241.483250000048,
+ "roundMs": 15168.336166999943,
"firstDeltaMs": null,
"calls": [
{
- "name": "detect_silence",
- "execMs": 0.026084000011906028
+ "name": "split_at",
+ "execMs": 1.0898750000051223
}
]
},
{
- "roundMs": 4378.996124999947,
+ "roundMs": 4865.428749999963,
"firstDeltaMs": null,
"calls": [
{
- "name": "read_skill",
- "execMs": 0.05520800000522286
+ "name": "split_at",
+ "execMs": 0.052499999990686774
}
]
},
{
- "roundMs": 12235.925333999912,
+ "roundMs": 1741.4233329999843,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 1.0909999998984858
+ "execMs": 0.06954200001200661
}
]
},
{
- "roundMs": 5049.534499999951,
+ "roundMs": 1770.1545000000042,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.08787500008475035
+ "name": "get_state",
+ "execMs": 0.7523749999818392
}
]
},
{
- "roundMs": 1560.8080830000108,
+ "roundMs": 3482.464290999982,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.05862499994691461
+ "name": "delete_item",
+ "execMs": 0.3408330000238493
}
]
},
{
- "roundMs": 1879.4714579998981,
+ "roundMs": 1648.2306660000468,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.262250000028871
+ "name": "delete_item",
+ "execMs": 0.021249999990686774
}
]
},
{
- "roundMs": 9522.795916000032,
+ "roundMs": 1525.404582999996,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.5496249999850988
+ "name": "get_state",
+ "execMs": 0.0359170000301674
}
]
},
{
- "roundMs": 2177.1469580000266,
+ "roundMs": 2274.700917000009,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.023874999955296516
+ "name": "refine_speech_cuts",
+ "execMs": 0.11612500000046566
}
]
},
{
- "roundMs": 1933.501249999972,
+ "roundMs": 2069.809917000006,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_cue",
- "execMs": 0.3115420000394806
+ "name": "get_state",
+ "execMs": 0.01570799999171868
}
]
},
{
- "roundMs": 2524.8027500000317,
+ "roundMs": 6125.005999999994,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.02812499995343387
+ "name": "delete_cue",
+ "execMs": 0.08254199998918921
}
]
},
{
- "roundMs": 1822.2251659999602,
+ "roundMs": 1970.2509590000263,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.04666600003838539
+ "execMs": 0.04870799998752773
}
]
},
{
- "roundMs": 1948.3669160000281,
+ "roundMs": 2108.3022919999785,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.021666000015102327
+ "execMs": 0.012042000016663224
}
]
},
{
- "roundMs": 1630.188000000082,
+ "roundMs": 2110.9266669999924,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.030207999981939793
+ "execMs": 0.009291999973356724
}
]
},
{
- "roundMs": 2043.4425420000916,
+ "roundMs": 2623.969708000077,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.07620799995493144
+ "name": "update_cue",
+ "execMs": 0.036416999995708466
}
]
},
{
- "roundMs": 4433.456417000038,
+ "roundMs": 2076.857874999987,
"firstDeltaMs": null,
"calls": [
{
- "name": "refine_speech_cuts",
- "execMs": 0.011584000079892576
+ "name": "get_state",
+ "execMs": 0.14449999993667006
}
]
},
{
- "roundMs": 3055.5845839999383,
- "firstDeltaMs": 2432.635666999966,
+ "roundMs": 2046.8292499999516,
+ "firstDeltaMs": 1794.8116669999436,
"calls": []
}
]
@@ -3826,312 +3517,403 @@
"pass": false,
"intent": "work",
"notes": [
- "mutating tools called: refine_speech_cuts, subtitles_generate"
+ "reply did not match /filler|um|uh|cut|remove|trim/i"
],
"trace": [
{
"name": "list_skills",
"round": 0,
- "atMs": 7736.0971660000505,
- "execMs": 0.02849999989848584
+ "atMs": 4088.656125000096,
+ "execMs": 0.11829100002069026
},
{
"name": "read_skill",
"round": 1,
- "atMs": 9660.483250000048,
- "execMs": 0.030790999997407198
+ "atMs": 5657.834458000027,
+ "execMs": 0.037499999976716936
},
{
"name": "detect_silence",
"round": 2,
- "atMs": 12752.107125000097,
- "execMs": 0.010083000059239566
- },
- {
- "name": "listen_audio",
- "round": 3,
- "atMs": 15883.883875000058,
- "execMs": 0.024957999936304986
+ "atMs": 8721.821958000073,
+ "execMs": 0.012499999953433871
},
{
"name": "listen_audio",
"round": 3,
- "atMs": 15883.917166,
- "execMs": 0.003082999959588051
+ "atMs": 11650.543958000024,
+ "execMs": 0.030458000022917986
},
{
"name": "detect_silence",
"round": 4,
- "atMs": 18892.66308300011,
- "execMs": 0.02666600001975894
- },
- {
- "name": "split_at",
- "round": 5,
- "atMs": 30100.3417910001,
- "execMs": 0.21187499992083758
- },
- {
- "name": "split_at",
- "round": 5,
- "atMs": 30100.56295800011,
- "execMs": 0.023999999975785613
+ "atMs": 21852.912291000015,
+ "execMs": 0.022875000024214387
},
{
"name": "split_at",
"round": 5,
- "atMs": 30100.58933300001,
- "execMs": 0.3405000000493601
+ "atMs": 30733.51120800001,
+ "execMs": 0.02870899997651577
},
{
"name": "get_state",
"round": 6,
- "atMs": 34972.07620800007,
- "execMs": 0.06445800000801682
- },
- {
- "name": "delete_item",
- "round": 7,
- "atMs": 38990.038875000086,
- "execMs": 0.08170799992512912
+ "atMs": 32392.99229100009,
+ "execMs": 0.04724999994505197
},
{
- "name": "delete_item",
+ "name": "split_at",
"round": 7,
- "atMs": 38990.129541,
- "execMs": 0.04212499991990626
+ "atMs": 34234.52029100002,
+ "execMs": 0.0702080000191927
},
{
"name": "get_state",
"round": 8,
- "atMs": 40800.39425000001,
- "execMs": 0.020125000039115548
+ "atMs": 35913.17987500003,
+ "execMs": 0.040125000057742
},
{
- "name": "refine_speech_cuts",
+ "name": "delete_item",
"round": 9,
- "atMs": 48242.95625000005,
- "execMs": 0.028374999994412065
+ "atMs": 38197.875583000015,
+ "execMs": 0.04075000004377216
},
{
- "name": "subtitles_generate",
+ "name": "get_state",
"round": 10,
- "atMs": 53606.6653750001,
- "execMs": 0.03408300003502518
+ "atMs": 40431.78600000008,
+ "execMs": 0.03999999992083758
},
{
- "name": "get_state",
+ "name": "split_at",
"round": 11,
- "atMs": 57070.346166,
- "execMs": 0.013584000058472157
+ "atMs": 45108.43400000001,
+ "execMs": 0.0556250000372529
},
{
- "name": "update_cue",
+ "name": "get_state",
"round": 12,
- "atMs": 63027.89950000006,
- "execMs": 0.031833000015467405
+ "atMs": 47051.865457999986,
+ "execMs": 0.02083399996627122
},
{
- "name": "update_cue",
- "round": 12,
- "atMs": 63027.938083000015,
- "execMs": 0.035042000003159046
+ "name": "delete_item",
+ "round": 13,
+ "atMs": 50772.81633300008,
+ "execMs": 0.024624999961815774
},
{
- "name": "delete_cue",
- "round": 12,
- "atMs": 63027.974083000096,
- "execMs": 0.024542000028304756
+ "name": "get_state",
+ "round": 14,
+ "atMs": 52472.851250000065,
+ "execMs": 0.030291000031866133
+ },
+ {
+ "name": "refine_speech_cuts",
+ "round": 15,
+ "atMs": 56572.56400000001,
+ "execMs": 0.05391600006259978
+ },
+ {
+ "name": "get_state",
+ "round": 16,
+ "atMs": 58357.743790999986,
+ "execMs": 0.017124999896623194
+ },
+ {
+ "name": "subtitles_generate",
+ "round": 17,
+ "atMs": 63292.22658300004,
+ "execMs": 0.018416000064462423
+ },
+ {
+ "name": "get_state",
+ "round": 18,
+ "atMs": 65086.0695000001,
+ "execMs": 0.02716700010932982
+ },
+ {
+ "name": "delete_cue",
+ "round": 19,
+ "atMs": 70197.03370800009,
+ "execMs": 0.027875000028871
},
{
"name": "update_cue",
- "round": 12,
- "atMs": 63027.99933300004,
- "execMs": 0.0070000000996515155
+ "round": 20,
+ "atMs": 72228.5,
+ "execMs": 0.022624999983236194
},
{
"name": "update_cue",
- "round": 12,
- "atMs": 63028.00983300002,
- "execMs": 0.005374999949708581
+ "round": 21,
+ "atMs": 74018.45166600007,
+ "execMs": 0.060624999925494194
+ },
+ {
+ "name": "update_cue",
+ "round": 22,
+ "atMs": 76743.06675,
+ "execMs": 0.04104199993889779
+ },
+ {
+ "name": "update_cue",
+ "round": 23,
+ "atMs": 81290.08820800006,
+ "execMs": 0.0744999999878928
}
],
"timings": {
- "gateMs": 1943.248957999982,
+ "gateMs": 1144.2803330000024,
"gateSkipped": false,
- "ttftMs": 66457.75741600001,
- "modelMs": 65084.78253899992,
- "toolMs": 1.0887059998931363,
- "totalMs": 67029.1202029998,
+ "ttftMs": null,
+ "modelMs": 80143.51562399988,
+ "toolMs": 0.9633319999556988,
+ "totalMs": 81288.75928899983,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5792.448791999952,
+ "roundMs": 2944.0556250000373,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.02849999989848584
+ "execMs": 0.11829100002069026
}
]
},
{
- "roundMs": 1924.2863329999382,
+ "roundMs": 1569.0067499999423,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.030790999997407198
+ "execMs": 0.037499999976716936
}
]
},
{
- "roundMs": 3091.523541999981,
+ "roundMs": 3063.921166999964,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.010083000059239566
+ "execMs": 0.012499999953433871
}
]
},
{
- "roundMs": 3131.7155410000123,
+ "roundMs": 2928.668416999979,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.024957999936304986
- },
- {
- "name": "listen_audio",
- "execMs": 0.003082999959588051
+ "execMs": 0.030458000022917986
}
]
},
{
- "roundMs": 3008.60791599995,
+ "roundMs": 10202.284916999983,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.02666600001975894
+ "execMs": 0.022875000024214387
}
]
},
{
- "roundMs": 11207.49866599997,
+ "roundMs": 8880.547667000093,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.21187499992083758
- },
+ "execMs": 0.02870899997651577
+ }
+ ]
+ },
+ {
+ "roundMs": 1659.4211249999935,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "split_at",
- "execMs": 0.023999999975785613
- },
+ "name": "get_state",
+ "execMs": 0.04724999994505197
+ }
+ ]
+ },
+ {
+ "roundMs": 1841.4270419999957,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "split_at",
- "execMs": 0.3405000000493601
+ "execMs": 0.0702080000191927
}
]
},
{
- "roundMs": 4870.975375000038,
+ "roundMs": 1678.5183340000222,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.06445800000801682
+ "execMs": 0.040125000057742
}
]
},
{
- "roundMs": 4017.718916999991,
+ "roundMs": 2284.6154999999562,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.08170799992512912
- },
+ "execMs": 0.04075000004377216
+ }
+ ]
+ },
+ {
+ "roundMs": 2233.8256249999395,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.03999999992083758
+ }
+ ]
+ },
+ {
+ "roundMs": 4676.532624999993,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "split_at",
+ "execMs": 0.0556250000372529
+ }
+ ]
+ },
+ {
+ "roundMs": 1943.3203750000102,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.02083399996627122
+ }
+ ]
+ },
+ {
+ "roundMs": 3720.907332999981,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "delete_item",
- "execMs": 0.04212499991990626
+ "execMs": 0.024624999961815774
}
]
},
{
- "roundMs": 1810.0725409999723,
+ "roundMs": 1699.9810409999918,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.020125000039115548
+ "execMs": 0.030291000031866133
}
]
},
{
- "roundMs": 7442.502792000072,
+ "roundMs": 4099.639915999956,
"firstDeltaMs": null,
"calls": [
{
"name": "refine_speech_cuts",
- "execMs": 0.028374999994412065
+ "execMs": 0.05391600006259978
+ }
+ ]
+ },
+ {
+ "roundMs": 1785.0797909999965,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.017124999896623194
}
]
},
{
- "roundMs": 5363.612542000017,
+ "roundMs": 4934.436457999982,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.03408300003502518
+ "execMs": 0.018416000064462423
}
]
},
{
- "roundMs": 3463.5998329999857,
+ "roundMs": 1793.7982499999925,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.013584000058472157
+ "execMs": 0.02716700010932982
}
]
},
{
- "roundMs": 5957.496916000033,
+ "roundMs": 5110.894749999978,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.031833000015467405
- },
+ "name": "delete_cue",
+ "execMs": 0.027875000028871
+ }
+ ]
+ },
+ {
+ "roundMs": 2031.4070000000065,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "update_cue",
- "execMs": 0.035042000003159046
- },
- {
- "name": "delete_cue",
- "execMs": 0.024542000028304756
- },
+ "execMs": 0.022624999983236194
+ }
+ ]
+ },
+ {
+ "roundMs": 1789.7792500000214,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "update_cue",
- "execMs": 0.0070000000996515155
- },
+ "execMs": 0.060624999925494194
+ }
+ ]
+ },
+ {
+ "roundMs": 2724.520416000043,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "update_cue",
- "execMs": 0.005374999949708581
+ "execMs": 0.04104199993889779
}
]
},
{
- "roundMs": 4002.722833000007,
- "firstDeltaMs": 3429.7181660000933,
- "calls": []
+ "roundMs": 4546.926250000019,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_cue",
+ "execMs": 0.0744999999878928
+ }
+ ]
}
]
}
@@ -4140,452 +3922,401 @@
"pass": false,
"intent": "work",
"notes": [
- "reply did not match /filler|um|uh|cut|remove|trim/i",
- "mutating tools called: add_clip"
+ "reply did not match /filler|um|uh|cut|remove|trim/i"
],
"trace": [
{
"name": "list_skills",
"round": 0,
- "atMs": 5205.158917000052,
- "execMs": 0.02549999998882413
+ "atMs": 7294.425917000044,
+ "execMs": 0.07191699999384582
},
{
"name": "read_skill",
"round": 1,
- "atMs": 6905.553125000093,
- "execMs": 0.01679100003093481
+ "atMs": 9455.55712500005,
+ "execMs": 0.0036669999826699495
},
{
- "name": "detect_silence",
+ "name": "read_skill",
"round": 2,
- "atMs": 10020.787167000002,
- "execMs": 0.006458000047132373
+ "atMs": 11932.650583000039,
+ "execMs": 0.009750000084750354
},
{
- "name": "listen_audio",
+ "name": "detect_silence",
"round": 3,
- "atMs": 13680.308042000048,
- "execMs": 0.007375000044703484
+ "atMs": 15303.428167000064,
+ "execMs": 0.11304099997505546
},
{
- "name": "detect_silence",
+ "name": "listen_audio",
"round": 4,
- "atMs": 18646.820459000068,
- "execMs": 0.017499999958090484
- },
- {
- "name": "split_at",
- "round": 5,
- "atMs": 33314.92320900003,
- "execMs": 0.22079199994914234
- },
- {
- "name": "split_at",
- "round": 5,
- "atMs": 33315.229625000036,
- "execMs": 0.0816249999916181
+ "atMs": 17971.717000000062,
+ "execMs": 0.008874999941326678
},
{
- "name": "split_at",
+ "name": "detect_silence",
"round": 5,
- "atMs": 33315.316458999994,
- "execMs": 0.28666700003668666
+ "atMs": 25217.813916999963,
+ "execMs": 0.013584000058472157
},
{
- "name": "get_state",
+ "name": "read_skill",
"round": 6,
- "atMs": 38523.27041700005,
- "execMs": 0.03683300002012402
- },
- {
- "name": "undo",
- "round": 7,
- "atMs": 45161.382375000045,
- "execMs": 0.03025000006891787
- },
- {
- "name": "undo",
- "round": 7,
- "atMs": 45161.43195900007,
- "execMs": 0.057625000015832484
+ "atMs": 32892.402458,
+ "execMs": 0.03437499993015081
},
{
- "name": "undo",
+ "name": "split_at",
"round": 7,
- "atMs": 45161.49312500004,
- "execMs": 0.001999999978579581
+ "atMs": 55374.66733299999,
+ "execMs": 0.23045900010038167
},
{
"name": "get_state",
"round": 8,
- "atMs": 47424.82141700003,
- "execMs": 0.06637500005308539
+ "atMs": 57445.114333000034,
+ "execMs": 0.03979099995922297
},
{
- "name": "select",
+ "name": "undo",
"round": 9,
- "atMs": 50600.447334000026,
- "execMs": 0.016959000029601157
+ "atMs": 61515.44949999999,
+ "execMs": 0.005249999929219484
},
{
- "name": "split_at",
+ "name": "get_state",
"round": 10,
- "atMs": 52681.67341699998,
- "execMs": 0.081499999971129
+ "atMs": 66169.97712499998,
+ "execMs": 0.021958000026643276
},
{
- "name": "get_state",
+ "name": "select",
"round": 11,
- "atMs": 54453.97483399999,
- "execMs": 0.06716599990613759
+ "atMs": 68756.15254200005,
+ "execMs": 0.015959000098519027
},
{
- "name": "select",
+ "name": "split_at",
"round": 12,
- "atMs": 57085.643667,
- "execMs": 0.06654100003652275
+ "atMs": 70469.008333,
+ "execMs": 0.08399999991524965
},
{
- "name": "split_at",
+ "name": "get_state",
"round": 13,
- "atMs": 58870.252125,
- "execMs": 0.577832999988459
+ "atMs": 72288.31662499998,
+ "execMs": 0.2920000000158325
},
{
- "name": "get_state",
+ "name": "undo",
"round": 14,
- "atMs": 60461.18520900002,
- "execMs": 0.03683300002012402
+ "atMs": 79636.21812500001,
+ "execMs": 0.02174999995622784
},
{
- "name": "select",
+ "name": "get_state",
"round": 15,
- "atMs": 63224.8440840001,
- "execMs": 0.11225000000558794
+ "atMs": 82597.51083300007,
+ "execMs": 0.09404100000392646
},
{
- "name": "split_at",
+ "name": "update_audio",
"round": 16,
- "atMs": 65146.45562500006,
- "execMs": 0.21054200001526624
+ "atMs": 88307.51237500005,
+ "execMs": 0.49420900002587587
},
{
"name": "get_state",
"round": 17,
- "atMs": 67235.81929200003,
- "execMs": 0.2540830000070855
+ "atMs": 90559.87087500002,
+ "execMs": 0.04804199992213398
},
{
- "name": "delete_item",
+ "name": "split_at",
"round": 18,
- "atMs": 76247.850125,
- "execMs": 0.14341600006446242
+ "atMs": 94733.62383299996,
+ "execMs": 0.07295900001190603
},
{
- "name": "get_state",
+ "name": "split_at",
"round": 19,
- "atMs": 77966.62829200004,
- "execMs": 0.02925000002142042
+ "atMs": 96850.27037499996,
+ "execMs": 0.02383299998473376
},
{
- "name": "delete_item",
+ "name": "split_at",
"round": 20,
- "atMs": 81629.95520900004,
- "execMs": 0.038416999974288046
+ "atMs": 100653.97787499998,
+ "execMs": 0.11100000003352761
},
{
"name": "get_state",
"round": 21,
- "atMs": 84501.18237500009,
- "execMs": 0.0656249999301508
- },
- {
- "name": "delete_item",
- "round": 22,
- "atMs": 88557.79070900008,
- "execMs": 0.3622909999685362
+ "atMs": 103147.17304200004,
+ "execMs": 0.09295799990650266
},
{
"name": "delete_item",
"round": 22,
- "atMs": 88558.18254200008,
- "execMs": 0.08004099992103875
+ "atMs": 106258.62270800001,
+ "execMs": 0.0232079999987036
},
{
- "name": "add_clip",
+ "name": "get_state",
"round": 23,
- "atMs": 90683.31854200002,
- "execMs": 0.08899999991990626
+ "atMs": 108248.44666700007,
+ "execMs": 0.1432080001104623
}
],
"timings": {
- "gateMs": 1197.2662500001024,
+ "gateMs": 1175.1465000000317,
"gateSkipped": false,
"ttftMs": null,
- "modelMs": 89479.88816799992,
- "toolMs": 3.087537999963388,
- "totalMs": 90680.24195599998,
+ "modelMs": 107069.72545799997,
+ "toolMs": 2.06983399996534,
+ "totalMs": 108246.94179199997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4007.4417080000276,
+ "roundMs": 6118.945583000081,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.02549999998882413
+ "execMs": 0.07191699999384582
}
]
},
{
- "roundMs": 1700.281083000009,
+ "roundMs": 2161.0177919999696,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.01679100003093481
+ "execMs": 0.0036669999826699495
}
]
},
{
- "roundMs": 3115.1650829999708,
+ "roundMs": 2477.0556250000373,
"firstDeltaMs": null,
"calls": [
{
- "name": "detect_silence",
- "execMs": 0.006458000047132373
+ "name": "read_skill",
+ "execMs": 0.009750000084750354
}
]
},
{
- "roundMs": 3659.4942919999594,
+ "roundMs": 3370.691792000085,
"firstDeltaMs": null,
"calls": [
{
- "name": "listen_audio",
- "execMs": 0.007375000044703484
+ "name": "detect_silence",
+ "execMs": 0.11304099997505546
}
]
},
{
- "roundMs": 4966.4688329999335,
+ "roundMs": 2668.090291999979,
"firstDeltaMs": null,
"calls": [
{
- "name": "detect_silence",
- "execMs": 0.017499999958090484
+ "name": "listen_audio",
+ "execMs": 0.008874999941326678
}
]
},
{
- "roundMs": 14668.022375000059,
+ "roundMs": 7246.064707999933,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.22079199994914234
- },
- {
- "name": "split_at",
- "execMs": 0.0816249999916181
- },
- {
- "name": "split_at",
- "execMs": 0.28666700003668666
+ "name": "detect_silence",
+ "execMs": 0.013584000058472157
}
]
},
{
- "roundMs": 5207.606458000024,
+ "roundMs": 7674.462249999982,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.03683300002012402
+ "name": "read_skill",
+ "execMs": 0.03437499993015081
}
]
},
{
- "roundMs": 6638.005042000092,
+ "roundMs": 22482.13387499994,
"firstDeltaMs": null,
"calls": [
{
- "name": "undo",
- "execMs": 0.03025000006891787
- },
- {
- "name": "undo",
- "execMs": 0.057625000015832484
- },
- {
- "name": "undo",
- "execMs": 0.001999999978579581
+ "name": "split_at",
+ "execMs": 0.23045900010038167
}
]
},
{
- "roundMs": 2263.2552089999663,
+ "roundMs": 2070.166041999939,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.06637500005308539
+ "execMs": 0.03979099995922297
}
]
},
{
- "roundMs": 3175.4957919999724,
+ "roundMs": 4070.269915999961,
"firstDeltaMs": null,
"calls": [
{
- "name": "select",
- "execMs": 0.016959000029601157
+ "name": "undo",
+ "execMs": 0.005249999929219484
}
]
},
{
- "roundMs": 2080.673666999908,
+ "roundMs": 4654.501791999908,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.081499999971129
+ "name": "get_state",
+ "execMs": 0.021958000026643276
}
]
},
{
- "roundMs": 1772.1576249999925,
+ "roundMs": 2586.130833000061,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.06716599990613759
+ "name": "select",
+ "execMs": 0.015959000098519027
}
]
},
{
- "roundMs": 2631.5337079999736,
+ "roundMs": 1712.7889999999898,
"firstDeltaMs": null,
"calls": [
{
- "name": "select",
- "execMs": 0.06654100003652275
+ "name": "split_at",
+ "execMs": 0.08399999991524965
}
]
},
{
- "roundMs": 1784.4599999999627,
+ "roundMs": 1819.1633750000037,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.577832999988459
+ "name": "get_state",
+ "execMs": 0.2920000000158325
}
]
},
{
- "roundMs": 1590.2592500000028,
+ "roundMs": 7347.547791999998,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.03683300002012402
+ "name": "undo",
+ "execMs": 0.02174999995622784
}
]
},
{
- "roundMs": 2763.338917000103,
+ "roundMs": 2961.2286669999594,
"firstDeltaMs": null,
"calls": [
{
- "name": "select",
- "execMs": 0.11225000000558794
+ "name": "get_state",
+ "execMs": 0.09404100000392646
}
]
},
{
- "roundMs": 1921.343332999968,
+ "roundMs": 5709.8162909999955,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.21054200001526624
+ "name": "update_audio",
+ "execMs": 0.49420900002587587
}
]
},
{
- "roundMs": 2089.0462919999845,
+ "roundMs": 2251.7637080000713,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.2540830000070855
+ "execMs": 0.04804199992213398
}
]
},
{
- "roundMs": 9011.704083000077,
+ "roundMs": 4173.647750000004,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.14341600006446242
+ "name": "split_at",
+ "execMs": 0.07295900001190603
}
]
},
{
- "roundMs": 1718.5892499999609,
+ "roundMs": 2116.530916000018,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.02925000002142042
+ "name": "split_at",
+ "execMs": 0.02383299998473376
}
]
},
{
- "roundMs": 3663.2652920000255,
+ "roundMs": 3803.6332089999923,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.038416999974288046
+ "name": "split_at",
+ "execMs": 0.11100000003352761
}
]
},
{
- "roundMs": 2871.135208999971,
+ "roundMs": 2493.014582999982,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.0656249999301508
+ "execMs": 0.09295799990650266
}
]
},
{
- "roundMs": 4056.188582999981,
+ "roundMs": 3111.3034169999883,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.3622909999685362
- },
- {
- "name": "delete_item",
- "execMs": 0.08004099992103875
+ "execMs": 0.0232079999987036
}
]
},
{
- "roundMs": 2124.957083999994,
+ "roundMs": 1989.7562500000931,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.08899999991990626
+ "name": "get_state",
+ "execMs": 0.1432080001104623
}
]
}
@@ -4601,27 +4332,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1229,
- "p95": 1501,
- "mean": 1264,
+ "p50": 1417,
+ "p95": 1918,
+ "mean": 1491,
"n": 3
},
"ttftMs": {
- "p50": 6516,
- "p95": 9281,
- "mean": 7210,
+ "p50": 5938,
+ "p95": 10035,
+ "mean": 7225,
"n": 3
},
"totalMs": {
- "p50": 6808,
- "p95": 10044,
- "mean": 7666,
+ "p50": 6097,
+ "p95": 10122,
+ "mean": 7414,
"n": 3
},
"modelMs": {
- "p50": 5307,
- "p95": 8815,
- "mean": 6402,
+ "p50": 4959,
+ "p95": 8204,
+ "mean": 5922,
"n": 3
},
"toolMs": {
@@ -4644,33 +4375,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 3982.4909580000676,
- "execMs": 0.010249999933876097
+ "atMs": 4252.135875000036,
+ "execMs": 0.04962499998509884
}
],
"timings": {
- "gateMs": 1228.9701659999555,
+ "gateMs": 1416.831165999989,
"gateSkipped": false,
- "ttftMs": 9280.621583000058,
- "modelMs": 8815.001999999979,
- "toolMs": 0.010249999933876097,
- "totalMs": 10043.982415999868,
+ "ttftMs": 5937.940375000006,
+ "modelMs": 4604.543543000007,
+ "toolMs": 0.04962499998509884,
+ "totalMs": 6021.424333999981,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2752.684332999983,
+ "roundMs": 2834.8823339999653,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.010249999933876097
+ "execMs": 0.04962499998509884
}
]
},
{
- "roundMs": 6062.317666999996,
- "firstDeltaMs": 5298.071250000037,
+ "roundMs": 1769.6612090000417,
+ "firstDeltaMs": 1685.7185000000754,
"calls": []
}
]
@@ -4684,33 +4415,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 4414.366708999965,
- "execMs": 0.027457999996840954
+ "atMs": 3860.2290839999914,
+ "execMs": 0.0043750000186264515
}
],
"timings": {
- "gateMs": 1501.4409999999916,
+ "gateMs": 1138.5568750000093,
"gateSkipped": false,
- "ttftMs": 6516.473791999975,
- "modelMs": 5306.791874999995,
- "toolMs": 0.027457999996840954,
- "totalMs": 6808.260332999984,
+ "ttftMs": 5702.3887920000125,
+ "modelMs": 4958.581792000099,
+ "toolMs": 0.0043750000186264515,
+ "totalMs": 6097.143042000127,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2912.4913330000127,
+ "roundMs": 2721.4691250000615,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.027457999996840954
+ "execMs": 0.0043750000186264515
}
]
},
{
- "roundMs": 2394.3005419999827,
- "firstDeltaMs": 2102.0146249999525,
+ "roundMs": 2237.1126670000376,
+ "firstDeltaMs": 1842.1448749999981,
"calls": []
}
]
@@ -4724,33 +4455,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 4154.171333000064,
- "execMs": 0.003000000026077032
+ "atMs": 7848.685791999917,
+ "execMs": 0.01120799989439547
}
],
"timings": {
- "gateMs": 1062.0017919999082,
+ "gateMs": 1918.0310419999296,
"gateSkipped": false,
- "ttftMs": 5831.74462500005,
- "modelMs": 5085.036874999991,
- "toolMs": 0.003000000026077032,
- "totalMs": 6147.041666999925,
+ "ttftMs": 10035.33579199994,
+ "modelMs": 8203.909541999921,
+ "toolMs": 0.01120799989439547,
+ "totalMs": 10121.951791999745,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3092.020750000025,
+ "roundMs": 5930.4807919999585,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.003000000026077032
+ "execMs": 0.01120799989439547
}
]
},
{
- "roundMs": 1993.0161249999655,
- "firstDeltaMs": 1677.5506250000326,
+ "roundMs": 2273.4287499999627,
+ "firstDeltaMs": 2186.6057080000173,
"calls": []
}
]
@@ -4765,27 +4496,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1250,
- "p95": 1688,
- "mean": 1321,
+ "p50": 1130,
+ "p95": 1260,
+ "mean": 1173,
"n": 3
},
"ttftMs": {
- "p50": 7194,
- "p95": 19155,
- "mean": 11033,
+ "p50": 10676,
+ "p95": 21734,
+ "mean": 14333,
"n": 3
},
"totalMs": {
- "p50": 7376,
- "p95": 19556,
- "mean": 11381,
+ "p50": 10956,
+ "p95": 22053,
+ "mean": 14613,
"n": 3
},
"modelMs": {
- "p50": 5961,
- "p95": 18530,
- "mean": 10060,
+ "p50": 9828,
+ "p95": 20793,
+ "mean": 13441,
"n": 3
},
"toolMs": {
@@ -4795,7 +4526,7 @@
"n": 3
},
"rounds": {
- "mean": 3,
+ "mean": 3.7,
"max": 5
}
},
@@ -4806,75 +4537,51 @@
"notes": [],
"trace": [
{
- "name": "generate_scene",
+ "name": "listen_audio",
"round": 0,
- "atMs": 5549.615416000015,
- "execMs": 0.16833399992901832
+ "atMs": 6012.60625000007,
+ "execMs": 0.18070899997837842
+ },
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 8634.772624999983,
+ "execMs": 0.2885839999653399
}
],
"timings": {
- "gateMs": 1688.0723749999888,
+ "gateMs": 1127.6795409999322,
"gateSkipped": false,
- "ttftMs": 7194.066416000016,
- "modelMs": 5688.172416000045,
- "toolMs": 0.16833399992901832,
- "totalMs": 7376.413124999963,
+ "ttftMs": 10676.018459000043,
+ "modelMs": 9828.11570900015,
+ "toolMs": 0.4692929999437183,
+ "totalMs": 10956.264543000027,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3861.233125000028,
+ "roundMs": 4884.26791699999,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.16833399992901832
+ "name": "listen_audio",
+ "execMs": 0.18070899997837842
}
]
},
{
- "roundMs": 1826.939291000017,
- "firstDeltaMs": 1644.2579160000896,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "work",
- "notes": [],
- "trace": [
- {
- "name": "generate_scene",
- "round": 0,
- "atMs": 5139.047791999998,
- "execMs": 0.008541999966837466
- }
- ],
- "timings": {
- "gateMs": 1249.5929999999935,
- "gateSkipped": false,
- "ttftMs": 6749.667375000077,
- "modelMs": 5961.494582000072,
- "toolMs": 0.008541999966837466,
- "totalMs": 7211.096124000032,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3889.022291000001,
+ "roundMs": 2621.93437500007,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.008541999966837466
+ "execMs": 0.2885839999653399
}
]
},
{
- "roundMs": 2072.472291000071,
- "firstDeltaMs": 1610.5825000000186,
+ "roundMs": 2321.9134170000907,
+ "firstDeltaMs": 2040.9205420000944,
"calls": []
}
]
@@ -4888,81 +4595,137 @@
{
"name": "list_skills",
"round": 0,
- "atMs": 4813.118833999964,
- "execMs": 0.05570900009479374
+ "atMs": 5742.512416000012,
+ "execMs": 0.04574999993201345
},
{
"name": "read_skill",
"round": 1,
- "atMs": 6628.777584000025,
- "execMs": 0.09645800001453608
+ "atMs": 7470.928541000001,
+ "execMs": 0.020249999943189323
},
{
"name": "listen_audio",
"round": 2,
- "atMs": 8785.503999999957,
- "execMs": 0.09416700003203005
+ "atMs": 9906.21666600008,
+ "execMs": 0.03966700006276369
},
{
"name": "generate_scene",
"round": 3,
- "atMs": 16306.41066699999,
- "execMs": 0.08683300006669015
+ "atMs": 19754.962083000108,
+ "execMs": 0.009250000002793968
}
],
"timings": {
- "gateMs": 1025.1362910000607,
+ "gateMs": 1260.2202920001,
"gateSkipped": false,
- "ttftMs": 19155.11966700002,
- "modelMs": 18530.232916000066,
- "toolMs": 0.33316700020805,
- "totalMs": 19555.702374000335,
+ "ttftMs": 21733.986416000058,
+ "modelMs": 20792.967835000018,
+ "toolMs": 0.11491699994076043,
+ "totalMs": 22053.30304400006,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3787.695832999889,
+ "roundMs": 4481.87729199999,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.05570900009479374
+ "execMs": 0.04574999993201345
}
]
},
{
- "roundMs": 1815.362625000067,
+ "roundMs": 1728.1141670000507,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.09645800001453608
+ "execMs": 0.020249999943189323
+ }
+ ]
+ },
+ {
+ "roundMs": 2435.174958999967,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.03966700006276369
+ }
+ ]
+ },
+ {
+ "roundMs": 9848.64300000004,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "generate_scene",
+ "execMs": 0.009250000002793968
}
]
},
{
- "roundMs": 2156.5238329999847,
+ "roundMs": 2299.1584169999696,
+ "firstDeltaMs": 1978.997042000061,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "work",
+ "notes": [],
+ "trace": [
+ {
+ "name": "listen_audio",
+ "round": 0,
+ "atMs": 6087.0756249999395,
+ "execMs": 0.007291999994777143
+ },
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 8722.273959000013,
+ "execMs": 0.015458000008948147
+ }
+ ],
+ "timings": {
+ "gateMs": 1129.6821250000503,
+ "gateSkipped": false,
+ "ttftMs": 10588.365166999982,
+ "modelMs": 9701.216917000012,
+ "toolMs": 0.02275000000372529,
+ "totalMs": 10830.921792000066,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 4957.283374999999,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.09416700003203005
+ "execMs": 0.007291999994777143
}
]
},
{
- "roundMs": 7520.746000000043,
+ "roundMs": 2635.1518749999814,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.08683300006669015
+ "execMs": 0.015458000008948147
}
]
},
{
- "roundMs": 3249.904625000083,
- "firstDeltaMs": 2848.512209000066,
+ "roundMs": 2108.781667000032,
+ "firstDeltaMs": 1866.0565419999184,
"calls": []
}
]
@@ -4977,32 +4740,32 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1105,
- "p95": 1902,
- "mean": 1357,
+ "p50": 1162,
+ "p95": 1226,
+ "mean": 1091,
"n": 3
},
"ttftMs": {
- "p50": 7978,
- "p95": 11359,
- "mean": 8803,
+ "p50": 7734,
+ "p95": 9790,
+ "mean": 8335,
"n": 3
},
"totalMs": {
- "p50": 8225,
- "p95": 11757,
- "mean": 9157,
+ "p50": 8168,
+ "p95": 10570,
+ "mean": 8952,
"n": 3
},
"modelMs": {
- "p50": 6421,
- "p95": 10652,
- "mean": 7799,
+ "p50": 7005,
+ "p95": 9686,
+ "mean": 7861,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
@@ -5020,33 +4783,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 5300.302625000011,
- "execMs": 0.5280829999828711
+ "atMs": 5684.724625000032,
+ "execMs": 0.09216599992942065
}
],
"timings": {
- "gateMs": 1065.4159999999683,
+ "gateMs": 1162.264957999927,
"gateSkipped": false,
- "ttftMs": 7070.814374999958,
- "modelMs": 6421.329167000018,
- "toolMs": 0.5280829999828711,
- "totalMs": 7487.273249999969,
+ "ttftMs": 7734.3421249999665,
+ "modelMs": 7005.268666999997,
+ "toolMs": 0.09216599992942065,
+ "totalMs": 8167.625790999853,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4234.12200000009,
+ "roundMs": 4522.203584000003,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.5280829999828711
+ "execMs": 0.09216599992942065
}
]
},
{
- "roundMs": 2187.2071669999277,
- "firstDeltaMs": 1769.8976249999832,
+ "roundMs": 2483.065082999994,
+ "firstDeltaMs": 2049.503541000071,
"calls": []
}
]
@@ -5060,33 +4823,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 6561.3610409999965,
- "execMs": 0.012916999985463917
+ "atMs": 7584.816249999916,
+ "execMs": 0.016500000027008355
}
],
"timings": {
- "gateMs": 1105.0622500000754,
+ "gateMs": 883.7283340001013,
"gateSkipped": false,
- "ttftMs": 11359.004041000037,
- "modelMs": 10652.314125000034,
- "toolMs": 0.012916999985463917,
- "totalMs": 11757.389292000094,
+ "ttftMs": 9789.676457999973,
+ "modelMs": 9685.934626000002,
+ "toolMs": 0.016500000027008355,
+ "totalMs": 10569.67946000013,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5455.993166,
+ "roundMs": 6700.9339169999585,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.012916999985463917
+ "execMs": 0.016500000027008355
}
]
},
{
- "roundMs": 5196.320959000033,
- "firstDeltaMs": 4797.619792000041,
+ "roundMs": 2985.0007090000436,
+ "firstDeltaMs": 2204.8192920000292,
"calls": []
}
]
@@ -5100,33 +4863,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 6339.782750000013,
- "execMs": 0.04808300000149757
+ "atMs": 5621.536666000029,
+ "execMs": 0.06941599992569536
}
],
"timings": {
- "gateMs": 1901.5523749999702,
+ "gateMs": 1226.2375410000095,
"gateSkipped": false,
- "ttftMs": 7978.290583000053,
- "modelMs": 6323.66737400007,
- "toolMs": 0.04808300000149757,
- "totalMs": 8225.267832000041,
+ "ttftMs": 7480.844999999972,
+ "modelMs": 6893.258583000046,
+ "toolMs": 0.06941599992569536,
+ "totalMs": 8119.565539999981,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4437.754582999973,
+ "roundMs": 4394.89391600003,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.04808300000149757
+ "execMs": 0.06941599992569536
}
]
},
{
- "roundMs": 1885.912791000097,
- "firstDeltaMs": 1638.268041000003,
+ "roundMs": 2498.364667000016,
+ "firstDeltaMs": 1859.1742089999607,
"calls": []
}
]
@@ -5137,42 +4900,42 @@
{
"name": "cartoon-style-reference-anchors-look",
"bucket": "single-tool",
- "passRate": 1,
+ "passRate": 0.3333333333333333,
"budgetBreaches": [],
"latency": {
"gateMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 1
},
"ttftMs": {
- "p50": 8684,
- "p95": 17392,
- "mean": 11248,
- "n": 3
+ "p50": 10582,
+ "p95": 10582,
+ "mean": 10582,
+ "n": 1
},
"totalMs": {
- "p50": 9180,
- "p95": 17667,
- "mean": 11627,
- "n": 3
+ "p50": 11134,
+ "p95": 11134,
+ "mean": 11134,
+ "n": 1
},
"modelMs": {
- "p50": 9179,
- "p95": 17666,
- "mean": 11627,
- "n": 3
+ "p50": 11133,
+ "p95": 11133,
+ "mean": 11133,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 1
},
"rounds": {
- "mean": 3.3,
- "max": 5
+ "mean": 2,
+ "max": 2
}
},
"runs": [
@@ -5184,38 +4947,98 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 5481.014542000019,
- "execMs": 0.24737500003539026
+ "atMs": 8623.969249999966,
+ "execMs": 0.1489999999757856
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 7667.011916999938,
- "modelMs": 8035.919165999978,
- "toolMs": 0.24737500003539026,
- "totalMs": 8036.166541000013,
+ "ttftMs": 10581.791958000045,
+ "modelMs": 11133.411500000046,
+ "toolMs": 0.1489999999757856,
+ "totalMs": 11133.560500000021,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5480.352791000041,
+ "roundMs": 8623.429250000045,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.24737500003539026
+ "execMs": 0.1489999999757856
}
]
},
{
- "roundMs": 2555.5663749999367,
- "firstDeltaMs": 2185.6620419999817,
+ "roundMs": 2509.982250000001,
+ "firstDeltaMs": 1957.6608749999432,
"calls": []
}
]
}
},
+ {
+ "pass": false,
+ "intent": "work",
+ "notes": [],
+ "trace": [],
+ "timings": null,
+ "error": "responses 502: {\"error\":\"provider_error\",\"message\":\"Gemini request failed.\",\"details\":{\"message\":\"request to https://oauth2.googleapis.com/token failed, reason: getaddrinfo ENOTFOUND oauth2.googleapis.com\"}}"
+ },
+ {
+ "pass": false,
+ "intent": "work",
+ "notes": [],
+ "trace": [],
+ "timings": null,
+ "error": "responses 502: {\"error\":\"provider_error\",\"message\":\"Gemini request failed.\",\"details\":{\"message\":\"request to https://oauth2.googleapis.com/token failed, reason: getaddrinfo ENOTFOUND oauth2.googleapis.com\"}}"
+ }
+ ]
+ },
+ {
+ "name": "cartoon-question-stays-in-chat",
+ "bucket": "chat",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 1431,
+ "p95": 1503,
+ "mean": 1313,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 17580,
+ "p95": 21466,
+ "mean": 17576,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 14566,
+ "p95": 22181,
+ "mean": 16988,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 13063,
+ "p95": 21176,
+ "mean": 15675,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 3.3,
+ "max": 4
+ }
+ },
+ "runs": [
{
"pass": true,
"intent": "work",
@@ -5224,81 +5047,105 @@
{
"name": "list_skills",
"round": 0,
- "atMs": 4801.339500000002,
- "execMs": 0.08195900009013712
+ "atMs": 10182.743916000007,
+ "execMs": 0.054875000030733645
},
{
"name": "read_skill",
"round": 1,
- "atMs": 6172.381916000042,
- "execMs": 0.011042000027373433
- },
- {
- "name": "listen_audio",
- "round": 2,
- "atMs": 8546.882041000063,
- "execMs": 0.05329100007656962
- },
- {
- "name": "generate_scene",
- "round": 3,
- "atMs": 15298.674583000015,
- "execMs": 0.03808299999218434
+ "atMs": 12247.328625000082,
+ "execMs": 0.003666000091470778
}
],
"timings": {
- "gateMs": 0,
- "gateSkipped": true,
- "ttftMs": 17391.858416000032,
- "modelMs": 17666.389541999903,
- "toolMs": 0.18437500018626451,
- "totalMs": 17666.57391700009,
- "retryMs": 0,
- "retries": 0,
+ "gateMs": 1503.4509170000674,
+ "gateSkipped": false,
+ "ttftMs": 17580.448041000054,
+ "modelMs": 13062.591917000012,
+ "toolMs": 0.05854100012220442,
+ "totalMs": 14566.101375000202,
+ "retryMs": 3678.8114160000114,
+ "retries": 2,
"rounds": [
{
- "roundMs": 4800.977375000017,
+ "roundMs": 4999.322959000012,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.08195900009013712
+ "execMs": 0.054875000030733645
}
]
},
{
- "roundMs": 1370.87820799998,
+ "roundMs": 2064.383166000014,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.011042000027373433
+ "execMs": 0.003666000091470778
}
]
},
{
- "roundMs": 2374.409667000058,
+ "roundMs": 5998.885791999986,
+ "firstDeltaMs": 5333.080916999956,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "work",
+ "notes": [],
+ "trace": [
+ {
+ "name": "list_skills",
+ "round": 0,
+ "atMs": 8490.23570900003,
+ "execMs": 0.030000000027939677
+ },
+ {
+ "name": "read_skill",
+ "round": 1,
+ "atMs": 10287.300749999937,
+ "execMs": 0.006999999983236194
+ }
+ ],
+ "timings": {
+ "gateMs": 1431.001209000009,
+ "gateSkipped": false,
+ "ttftMs": 13682.181499999948,
+ "modelMs": 12785.041916000075,
+ "toolMs": 0.03700000001117587,
+ "totalMs": 14216.080125000095,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 7058.742582999985,
"firstDeltaMs": null,
"calls": [
{
- "name": "listen_audio",
- "execMs": 0.05329100007656962
+ "name": "list_skills",
+ "execMs": 0.030000000027939677
}
]
},
{
- "roundMs": 6751.664208999951,
+ "roundMs": 1796.9295410000486,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.03808299999218434
+ "name": "read_skill",
+ "execMs": 0.006999999983236194
}
]
},
{
- "roundMs": 2368.4600829998963,
- "firstDeltaMs": 2093.079124999931,
+ "roundMs": 3929.3697920000413,
+ "firstDeltaMs": 3394.7997090000426,
"calls": []
}
]
@@ -5310,51 +5157,67 @@
"notes": [],
"trace": [
{
- "name": "listen_audio",
+ "name": "list_skills",
"round": 0,
- "atMs": 4626.89383299998,
- "execMs": 0.008542000083252788
+ "atMs": 8112.074250000063,
+ "execMs": 0.03479099995456636
},
{
- "name": "generate_scene",
+ "name": "read_skill",
"round": 1,
- "atMs": 6933.956542000058,
- "execMs": 0.023167000035755336
+ "atMs": 10538.826750000007,
+ "execMs": 0.00633400003425777
+ },
+ {
+ "name": "listen_audio",
+ "round": 2,
+ "atMs": 13384.826625000103,
+ "execMs": 0.022166999988257885
}
],
"timings": {
- "gateMs": 0,
- "gateSkipped": true,
- "ttftMs": 8683.935999999987,
- "modelMs": 9179.497790999943,
- "toolMs": 0.031709000119008124,
- "totalMs": 9179.529500000062,
+ "gateMs": 1004.6382920000469,
+ "gateSkipped": false,
+ "ttftMs": 21465.826833000057,
+ "modelMs": 21176.360084000044,
+ "toolMs": 0.06329199997708201,
+ "totalMs": 22181.061668000068,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4626.684583000024,
+ "roundMs": 7107.237042000052,
"firstDeltaMs": null,
"calls": [
{
- "name": "listen_audio",
- "execMs": 0.008542000083252788
+ "name": "list_skills",
+ "execMs": 0.03479099995456636
}
]
},
{
- "roundMs": 2306.9598749999423,
+ "roundMs": 2426.665874999948,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.023167000035755336
+ "name": "read_skill",
+ "execMs": 0.00633400003425777
+ }
+ ]
+ },
+ {
+ "roundMs": 2845.935625000042,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.022166999988257885
}
]
},
{
- "roundMs": 2245.8533329999773,
- "firstDeltaMs": 1749.908582999953,
+ "roundMs": 8796.521542000002,
+ "firstDeltaMs": 8080.920542000094,
"calls": []
}
]
@@ -5363,33 +5226,33 @@
]
},
{
- "name": "cartoon-question-stays-in-chat",
- "bucket": "chat",
+ "name": "clip-mentions-redo-their-shots",
+ "bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1778,
- "p95": 1871,
- "mean": 1647,
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
"n": 3
},
"ttftMs": {
- "p50": 6861,
- "p95": 8540,
- "mean": 7135,
+ "p50": 23625,
+ "p95": 34241,
+ "mean": 25744,
"n": 3
},
"totalMs": {
- "p50": 7449,
- "p95": 8941,
- "mean": 7666,
+ "p50": 23887,
+ "p95": 34675,
+ "mean": 26104,
"n": 3
},
"modelMs": {
- "p50": 5578,
- "p95": 7163,
- "mean": 6019,
+ "p50": 23887,
+ "p95": 34675,
+ "mean": 26104,
"n": 3
},
"toolMs": {
@@ -5399,8 +5262,8 @@
"n": 3
},
"rounds": {
- "mean": 1,
- "max": 1
+ "mean": 6.3,
+ "max": 9
}
},
"runs": [
@@ -5408,485 +5271,345 @@
"pass": true,
"intent": "work",
"notes": [],
- "trace": [],
+ "trace": [
+ {
+ "name": "listen_audio",
+ "round": 0,
+ "atMs": 6766.635249999934,
+ "execMs": 0.1645420000422746
+ },
+ {
+ "name": "watch_video",
+ "round": 1,
+ "atMs": 12356.706459000008,
+ "execMs": 0.008749999920837581
+ },
+ {
+ "name": "watch_video",
+ "round": 1,
+ "atMs": 12356.725499999942,
+ "execMs": 0.000750000006519258
+ },
+ {
+ "name": "list_skills",
+ "round": 2,
+ "atMs": 15037.702416999964,
+ "execMs": 0.022499999962747097
+ },
+ {
+ "name": "read_skill",
+ "round": 3,
+ "atMs": 16675.16658399999,
+ "execMs": 0.0058750000316649675
+ },
+ {
+ "name": "recut_scene",
+ "round": 4,
+ "atMs": 23897.604500000016,
+ "execMs": 0.04129099997226149
+ },
+ {
+ "name": "get_state",
+ "round": 5,
+ "atMs": 26517.642374999938,
+ "execMs": 0.01624999998603016
+ },
+ {
+ "name": "wait_for_renders",
+ "round": 6,
+ "atMs": 28619.41837500001,
+ "execMs": 0.01049999997485429
+ },
+ {
+ "name": "get_state",
+ "round": 7,
+ "atMs": 30507.874583999976,
+ "execMs": 0.023333000019192696
+ }
+ ],
"timings": {
- "gateMs": 1871.1832500000019,
- "gateSkipped": false,
- "ttftMs": 6861.035999999964,
- "modelMs": 5578.064624999999,
- "toolMs": 0,
- "totalMs": 7449.247875000001,
+ "gateMs": 0,
+ "gateSkipped": true,
+ "ttftMs": 34241.398874999955,
+ "modelMs": 34674.622624000185,
+ "toolMs": 0.29379099991638213,
+ "totalMs": 34674.9164150001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5578.064624999999,
- "firstDeltaMs": 4989.662124999915,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "work",
- "notes": [],
- "trace": [],
- "timings": {
- "gateMs": 1290.5897089999635,
- "gateSkipped": false,
- "ttftMs": 6005.163917000056,
- "modelMs": 5316.767041999963,
- "toolMs": 0,
- "totalMs": 6607.356750999927,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 5316.767041999963,
- "firstDeltaMs": 4714.46770899999,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "work",
- "notes": [],
- "trace": [],
- "timings": {
- "gateMs": 1777.9000829999568,
- "gateSkipped": false,
- "ttftMs": 8540.008291000035,
- "modelMs": 7162.912667000084,
- "toolMs": 0,
- "totalMs": 8940.812750000041,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 7162.912667000084,
- "firstDeltaMs": 6761.866792000015,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "clip-mentions-redo-their-shots",
- "bucket": "multi-tool",
- "passRate": 0,
- "budgetBreaches": [],
- "latency": null,
- "runs": [
- {
- "pass": false,
- "intent": "work",
- "notes": [
- "mutating tools called: subtitles_generate, wait_for_renders"
- ],
- "trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 4308.13987500011,
- "execMs": 0.1568330000154674
- },
- {
- "name": "watch_video",
- "round": 1,
- "atMs": 7424.575292000081,
- "execMs": 0.03216600010637194
- },
- {
- "name": "watch_video",
- "round": 1,
- "atMs": 7424.635667000082,
- "execMs": 0.0009169999975711107
- },
- {
- "name": "subtitles_generate",
- "round": 2,
- "atMs": 11586.422084000078,
- "execMs": 0.01733299996703863
- },
- {
- "name": "get_state",
- "round": 3,
- "atMs": 14119.13408400002,
- "execMs": 0.031749999965541065
- },
- {
- "name": "regenerate_shot",
- "round": 4,
- "atMs": 23700.818750000093,
- "execMs": 0.14254200004506856
- },
- {
- "name": "regenerate_shot",
- "round": 4,
- "atMs": 23700.983125000028,
- "execMs": 0.014124999986961484
- },
- {
- "name": "wait_for_renders",
- "round": 5,
- "atMs": 25995.3921670001,
- "execMs": 0.03879100002814084
- }
- ],
- "timings": {
- "gateMs": 0,
- "gateSkipped": true,
- "ttftMs": 29204.339917000034,
- "modelMs": 29555.366833999986,
- "toolMs": 0.43445700011216104,
- "totalMs": 29555.801291000098,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 4307.633334000013,
+ "roundMs": 6765.621874999953,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.1568330000154674
+ "execMs": 0.1645420000422746
}
]
},
{
- "roundMs": 3116.160499999998,
+ "roundMs": 5589.860458999989,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.03216600010637194
+ "execMs": 0.008749999920837581
},
{
"name": "watch_video",
- "execMs": 0.0009169999975711107
+ "execMs": 0.000750000006519258
}
]
},
{
- "roundMs": 4161.72970799997,
+ "roundMs": 2680.9366660000524,
"firstDeltaMs": null,
"calls": [
{
- "name": "subtitles_generate",
- "execMs": 0.01733299996703863
+ "name": "list_skills",
+ "execMs": 0.022499999962747097
}
]
},
{
- "roundMs": 2532.6328340000473,
+ "roundMs": 1637.3984160000691,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.031749999965541065
+ "name": "read_skill",
+ "execMs": 0.0058750000316649675
}
]
},
{
- "roundMs": 9581.584082999965,
+ "roundMs": 7222.398207999999,
"firstDeltaMs": null,
"calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.14254200004506856
- },
+ "name": "recut_scene",
+ "execMs": 0.04129099997226149
+ }
+ ]
+ },
+ {
+ "roundMs": 2619.9519579999615,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.014124999986961484
+ "name": "get_state",
+ "execMs": 0.01624999998603016
}
]
},
{
- "roundMs": 2294.1887080000015,
+ "roundMs": 2101.6692500000354,
"firstDeltaMs": null,
"calls": [
{
"name": "wait_for_renders",
- "execMs": 0.03879100002814084
+ "execMs": 0.01049999997485429
+ }
+ ]
+ },
+ {
+ "roundMs": 1888.2911670000758,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.023333000019192696
}
]
},
{
- "roundMs": 3561.437666999991,
- "firstDeltaMs": 3208.7707919999957,
+ "roundMs": 4168.49462500005,
+ "firstDeltaMs": 3733.4538340000436,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: wait_for_renders"
- ],
+ "notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 4245.001416999963,
- "execMs": 0.09966700000222772
- },
- {
- "name": "watch_video",
- "round": 1,
- "atMs": 8061.111832999974,
- "execMs": 0.03245800000149757
+ "atMs": 4522.133374999976,
+ "execMs": 0.11108399997465312
},
{
- "name": "watch_video",
+ "name": "recut_scene",
"round": 1,
- "atMs": 8061.172750000027,
- "execMs": 0.0020410000579431653
- },
- {
- "name": "regenerate_shot",
- "round": 2,
- "atMs": 14519.447082999977,
- "execMs": 0.04958300001453608
+ "atMs": 14801.251082999981,
+ "execMs": 0.0272910000057891
},
{
- "name": "regenerate_shot",
+ "name": "wait_for_renders",
"round": 2,
- "atMs": 14519.509667000035,
- "execMs": 0.016583000076934695
+ "atMs": 18928.839917000034,
+ "execMs": 0.020708000054582953
},
{
- "name": "wait_for_renders",
+ "name": "get_state",
"round": 3,
- "atMs": 17736.374500000034,
- "execMs": 0.036041000043042004
+ "atMs": 21719.009375000023,
+ "execMs": 0.015125000034458935
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 20059.618875000044,
- "modelMs": 20401.887292000116,
- "toolMs": 0.23637300019618124,
- "totalMs": 20402.123665000312,
+ "ttftMs": 23625.309750000015,
+ "modelMs": 23887.040666999994,
+ "toolMs": 0.17420800006948411,
+ "totalMs": 23887.214875000063,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4244.497500000056,
+ "roundMs": 4521.701333000092,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.09966700000222772
+ "execMs": 0.11108399997465312
}
]
},
{
- "roundMs": 3815.8162079999456,
+ "roundMs": 10278.818874999997,
"firstDeltaMs": null,
"calls": [
{
- "name": "watch_video",
- "execMs": 0.03245800000149757
- },
- {
- "name": "watch_video",
- "execMs": 0.0020410000579431653
+ "name": "recut_scene",
+ "execMs": 0.0272910000057891
}
]
},
{
- "roundMs": 6458.196958000073,
+ "roundMs": 4127.413291999954,
"firstDeltaMs": null,
"calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.04958300001453608
- },
- {
- "name": "regenerate_shot",
- "execMs": 0.016583000076934695
+ "name": "wait_for_renders",
+ "execMs": 0.020708000054582953
}
]
},
{
- "roundMs": 3216.747583999997,
+ "roundMs": 2790.1045419999864,
"firstDeltaMs": null,
"calls": [
{
- "name": "wait_for_renders",
- "execMs": 0.036041000043042004
+ "name": "get_state",
+ "execMs": 0.015125000034458935
}
]
},
{
- "roundMs": 2666.629042000044,
- "firstDeltaMs": 2323.097417000099,
+ "roundMs": 2169.0026249999646,
+ "firstDeltaMs": 1906.260416999925,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: wait_for_renders"
- ],
+ "notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 3261.635666999966,
- "execMs": 0.012332999962382019
+ "atMs": 4242.994624999934,
+ "execMs": 0.06874999997671694
},
{
"name": "watch_video",
"round": 1,
- "atMs": 10368.450416999985,
- "execMs": 0.0058750000316649675
+ "atMs": 10180.30224999995,
+ "execMs": 0.010207999963313341
},
{
"name": "watch_video",
"round": 1,
- "atMs": 10368.459458999918,
- "execMs": 0.00041699991561472416
+ "atMs": 10180.321166999987,
+ "execMs": 0.0006249999860301614
},
{
- "name": "list_skills",
+ "name": "recut_scene",
"round": 2,
- "atMs": 18040.699000000022,
- "execMs": 0.02604200004134327
- },
- {
- "name": "read_skill",
- "round": 3,
- "atMs": 22143.29383400001,
- "execMs": 0.032291000010445714
- },
- {
- "name": "get_state",
- "round": 4,
- "atMs": 26513.360916999984,
- "execMs": 0.009500000043772161
- },
- {
- "name": "regenerate_shot",
- "round": 5,
- "atMs": 38419.23016699997,
- "execMs": 0.015292000025510788
- },
- {
- "name": "regenerate_shot",
- "round": 5,
- "atMs": 38419.25333400001,
- "execMs": 0.0026669999351724982
+ "atMs": 15810.0554999999,
+ "execMs": 0.009457999956794083
},
{
"name": "wait_for_renders",
- "round": 6,
- "atMs": 40950.077541999985,
- "execMs": 0.026084000011906028
+ "round": 3,
+ "atMs": 17452.86049999995,
+ "execMs": 0.018207999994046986
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 44194.23091699998,
- "modelMs": 44416.92487300013,
- "toolMs": 0.13050099997781217,
- "totalMs": 44417.055374000105,
+ "ttftMs": 19366.294083999936,
+ "modelMs": 19751.166499000043,
+ "toolMs": 0.10724899987690151,
+ "totalMs": 19751.27374799992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3261.1300000000047,
+ "roundMs": 4242.833291999996,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.012332999962382019
+ "execMs": 0.06874999997671694
}
]
},
{
- "roundMs": 7106.731499999994,
+ "roundMs": 5937.16183300002,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.0058750000316649675
+ "execMs": 0.010207999963313341
},
{
"name": "watch_video",
- "execMs": 0.00041699991561472416
- }
- ]
- },
- {
- "roundMs": 7672.144582999987,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.02604200004134327
- }
- ]
- },
- {
- "roundMs": 4102.482041000039,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.032291000010445714
- }
- ]
- },
- {
- "roundMs": 4369.966542000067,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.009500000043772161
+ "execMs": 0.0006249999860301614
}
]
},
{
- "roundMs": 11905.814458000008,
+ "roundMs": 5629.7169170000125,
"firstDeltaMs": null,
"calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.015292000025510788
- },
- {
- "name": "regenerate_shot",
- "execMs": 0.0026669999351724982
+ "name": "recut_scene",
+ "execMs": 0.009457999956794083
}
]
},
{
- "roundMs": 2530.695041000028,
+ "roundMs": 1642.7544159999816,
"firstDeltaMs": null,
"calls": [
{
"name": "wait_for_renders",
- "execMs": 0.026084000011906028
+ "execMs": 0.018207999994046986
}
]
},
{
- "roundMs": 3467.9607079999987,
- "firstDeltaMs": 3244.0450829999754,
+ "roundMs": 2298.700041000033,
+ "firstDeltaMs": 1913.331082999939,
"calls": []
}
]
@@ -5905,68 +5628,68 @@
"cases": 7,
"passRate": 1,
"gateMs": {
- "p50": 1075,
- "p95": 1257,
- "mean": 819,
+ "p50": 1183,
+ "p95": 1827,
+ "mean": 1040,
"n": 7
},
"ttftMs": {
- "p50": 2148,
- "p95": 2710,
- "mean": 2205,
+ "p50": 2315,
+ "p95": 2933,
+ "mean": 2317,
"n": 7
},
"totalMs": {
- "p50": 2742,
- "p95": 3479,
- "mean": 2749,
+ "p50": 2757,
+ "p95": 3563,
+ "mean": 2896,
"n": 7
},
"meanRounds": 1
},
"multi-tool": {
"cases": 4,
- "passRate": 0.75,
+ "passRate": 0.83,
"gateMs": {
"p50": 0,
- "p95": 1941,
- "mean": 787,
+ "p95": 1116,
+ "mean": 535,
"n": 4
},
"ttftMs": {
- "p50": 6607,
- "p95": 139632,
- "mean": 39503,
+ "p50": 4232,
+ "p95": 8102,
+ "mean": 5621,
"n": 4
},
"totalMs": {
- "p50": 6939,
- "p95": 19356,
- "mean": 9841,
+ "p50": 4862,
+ "p95": 8490,
+ "mean": 6151,
"n": 4
},
- "meanRounds": 4.7
+ "meanRounds": 3.6
},
"single-tool": {
"cases": 15,
- "passRate": 0.78,
+ "passRate": 0.91,
"gateMs": {
- "p50": 1228,
- "p95": 1717,
- "mean": 1153,
- "n": 12
+ "p50": 1074,
+ "p95": 1842,
+ "mean": 1094,
+ "n": 14
},
"ttftMs": {
- "p50": 4196,
- "p95": 4928,
- "mean": 4146,
- "n": 12
+ "p50": 3970,
+ "p95": 4557,
+ "mean": 3999,
+ "n": 14
},
"totalMs": {
- "p50": 4489,
- "p95": 5537,
- "mean": 4555,
- "n": 12
+ "p50": 4330,
+ "p95": 4925,
+ "mean": 4339,
+ "n": 14
},
"meanRounds": 2
}
@@ -5985,21 +5708,21 @@
"n": 3
},
"ttftMs": {
- "p50": 2710,
- "p95": 5842,
- "mean": 3527,
+ "p50": 1876,
+ "p95": 5449,
+ "mean": 2926,
"n": 3
},
"totalMs": {
- "p50": 2903,
- "p95": 6531,
- "mean": 4056,
+ "p50": 2087,
+ "p95": 5600,
+ "mean": 3142,
"n": 3
},
"modelMs": {
- "p50": 2903,
- "p95": 6531,
- "mean": 4056,
+ "p50": 2087,
+ "p95": 5600,
+ "mean": 3142,
"n": 3
},
"toolMs": {
@@ -6022,16 +5745,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5842.039291,
- "modelMs": 6531.047042,
+ "ttftMs": 5448.706083,
+ "modelMs": 5600.065625,
"toolMs": 0,
- "totalMs": 6531.047042,
+ "totalMs": 5600.065625,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6531.047042,
- "firstDeltaMs": 5841.42025,
+ "roundMs": 5600.065625,
+ "firstDeltaMs": 5448.1633329999995,
"calls": []
}
]
@@ -6045,16 +5768,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 2709.647750000001,
- "modelMs": 2903.475792,
+ "ttftMs": 1452.0524169999999,
+ "modelMs": 1737.929000000001,
"toolMs": 0,
- "totalMs": 2903.475792,
+ "totalMs": 1737.929000000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2903.475792,
- "firstDeltaMs": 2709.417292000001,
+ "roundMs": 1737.929000000001,
+ "firstDeltaMs": 1451.8628749999998,
"calls": []
}
]
@@ -6068,16 +5791,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 2028.369208,
- "modelMs": 2734.4187079999992,
+ "ttftMs": 1876.3337500000016,
+ "modelMs": 2087.12125,
"toolMs": 0,
- "totalMs": 2734.4187079999992,
+ "totalMs": 2087.12125,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2734.4187079999992,
- "firstDeltaMs": 2028.1500829999986,
+ "roundMs": 2087.12125,
+ "firstDeltaMs": 1876.1494590000002,
"calls": []
}
]
@@ -6092,27 +5815,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1075,
- "p95": 1254,
- "mean": 1093,
+ "p50": 1619,
+ "p95": 1735,
+ "mean": 1411,
"n": 3
},
"ttftMs": {
- "p50": 2116,
- "p95": 2319,
- "mean": 2145,
+ "p50": 2927,
+ "p95": 3184,
+ "mean": 2692,
"n": 3
},
"totalMs": {
- "p50": 2343,
- "p95": 2581,
- "mean": 2406,
+ "p50": 3290,
+ "p95": 3805,
+ "mean": 3092,
"n": 3
},
"modelMs": {
- "p50": 1344,
- "p95": 1506,
- "mean": 1313,
+ "p50": 1556,
+ "p95": 2187,
+ "mean": 1681,
"n": 3
},
"toolMs": {
@@ -6133,18 +5856,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1253.9504589999997,
+ "gateMs": 879.8068750000002,
"gateSkipped": false,
- "ttftMs": 2115.8470419999994,
- "modelMs": 1088.667375000001,
+ "ttftMs": 1964.368042,
+ "modelMs": 1299.33475,
"toolMs": 0,
- "totalMs": 2342.6178340000006,
+ "totalMs": 2179.141625,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1088.667375000001,
- "firstDeltaMs": 860.7248330000002,
+ "roundMs": 1299.33475,
+ "firstDeltaMs": 1084.0000830000008,
"calls": []
}
]
@@ -6156,18 +5879,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 950.6264580000006,
+ "gateMs": 1734.776667,
"gateSkipped": false,
- "ttftMs": 2001.3307079999995,
- "modelMs": 1344.197916000001,
+ "ttftMs": 2927.244208,
+ "modelMs": 1555.5578330000008,
"toolMs": 0,
- "totalMs": 2294.8243740000016,
+ "totalMs": 3290.334500000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1344.197916000001,
- "firstDeltaMs": 1050.4176660000012,
+ "roundMs": 1555.5578330000008,
+ "firstDeltaMs": 1192.2530000000006,
"calls": []
}
]
@@ -6179,18 +5902,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1075.234625000001,
+ "gateMs": 1618.8550830000004,
"gateSkipped": false,
- "ttftMs": 2318.84375,
- "modelMs": 1505.7200000000012,
+ "ttftMs": 3183.765874999999,
+ "modelMs": 2186.6121669999993,
"toolMs": 0,
- "totalMs": 2580.954625000002,
+ "totalMs": 3805.4672499999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1505.7200000000012,
- "firstDeltaMs": 1243.4221670000006,
+ "roundMs": 2186.6121669999993,
+ "firstDeltaMs": 1564.718667000001,
"calls": []
}
]
@@ -6205,27 +5928,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1257,
- "p95": 1364,
- "mean": 1237,
+ "p50": 1029,
+ "p95": 1212,
+ "mean": 1075,
"n": 3
},
"ttftMs": {
- "p50": 2068,
- "p95": 2689,
- "mean": 2221,
+ "p50": 2233,
+ "p95": 2392,
+ "mean": 2140,
"n": 3
},
"totalMs": {
- "p50": 2477,
- "p95": 3481,
- "mean": 2729,
+ "p50": 2692,
+ "p95": 3216,
+ "mean": 2603,
"n": 3
},
"modelMs": {
- "p50": 1220,
- "p95": 2117,
- "mean": 1491,
+ "p50": 1480,
+ "p95": 2186,
+ "mean": 1527,
"n": 3
},
"toolMs": {
@@ -6246,18 +5969,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1256.9237079999984,
+ "gateMs": 1212.4599579999995,
"gateSkipped": false,
- "ttftMs": 2068.1307080000006,
- "modelMs": 1220.0076660000013,
+ "ttftMs": 2233.3255839999983,
+ "modelMs": 1479.8240409999999,
"toolMs": 0,
- "totalMs": 2476.931374,
+ "totalMs": 2692.2839989999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1220.0076660000013,
- "firstDeltaMs": 810.5717499999992,
+ "roundMs": 1479.8240409999999,
+ "firstDeltaMs": 1020.7070830000011,
"calls": []
}
]
@@ -6269,18 +5992,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1364.080249999999,
+ "gateMs": 1029.4677090000005,
"gateSkipped": false,
- "ttftMs": 2689.105834000002,
- "modelMs": 2116.9812500000007,
+ "ttftMs": 2391.5550829999993,
+ "modelMs": 2186.1085840000014,
"toolMs": 0,
- "totalMs": 3481.0615,
+ "totalMs": 3215.576293000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2116.9812500000007,
- "firstDeltaMs": 1324.7739159999983,
+ "roundMs": 2186.1085840000014,
+ "firstDeltaMs": 1361.838667,
"calls": []
}
]
@@ -6292,18 +6015,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1090.7956249999988,
+ "gateMs": 984.2342500000013,
"gateSkipped": false,
- "ttftMs": 1906.214082999999,
- "modelMs": 1137.2621670000008,
+ "ttftMs": 1796.248208000001,
+ "modelMs": 915.4589159999996,
"toolMs": 0,
- "totalMs": 2228.0577919999996,
+ "totalMs": 1899.693166000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1137.2621670000008,
- "firstDeltaMs": 815.2143340000002,
+ "roundMs": 915.4589159999996,
+ "firstDeltaMs": 811.9484579999989,
"calls": []
}
]
@@ -6318,33 +6041,33 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1208,
- "p95": 1339,
- "mean": 1135,
+ "p50": 1116,
+ "p95": 1638,
+ "mean": 1288,
"n": 3
},
"ttftMs": {
- "p50": 4436,
- "p95": 5107,
- "mean": 4620,
+ "p50": 4232,
+ "p95": 4527,
+ "mean": 4210,
"n": 3
},
"totalMs": {
- "p50": 4964,
- "p95": 5354,
- "mean": 4982,
+ "p50": 4862,
+ "p95": 4986,
+ "mean": 4643,
"n": 3
},
"modelMs": {
- "p50": 3769,
- "p95": 4014,
- "mean": 3846,
+ "p50": 3224,
+ "p95": 3876,
+ "mean": 3355,
"n": 3
},
"toolMs": {
"p50": 0,
"p95": 1,
- "mean": 1,
+ "mean": 0,
"n": 3
},
"rounds": {
@@ -6361,73 +6084,43 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 3044.2589579999985,
- "execMs": 1.0701249999983702
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 3045.3700409999983,
- "execMs": 0.012542000000394182
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 3045.3921249999985,
- "execMs": 0.05216700000164565
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 3045.4501249999994,
- "execMs": 0.013416999998298706
+ "atMs": 3262.237416,
+ "execMs": 0.7116669999995793
},
{
"name": "update_cue",
"round": 0,
- "atMs": 3045.467915999998,
- "execMs": 0.011249999999563443
+ "atMs": 3262.9713749999974,
+ "execMs": 0.009958000002370682
}
],
"timings": {
- "gateMs": 1208.309333000001,
+ "gateMs": 1637.534584000001,
"gateSkipped": false,
- "ttftMs": 4436.498082999999,
- "modelMs": 3754.755291999998,
- "toolMs": 1.1595009999982722,
- "totalMs": 4964.224125999997,
+ "ttftMs": 4526.655790999997,
+ "modelMs": 3224.2191659999953,
+ "toolMs": 0.72162500000195,
+ "totalMs": 4862.475374999998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1834.4467089999998,
+ "roundMs": 1623.974124999997,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 1.0701249999983702
- },
- {
- "name": "update_cue",
- "execMs": 0.012542000000394182
- },
- {
- "name": "update_cue",
- "execMs": 0.05216700000164565
+ "execMs": 0.7116669999995793
},
{
"name": "update_cue",
- "execMs": 0.013416999998298706
- },
- {
- "name": "update_cue",
- "execMs": 0.011249999999563443
+ "execMs": 0.009958000002370682
}
]
},
{
- "roundMs": 1920.3085829999982,
- "firstDeltaMs": 1390.992707999998,
+ "roundMs": 1600.2450409999983,
+ "firstDeltaMs": 1263.654040999998,
"calls": []
}
]
@@ -6441,63 +6134,53 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 3174.100375000002,
- "execMs": 0.033250000000407454
+ "atMs": 2896.0690840000025,
+ "execMs": 0.007208000002719928
},
{
"name": "update_cue",
"round": 0,
- "atMs": 3174.147417,
- "execMs": 0.014208000000508036
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 3174.1670830000003,
- "execMs": 0.337666999999783
+ "atMs": 2896.0786670000016,
+ "execMs": 0.0034999999988940544
},
{
- "name": "update_cue",
+ "name": "merge_cue",
"round": 0,
- "atMs": 3174.512042000002,
- "execMs": 0.013915999996243045
+ "atMs": 2896.0831670000043,
+ "execMs": 0.10974999999598367
}
],
"timings": {
- "gateMs": 1338.9557080000013,
+ "gateMs": 1110.1106670000008,
"gateSkipped": false,
- "ttftMs": 5107.0040420000005,
- "modelMs": 4014.250957999997,
- "toolMs": 0.39904099999694154,
- "totalMs": 5353.605706999995,
+ "ttftMs": 4232.3761250000025,
+ "modelMs": 3875.842583000005,
+ "toolMs": 0.12045799999759765,
+ "totalMs": 4986.0737080000035,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1834.788708,
+ "roundMs": 1785.6399999999994,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.033250000000407454
+ "execMs": 0.007208000002719928
},
{
"name": "update_cue",
- "execMs": 0.014208000000508036
- },
- {
- "name": "delete_cue",
- "execMs": 0.337666999999783
+ "execMs": 0.0034999999988940544
},
{
- "name": "update_cue",
- "execMs": 0.013915999996243045
+ "name": "merge_cue",
+ "execMs": 0.10974999999598367
}
]
},
{
- "roundMs": 2179.462249999997,
- "firstDeltaMs": 1932.435666999998,
+ "roundMs": 2090.2025830000057,
+ "firstDeltaMs": 1336.173625000003,
"calls": []
}
]
@@ -6511,63 +6194,63 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 3102.6487500000003,
- "execMs": 0.024457999999867752
+ "atMs": 2731.3043749999997,
+ "execMs": 0.08862499999668216
},
{
"name": "update_cue",
"round": 0,
- "atMs": 3102.688083000001,
- "execMs": 0.01716599999781465
+ "atMs": 2731.4164999999994,
+ "execMs": 0.01591700000426499
},
{
"name": "delete_cue",
"round": 0,
- "atMs": 3102.7105420000007,
- "execMs": 0.04208400000061374
+ "atMs": 2731.4381670000002,
+ "execMs": 0.24258299999928568
},
{
"name": "update_cue",
"round": 0,
- "atMs": 3102.7571670000034,
- "execMs": 0.010750000001280569
+ "atMs": 2731.6870419999977,
+ "execMs": 0.013790999997581821
}
],
"timings": {
- "gateMs": 858.1651670000065,
+ "gateMs": 1115.6544159999976,
"gateSkipped": false,
- "ttftMs": 4316.4937500000015,
- "modelMs": 3769.015124999998,
- "toolMs": 0.09445799999957671,
- "totalMs": 4627.274750000004,
+ "ttftMs": 3872.114916999999,
+ "modelMs": 2965.2341249999954,
+ "toolMs": 0.36091599999781465,
+ "totalMs": 4081.249456999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2244.1942500000005,
+ "roundMs": 1615.355958,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.024457999999867752
+ "execMs": 0.08862499999668216
},
{
"name": "update_cue",
- "execMs": 0.01716599999781465
+ "execMs": 0.01591700000426499
},
{
"name": "delete_cue",
- "execMs": 0.04208400000061374
+ "execMs": 0.24258299999928568
},
{
"name": "update_cue",
- "execMs": 0.010750000001280569
+ "execMs": 0.013790999997581821
}
]
},
{
- "roundMs": 1524.8208749999976,
- "firstDeltaMs": 1213.6810830000031,
+ "roundMs": 1349.8781669999953,
+ "firstDeltaMs": 1140.3388750000013,
"calls": []
}
]
@@ -6582,27 +6265,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1038,
- "p95": 1112,
- "mean": 1021,
+ "p50": 1183,
+ "p95": 1856,
+ "mean": 1369,
"n": 3
},
"ttftMs": {
- "p50": 2148,
- "p95": 2311,
- "mean": 2148,
+ "p50": 2315,
+ "p95": 3022,
+ "mean": 2502,
"n": 3
},
"totalMs": {
- "p50": 2334,
- "p95": 2550,
- "mean": 2384,
+ "p50": 2757,
+ "p95": 3277,
+ "mean": 2837,
"n": 3
},
"modelMs": {
- "p50": 1355,
- "p95": 1438,
- "mean": 1363,
+ "p50": 1421,
+ "p95": 1575,
+ "mean": 1468,
"n": 3
},
"toolMs": {
@@ -6623,18 +6306,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1111.783792000002,
+ "gateMs": 1855.9221249999973,
"gateSkipped": false,
- "ttftMs": 2310.7707089999967,
- "modelMs": 1437.8076250000013,
+ "ttftMs": 3022.3792080000057,
+ "modelMs": 1421.0354169999991,
"toolMs": 0,
- "totalMs": 2549.5914170000033,
+ "totalMs": 3276.9575419999965,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1437.8076250000013,
- "firstDeltaMs": 1198.5101250000007,
+ "roundMs": 1421.0354169999991,
+ "firstDeltaMs": 1166.0170419999995,
"calls": []
}
]
@@ -6646,18 +6329,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 912.5085830000025,
+ "gateMs": 1182.6340000000055,
"gateSkipped": false,
- "ttftMs": 1984.6794999999984,
- "modelMs": 1355.3630000000048,
+ "ttftMs": 2314.682458999996,
+ "modelMs": 1574.8548750000045,
"toolMs": 0,
- "totalMs": 2267.8715830000074,
+ "totalMs": 2757.48887500001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1355.3630000000048,
- "firstDeltaMs": 1071.842875000002,
+ "roundMs": 1574.8548750000045,
+ "firstDeltaMs": 1131.794833,
"calls": []
}
]
@@ -6669,18 +6352,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1038.2362089999951,
+ "gateMs": 1067.8795830000017,
"gateSkipped": false,
- "ttftMs": 2147.8408750000017,
- "modelMs": 1295.606832999998,
+ "ttftMs": 2167.474792000001,
+ "modelMs": 1409.1963749999995,
"toolMs": 0,
- "totalMs": 2333.843041999993,
+ "totalMs": 2477.0759580000013,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1295.606832999998,
- "firstDeltaMs": 1109.3114580000038,
+ "roundMs": 1409.1963749999995,
+ "firstDeltaMs": 1099.4859159999978,
"calls": []
}
]
@@ -6691,42 +6374,42 @@
{
"name": "photos-request-places-clips",
"bucket": "multi-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 0.3333333333333333,
"budgetBreaches": [],
"latency": {
"gateMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"ttftMs": {
- "p50": 6607,
- "p95": 10960,
- "mean": 8784,
- "n": 2
+ "p50": 2793,
+ "p95": 2793,
+ "mean": 2793,
+ "n": 1
},
"totalMs": {
- "p50": 6939,
- "p95": 11233,
- "mean": 9086,
- "n": 2
+ "p50": 3120,
+ "p95": 3120,
+ "mean": 3120,
+ "n": 1
},
"modelMs": {
- "p50": 6939,
- "p95": 11233,
- "mean": 9086,
- "n": 2
+ "p50": 3120,
+ "p95": 3120,
+ "mean": 3120,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 5.5,
- "max": 6
+ "mean": 2,
+ "max": 2
}
},
"runs": [
@@ -6734,101 +6417,97 @@
"pass": false,
"intent": "work",
"notes": [
- "mutating tools called: place_clip, place_clip"
+ "mutating tools called: trim_clip, place_clip"
],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 2120.048999999999,
- "execMs": 0.004042000000481494
+ "atMs": 1731.418875000003,
+ "execMs": 0.0020000000004074536
},
{
"name": "add_clip",
"round": 0,
- "atMs": 2120.0820840000015,
- "execMs": 0.0066250000018044375
+ "atMs": 1731.4348329999993,
+ "execMs": 0.0039590000014868565
},
{
"name": "add_clip",
"round": 1,
- "atMs": 3570.038416999996,
- "execMs": 0.005000000004656613
+ "atMs": 3041.2998750000042,
+ "execMs": 0.0029999999969732016
},
{
- "name": "add_clip",
- "round": 1,
- "atMs": 3570.054458999999,
- "execMs": 0.00045900000259280205
- },
- {
- "name": "place_clip",
+ "name": "trim_clip",
"round": 2,
- "atMs": 4882.120499999997,
- "execMs": 0.03487499999755528
+ "atMs": 5092.411040999999,
+ "execMs": 0.01579199999832781
},
{
"name": "place_clip",
- "round": 2,
- "atMs": 4882.166874999995,
- "execMs": 0.02149999999528518
+ "round": 3,
+ "atMs": 6426.320749999999,
+ "execMs": 0.03229200000350829
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6149.027167,
- "modelMs": 6470.583499,
- "toolMs": 0.0725010000023758,
- "totalMs": 6470.656000000003,
+ "ttftMs": 7504.480208000001,
+ "modelMs": 7993.3407929999885,
+ "toolMs": 0.057043000000703614,
+ "totalMs": 7993.397835999989,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2119.0266249999986,
+ "roundMs": 1730.7749169999952,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.004042000000481494
+ "execMs": 0.0020000000004074536
},
{
"name": "add_clip",
- "execMs": 0.0066250000018044375
+ "execMs": 0.0039590000014868565
}
]
},
{
- "roundMs": 1449.7309999999998,
+ "roundMs": 1309.771541999995,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.005000000004656613
- },
- {
- "name": "add_clip",
- "execMs": 0.00045900000259280205
+ "execMs": 0.0029999999969732016
}
]
},
{
- "roundMs": 1311.9222080000036,
+ "roundMs": 2051.0467500000013,
"firstDeltaMs": null,
"calls": [
{
- "name": "place_clip",
- "execMs": 0.03487499999755528
- },
+ "name": "trim_clip",
+ "execMs": 0.01579199999832781
+ }
+ ]
+ },
+ {
+ "roundMs": 1333.8022920000003,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "place_clip",
- "execMs": 0.02149999999528518
+ "execMs": 0.03229200000350829
}
]
},
{
- "roundMs": 1589.9036659999983,
- "firstDeltaMs": 1266.8021249999947,
+ "roundMs": 1567.9452919999967,
+ "firstDeltaMs": 1078.0244579999999,
"calls": []
}
]
@@ -6842,225 +6521,131 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 1412.4798329999976,
- "execMs": 0.004292000005079899
+ "atMs": 1596.4076249999998,
+ "execMs": 0.0018330000020796433
},
{
"name": "add_clip",
"round": 0,
- "atMs": 1412.5265,
- "execMs": 0.000375000003259629
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 2770.516208000001,
- "execMs": 0.004041000000142958
- },
- {
- "name": "add_clip",
- "round": 2,
- "atMs": 4255.6182499999995,
- "execMs": 0.0005830000009154901
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 8105.343000000001,
- "execMs": 0.0010840000031748787
- },
- {
- "name": "add_clip",
- "round": 4,
- "atMs": 9850.606708,
- "execMs": 0.003416999999899417
- },
- {
- "name": "add_clip",
- "round": 4,
- "atMs": 9850.620665999995,
- "execMs": 0.0001669999910518527
+ "atMs": 1596.4183340000018,
+ "execMs": 0.003125000002910383
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 10960.154791,
- "modelMs": 11232.818334000018,
- "toolMs": 0.013959000003524125,
- "totalMs": 11232.832293000021,
+ "ttftMs": 2793.2215419999993,
+ "modelMs": 3119.672292000003,
+ "toolMs": 0.004958000004990026,
+ "totalMs": 3119.677250000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1412.3732090000049,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.004292000005079899
- },
- {
- "name": "add_clip",
- "execMs": 0.000375000003259629
- }
- ]
- },
- {
- "roundMs": 1357.9104169999991,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.004041000000142958
- }
- ]
- },
- {
- "roundMs": 1485.0319580000069,
+ "roundMs": 1596.0577090000006,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.0005830000009154901
- }
- ]
- },
- {
- "roundMs": 3849.7105410000004,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0010840000031748787
- }
- ]
- },
- {
- "roundMs": 1745.2216670000053,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.003416999999899417
+ "execMs": 0.0018330000020796433
},
{
"name": "add_clip",
- "execMs": 0.0001669999910518527
+ "execMs": 0.003125000002910383
}
]
},
{
- "roundMs": 1382.5705420000013,
- "firstDeltaMs": 1109.5066669999942,
+ "roundMs": 1523.6145830000023,
+ "firstDeltaMs": 1196.7675000000017,
"calls": []
}
]
}
},
{
- "pass": true,
+ "pass": false,
"intent": "work",
- "notes": [],
+ "notes": [
+ "mutating tools called: trim_clip, place_clip, place_clip"
+ ],
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 1375.947417000003,
- "execMs": 0.004125000006752089
+ "atMs": 4729.623832999998,
+ "execMs": 0.0027919999993173406
},
{
"name": "add_clip",
"round": 0,
- "atMs": 1375.9621670000051,
- "execMs": 0.0003749999887077138
+ "atMs": 4729.637666999995,
+ "execMs": 0.00016599999798927456
},
{
- "name": "add_clip",
+ "name": "trim_clip",
"round": 1,
- "atMs": 2742.452667000005,
- "execMs": 0.008375000004889444
+ "atMs": 6766.426708000006,
+ "execMs": 0.00995800000964664
},
{
- "name": "add_clip",
+ "name": "place_clip",
"round": 1,
- "atMs": 2742.4716670000053,
- "execMs": 0.0004589999880408868
- },
- {
- "name": "add_clip",
- "round": 2,
- "atMs": 4163.748041999992,
- "execMs": 0.004207999998470768
+ "atMs": 6766.4393749999945,
+ "execMs": 0.008375000004889444
},
{
- "name": "add_clip",
- "round": 3,
- "atMs": 5525.709459000005,
- "execMs": 0.004249999998137355
+ "name": "place_clip",
+ "round": 1,
+ "atMs": 6766.448458000006,
+ "execMs": 0.00020799999765586108
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6607.117333999995,
- "modelMs": 6939.357167000009,
- "toolMs": 0.021791999984998256,
- "totalMs": 6939.378958999994,
+ "ttftMs": 8042.978041999995,
+ "modelMs": 8337.709833999987,
+ "toolMs": 0.02149900000949856,
+ "totalMs": 8337.731332999996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1375.7036669999943,
+ "roundMs": 4729.268541999998,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.004125000006752089
+ "execMs": 0.0027919999993173406
},
{
"name": "add_clip",
- "execMs": 0.0003749999887077138
+ "execMs": 0.00016599999798927456
}
]
},
{
- "roundMs": 1366.4125000000058,
+ "roundMs": 2036.7425829999993,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.008375000004889444
+ "name": "trim_clip",
+ "execMs": 0.00995800000964664
},
{
- "name": "add_clip",
- "execMs": 0.0004589999880408868
- }
- ]
- },
- {
- "roundMs": 1421.1983750000072,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.004207999998470768
- }
- ]
- },
- {
- "roundMs": 1361.8822500000097,
- "firstDeltaMs": null,
- "calls": [
+ "name": "place_clip",
+ "execMs": 0.008375000004889444
+ },
{
- "name": "add_clip",
- "execMs": 0.004249999998137355
+ "name": "place_clip",
+ "execMs": 0.00020799999765586108
}
]
},
{
- "roundMs": 1414.1603749999922,
- "firstDeltaMs": 1081.363124999989,
+ "roundMs": 1571.6987089999893,
+ "firstDeltaMs": 1276.5069169999915,
"calls": []
}
]
@@ -7081,21 +6666,21 @@
"n": 3
},
"ttftMs": {
- "p50": 1398,
- "p95": 1518,
- "mean": 1429,
+ "p50": 1326,
+ "p95": 1373,
+ "mean": 1278,
"n": 3
},
"totalMs": {
- "p50": 2742,
- "p95": 2760,
- "mean": 2745,
+ "p50": 2418,
+ "p95": 2524,
+ "mean": 2422,
"n": 3
},
"modelMs": {
- "p50": 2742,
- "p95": 2760,
- "mean": 2745,
+ "p50": 2418,
+ "p95": 2524,
+ "mean": 2422,
"n": 3
},
"toolMs": {
@@ -7118,16 +6703,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 1398.1315409999952,
- "modelMs": 2733.5647089999984,
+ "ttftMs": 1373.2701659999875,
+ "modelMs": 2325.4849160000012,
"toolMs": 0,
- "totalMs": 2733.5647089999984,
+ "totalMs": 2325.4849160000012,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2733.5647089999984,
- "firstDeltaMs": 1397.6775840000046,
+ "roundMs": 2325.4849160000012,
+ "firstDeltaMs": 1372.8845830000064,
"calls": []
}
]
@@ -7141,16 +6726,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 1517.507083000004,
- "modelMs": 2741.6817080000037,
+ "ttftMs": 1326.3287910000072,
+ "modelMs": 2523.5225419999915,
"toolMs": 0,
- "totalMs": 2741.6817080000037,
+ "totalMs": 2523.5225419999915,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2741.6817080000037,
- "firstDeltaMs": 1517.2455409999966,
+ "roundMs": 2523.5225419999915,
+ "firstDeltaMs": 1326.2543329999899,
"calls": []
}
]
@@ -7164,16 +6749,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 1371.4233749999985,
- "modelMs": 2760.3098750000063,
+ "ttftMs": 1133.4992090000014,
+ "modelMs": 2417.9269170000043,
"toolMs": 0,
- "totalMs": 2760.3098750000063,
+ "totalMs": 2417.9269170000043,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2760.3098750000063,
- "firstDeltaMs": 1371.3615839999984,
+ "roundMs": 2417.9269170000043,
+ "firstDeltaMs": 1133.222167,
"calls": []
}
]
@@ -7188,27 +6773,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1208,
- "p95": 1670,
- "mean": 1348,
+ "p50": 1827,
+ "p95": 1860,
+ "mean": 1579,
"n": 3
},
"ttftMs": {
- "p50": 2401,
- "p95": 2835,
- "mean": 2526,
+ "p50": 2933,
+ "p95": 3724,
+ "mean": 3091,
"n": 3
},
"totalMs": {
- "p50": 2965,
- "p95": 3343,
- "mean": 3054,
+ "p50": 3563,
+ "p95": 4309,
+ "mean": 3739,
"n": 3
},
"modelMs": {
- "p50": 1673,
- "p95": 1799,
- "mean": 1706,
+ "p50": 2296,
+ "p95": 2449,
+ "mean": 2160,
"n": 3
},
"toolMs": {
@@ -7229,18 +6814,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1165.798708000002,
+ "gateMs": 1860.0239160000056,
"gateSkipped": false,
- "ttftMs": 2341.818625,
- "modelMs": 1799.195583000008,
+ "ttftMs": 3724.196499999991,
+ "modelMs": 2449.475999999995,
"toolMs": 0,
- "totalMs": 2964.99429100001,
+ "totalMs": 4309.499916000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1799.195583000008,
- "firstDeltaMs": 1175.448292000001,
+ "roundMs": 2449.475999999995,
+ "firstDeltaMs": 1863.8114999999962,
"calls": []
}
]
@@ -7252,18 +6837,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1669.8032499999972,
+ "gateMs": 1827.289665999997,
"gateSkipped": false,
- "ttftMs": 2835.2124170000025,
- "modelMs": 1673.2642080000078,
+ "ttftMs": 2933.302541000012,
+ "modelMs": 1735.4502499999944,
"toolMs": 0,
- "totalMs": 3343.067458000005,
+ "totalMs": 3562.7399159999914,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1673.2642080000078,
- "firstDeltaMs": 1165.1783330000035,
+ "roundMs": 1735.4502499999944,
+ "firstDeltaMs": 1105.5659159999923,
"calls": []
}
]
@@ -7275,18 +6860,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1207.794458000004,
+ "gateMs": 1049.6073750000069,
"gateSkipped": false,
- "ttftMs": 2401.2466249999998,
- "modelMs": 1646.2302499999932,
+ "ttftMs": 2615.1658339999994,
+ "modelMs": 2295.594957999987,
"toolMs": 0,
- "totalMs": 2854.024707999997,
+ "totalMs": 3345.202332999994,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1646.2302499999932,
- "firstDeltaMs": 1193.1767089999921,
+ "roundMs": 2295.594957999987,
+ "firstDeltaMs": 1565.3199170000007,
"calls": []
}
]
@@ -7301,27 +6886,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1471,
- "p95": 1863,
- "mean": 1421,
+ "p50": 1842,
+ "p95": 1909,
+ "mean": 1688,
"n": 3
},
"ttftMs": {
- "p50": 4252,
- "p95": 4804,
- "mean": 4195,
+ "p50": 4557,
+ "p95": 4603,
+ "mean": 4458,
"n": 3
},
"totalMs": {
- "p50": 4535,
- "p95": 5233,
- "mean": 4559,
+ "p50": 4925,
+ "p95": 5039,
+ "mean": 4824,
"n": 3
},
"modelMs": {
- "p50": 3064,
- "p95": 3370,
- "mean": 3138,
+ "p50": 3130,
+ "p95": 3193,
+ "mean": 3136,
"n": 3
},
"toolMs": {
@@ -7344,33 +6929,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 3075.175833000001,
- "execMs": 0.013291999988723546
+ "atMs": 3537.237083,
+ "execMs": 0.004125000006752089
}
],
"timings": {
- "gateMs": 1471.278084000005,
+ "gateMs": 1908.5142920000071,
"gateSkipped": false,
- "ttftMs": 4251.876583000005,
- "modelMs": 3064.15912500002,
- "toolMs": 0.013291999988723546,
- "totalMs": 4535.450501000014,
+ "ttftMs": 4602.685583000013,
+ "modelMs": 3130.298833000008,
+ "toolMs": 0.004125000006752089,
+ "totalMs": 5038.817250000022,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1603.3026250000112,
+ "roundMs": 1628.3550420000101,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.013291999988723546
+ "execMs": 0.004125000006752089
}
]
},
{
- "roundMs": 1460.856500000009,
- "firstDeltaMs": 1176.6073750000069,
+ "roundMs": 1501.9437909999979,
+ "firstDeltaMs": 1065.4169580000016,
"calls": []
}
]
@@ -7384,33 +6969,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 2445.0891669999983,
- "execMs": 0.03787499999452848
+ "atMs": 3031.0929170000018,
+ "execMs": 0.029416999997920357
}
],
"timings": {
- "gateMs": 928.9127080000035,
+ "gateMs": 1313.5652499999997,
"gateSkipped": false,
- "ttftMs": 3530.9238329999935,
- "modelMs": 2979.2077500000014,
- "toolMs": 0.03787499999452848,
- "totalMs": 3908.1583329999994,
+ "ttftMs": 4213.957834000001,
+ "modelMs": 3192.9425829999964,
+ "toolMs": 0.029416999997920357,
+ "totalMs": 4506.537249999994,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1515.96875,
+ "roundMs": 1717.200291999994,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.03787499999452848
+ "execMs": 0.029416999997920357
}
]
},
{
- "roundMs": 1463.2390000000014,
- "firstDeltaMs": 1085.7169580000045,
+ "roundMs": 1475.7422910000023,
+ "firstDeltaMs": 1182.7697499999922,
"calls": []
}
]
@@ -7424,33 +7009,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 3714.9896670000016,
- "execMs": 0.008000000001629815
+ "atMs": 3133.316374999995,
+ "execMs": 0.010708000001613982
}
],
"timings": {
- "gateMs": 1862.833083999998,
+ "gateMs": 1841.9359170000098,
"gateSkipped": false,
- "ttftMs": 4803.690583999996,
- "modelMs": 3370.469750000004,
- "toolMs": 0.008000000001629815,
- "totalMs": 5233.3108340000035,
+ "ttftMs": 4556.7174159999995,
+ "modelMs": 3083.355165999994,
+ "toolMs": 0.010708000001613982,
+ "totalMs": 4925.301791000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1852.0635000000038,
+ "roundMs": 1291.135957999999,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.008000000001629815
+ "execMs": 0.010708000001613982
}
]
},
{
- "roundMs": 1518.40625,
- "firstDeltaMs": 1088.6654169999965,
+ "roundMs": 1792.219207999995,
+ "firstDeltaMs": 1423.303332999989,
"calls": []
}
]
@@ -7465,27 +7050,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1244,
- "p95": 1590,
- "mean": 1298,
+ "p50": 1322,
+ "p95": 1704,
+ "mean": 1357,
"n": 3
},
"ttftMs": {
- "p50": 4436,
- "p95": 4477,
- "mean": 4359,
+ "p50": 4359,
+ "p95": 4914,
+ "mean": 4382,
"n": 3
},
"totalMs": {
- "p50": 4806,
- "p95": 5200,
- "mean": 4856,
+ "p50": 4865,
+ "p95": 5619,
+ "mean": 4922,
"n": 3
},
"modelMs": {
- "p50": 3562,
- "p95": 4140,
- "mean": 3558,
+ "p50": 3818,
+ "p95": 3914,
+ "mean": 3564,
"n": 3
},
"toolMs": {
@@ -7508,33 +7093,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 3336.3469170000026,
- "execMs": 0.30420800000138115
+ "atMs": 2627.0454579999932,
+ "execMs": 0.44537499999569263
}
],
"timings": {
- "gateMs": 1243.5255840000027,
+ "gateMs": 1046.3580839999922,
"gateSkipped": false,
- "ttftMs": 4476.856333000003,
- "modelMs": 3562.290500999996,
- "toolMs": 0.30420800000138115,
- "totalMs": 4806.120293,
+ "ttftMs": 4359.357457999999,
+ "modelMs": 3818.1363329999876,
+ "toolMs": 0.44537499999569263,
+ "totalMs": 4864.939791999976,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2092.0751670000027,
+ "roundMs": 1580.2939999999944,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.30420800000138115
+ "execMs": 0.44537499999569263
}
]
},
{
- "roundMs": 1470.2153339999932,
- "firstDeltaMs": 1140.1649999999936,
+ "roundMs": 2237.8423329999932,
+ "firstDeltaMs": 1731.7602920000063,
"calls": []
}
]
@@ -7548,33 +7133,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 3016.591042,
- "execMs": 0.015457999994396232
+ "atMs": 3330.5359590000007,
+ "execMs": 0.016041000009863637
}
],
"timings": {
- "gateMs": 1589.9344580000034,
+ "gateMs": 1704.052834000002,
"gateSkipped": false,
- "ttftMs": 4164.299291999996,
- "modelMs": 2971.579332999987,
- "toolMs": 0.015457999994396232,
- "totalMs": 4561.529248999985,
+ "ttftMs": 4914.1885420000035,
+ "modelMs": 3914.4429999999993,
+ "toolMs": 0.016041000009863637,
+ "totalMs": 5618.511875000011,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1426.440957999992,
+ "roundMs": 1626.3777499999997,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.015457999994396232
+ "execMs": 0.016041000009863637
}
]
},
{
- "roundMs": 1545.138374999995,
- "firstDeltaMs": 1147.658666999996,
+ "roundMs": 2288.0652499999997,
+ "firstDeltaMs": 1583.5504160000128,
"calls": []
}
]
@@ -7588,33 +7173,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 3167.4812089999905,
- "execMs": 0.010542000003624707
+ "atMs": 2701.6382500000036,
+ "execMs": 0.005667000004905276
}
],
"timings": {
- "gateMs": 1059.5503330000065,
+ "gateMs": 1321.8989579999907,
"gateSkipped": false,
- "ttftMs": 4436.048249999993,
- "modelMs": 4140.019625000001,
- "toolMs": 0.010542000003624707,
- "totalMs": 5199.580500000011,
+ "ttftMs": 3873.2821250000125,
+ "modelMs": 2959.3408329999947,
+ "toolMs": 0.005667000004905276,
+ "totalMs": 4281.24545799999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2107.7405,
+ "roundMs": 1379.5172500000044,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.010542000003624707
+ "execMs": 0.005667000004905276
}
]
},
{
- "roundMs": 2032.279125000001,
- "firstDeltaMs": 1268.5378749999945,
+ "roundMs": 1579.8235829999903,
+ "firstDeltaMs": 1171.617499999993,
"calls": []
}
]
@@ -7625,38 +7210,38 @@
{
"name": "mute-all-does-not-redo-prior-turn",
"bucket": "single-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 935,
- "p95": 1150,
- "mean": 1043,
- "n": 2
+ "p50": 1112,
+ "p95": 1534,
+ "mean": 1236,
+ "n": 3
},
"ttftMs": {
- "p50": 4755,
- "p95": 6029,
- "mean": 5392,
- "n": 2
+ "p50": 4371,
+ "p95": 6561,
+ "mean": 4897,
+ "n": 3
},
"totalMs": {
- "p50": 5537,
- "p95": 6375,
- "mean": 5956,
- "n": 2
+ "p50": 4629,
+ "p95": 6820,
+ "mean": 5138,
+ "n": 3
},
"modelMs": {
- "p50": 4386,
- "p95": 5440,
- "mean": 4913,
- "n": 2
+ "p50": 3094,
+ "p95": 5708,
+ "mean": 3902,
+ "n": 3
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 3
},
"rounds": {
"mean": 2,
@@ -7672,115 +7257,113 @@
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 4916.985375000004,
- "execMs": 0.301791999998386
+ "atMs": 3219.9752089999965,
+ "execMs": 0.26783399999840185
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 4917.3217920000025,
- "execMs": 0.00491600000532344
+ "atMs": 3220.300875000001,
+ "execMs": 0.0035419999912846833
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 4917.339749999999,
- "execMs": 0.002207999990787357
+ "atMs": 3220.308499999999,
+ "execMs": 0.0007909999985713512
}
],
"timings": {
- "gateMs": 935.2387920000037,
+ "gateMs": 1534.4185829999915,
"gateSkipped": false,
- "ttftMs": 6029.220291999998,
- "modelMs": 5439.915500000003,
- "toolMs": 0.3089159999944968,
- "totalMs": 6375.463208000001,
+ "ttftMs": 4371.320292000004,
+ "modelMs": 3094.4679570000153,
+ "toolMs": 0.2721669999882579,
+ "totalMs": 4629.158706999995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3980.935249999995,
+ "roundMs": 1683.4417910000047,
"firstDeltaMs": null,
"calls": [
{
"name": "set_clip_muted",
- "execMs": 0.301791999998386
+ "execMs": 0.26783399999840185
},
{
"name": "set_clip_muted",
- "execMs": 0.00491600000532344
+ "execMs": 0.0035419999912846833
},
{
"name": "set_clip_muted",
- "execMs": 0.002207999990787357
+ "execMs": 0.0007909999985713512
}
]
},
{
- "roundMs": 1458.9802500000078,
- "firstDeltaMs": 1111.8421250000101,
+ "roundMs": 1411.0261660000106,
+ "firstDeltaMs": 1150.9711249999964,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "reply did not match /mut/i"
- ],
+ "notes": [],
"trace": [
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 3710.740917000003,
- "execMs": 0.031208000000333413
+ "atMs": 5255.688125000001,
+ "execMs": 0.022792000003391877
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 3710.7838749999937,
- "execMs": 0.0018330000020796433
+ "atMs": 5255.726041999995,
+ "execMs": 0.0010000000038417056
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 3710.787666999997,
- "execMs": 0.0010840000031748787
+ "atMs": 5255.72825,
+ "execMs": 0.0005839999939780682
}
],
"timings": {
- "gateMs": 1599.208541999993,
+ "gateMs": 1112.3079170000128,
"gateSkipped": false,
- "ttftMs": 4966.215500000006,
- "modelMs": 3747.3297489999823,
- "toolMs": 0.034125000005587935,
- "totalMs": 5346.572415999981,
+ "ttftMs": 6561.047583000007,
+ "modelMs": 5708.088416000013,
+ "toolMs": 0.02437600000121165,
+ "totalMs": 6820.420709000027,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2111.0158329999977,
+ "roundMs": 4142.879790999999,
"firstDeltaMs": null,
"calls": [
{
"name": "set_clip_muted",
- "execMs": 0.031208000000333413
+ "execMs": 0.022792000003391877
},
{
"name": "set_clip_muted",
- "execMs": 0.0018330000020796433
+ "execMs": 0.0010000000038417056
},
{
"name": "set_clip_muted",
- "execMs": 0.0010840000031748787
+ "execMs": 0.0005839999939780682
}
]
},
{
- "roundMs": 1636.3139159999846,
- "firstDeltaMs": 1255.3912079999864,
+ "roundMs": 1565.2086250000139,
+ "firstDeltaMs": 1305.2740420000046,
"calls": []
}
]
@@ -7794,53 +7377,53 @@
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 3295.4305419999873,
- "execMs": 0.0058339999814052135
+ "atMs": 2652.4899580000056,
+ "execMs": 0.006250000005820766
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 3295.438958999992,
- "execMs": 0.0002500000118743628
+ "atMs": 2652.499416999999,
+ "execMs": 0.00045900000259280205
},
{
"name": "set_clip_muted",
"round": 0,
- "atMs": 3295.4395419999782,
- "execMs": 0.0001660000125411898
+ "atMs": 2652.5002499999973,
+ "execMs": 0.0002500000118743628
}
],
"timings": {
- "gateMs": 1150.4422499999928,
+ "gateMs": 1061.4372500000027,
"gateSkipped": false,
- "ttftMs": 4754.574791999999,
- "modelMs": 4386.206833000004,
- "toolMs": 0.006250000005820766,
- "totalMs": 5536.655333000002,
+ "ttftMs": 3759.5074170000007,
+ "modelMs": 2902.0028329999914,
+ "toolMs": 0.006959000020287931,
+ "totalMs": 3963.4470420000143,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2144.605500000005,
+ "roundMs": 1590.6940829999949,
"firstDeltaMs": null,
"calls": [
{
"name": "set_clip_muted",
- "execMs": 0.0058339999814052135
+ "execMs": 0.006250000005820766
},
{
"name": "set_clip_muted",
- "execMs": 0.0002500000118743628
+ "execMs": 0.00045900000259280205
},
{
"name": "set_clip_muted",
- "execMs": 0.0001660000125411898
+ "execMs": 0.0002500000118743628
}
]
},
{
- "roundMs": 2241.6013329999987,
- "firstDeltaMs": 1459.126791000017,
+ "roundMs": 1311.3087499999965,
+ "firstDeltaMs": 1106.992166000011,
"calls": []
}
]
@@ -7855,27 +7438,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1258,
- "p95": 1701,
- "mean": 1321,
+ "p50": 1372,
+ "p95": 1510,
+ "mean": 1329,
"n": 3
},
"ttftMs": {
- "p50": 4928,
- "p95": 5181,
- "mean": 4578,
+ "p50": 4425,
+ "p95": 4625,
+ "mean": 4423,
"n": 3
},
"totalMs": {
- "p50": 5336,
- "p95": 5549,
- "mean": 4952,
+ "p50": 4729,
+ "p95": 5452,
+ "mean": 4918,
"n": 3
},
"modelMs": {
- "p50": 3848,
- "p95": 4079,
- "mean": 3631,
+ "p50": 3623,
+ "p95": 3942,
+ "mean": 3589,
"n": 3
},
"toolMs": {
@@ -7898,33 +7481,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 2620.5324580000015,
- "execMs": 0.0952079999842681
+ "atMs": 2838.838167000009,
+ "execMs": 0.22641599999042228
}
],
"timings": {
- "gateMs": 1003.0530409999774,
+ "gateMs": 1106.098417000001,
"gateSkipped": false,
- "ttftMs": 3625.3310420000053,
- "modelMs": 2965.6724579999864,
- "toolMs": 0.0952079999842681,
- "totalMs": 3968.820706999948,
+ "ttftMs": 4424.896375000011,
+ "modelMs": 3623.112334000034,
+ "toolMs": 0.22641599999042228,
+ "totalMs": 4729.4371670000255,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1616.692458000005,
+ "roundMs": 1731.9346250000235,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.0952079999842681
+ "execMs": 0.22641599999042228
}
]
},
{
- "roundMs": 1348.9799999999814,
- "firstDeltaMs": 1004.6903329999768,
+ "roundMs": 1891.1777090000105,
+ "firstDeltaMs": 1585.8071250000212,
"calls": []
}
]
@@ -7938,33 +7521,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 3483.157666999992,
- "execMs": 0.04625000001396984
+ "atMs": 2901.053917000012,
+ "execMs": 0.0412089999881573
}
],
"timings": {
- "gateMs": 1257.7520829999994,
+ "gateMs": 1510.0536250000005,
"gateSkipped": false,
- "ttftMs": 4928.0255419999885,
- "modelMs": 4078.6073739999556,
- "toolMs": 0.04625000001396984,
- "totalMs": 5336.405706999969,
+ "ttftMs": 4625.0814170000085,
+ "modelMs": 3941.5287920000264,
+ "toolMs": 0.0412089999881573,
+ "totalMs": 5451.623626000015,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2225.1295829999726,
+ "roundMs": 1390.7419590000063,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.04625000001396984
+ "execMs": 0.0412089999881573
}
]
},
{
- "roundMs": 1853.477790999983,
- "firstDeltaMs": 1444.780207999982,
+ "roundMs": 2550.78683300002,
+ "firstDeltaMs": 1723.9406250000175,
"calls": []
}
]
@@ -7978,33 +7561,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 3928.7020000000193,
- "execMs": 0.024166000017430633
+ "atMs": 2978.646374999982,
+ "execMs": 0.0262079999956768
}
],
"timings": {
- "gateMs": 1701.0930829999852,
+ "gateMs": 1372.1904170000053,
"gateSkipped": false,
- "ttftMs": 5181.341208000027,
- "modelMs": 3848.303500000009,
- "toolMs": 0.024166000017430633,
- "totalMs": 5549.420749000012,
+ "ttftMs": 4219.875124999991,
+ "modelMs": 3202.160749999981,
+ "toolMs": 0.0262079999956768,
+ "totalMs": 4574.377374999982,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2227.349583000003,
+ "roundMs": 1606.196499999991,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.024166000017430633
+ "execMs": 0.0262079999956768
}
]
},
{
- "roundMs": 1620.9539170000062,
- "firstDeltaMs": 1252.6002500000177,
+ "roundMs": 1595.96424999999,
+ "firstDeltaMs": 1241.1663750000007,
"calls": []
}
]
@@ -8015,38 +7598,38 @@
{
"name": "audio-ask-single-generate",
"bucket": "single-tool",
- "passRate": 1,
+ "passRate": 0.6666666666666666,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1289,
- "p95": 1585,
- "mean": 1332,
- "n": 3
+ "p50": 1074,
+ "p95": 1392,
+ "mean": 1233,
+ "n": 2
},
"ttftMs": {
- "p50": 4429,
- "p95": 4553,
- "mean": 4368,
- "n": 3
+ "p50": 3812,
+ "p95": 3865,
+ "mean": 3839,
+ "n": 2
},
"totalMs": {
- "p50": 4811,
- "p95": 4827,
- "mean": 4773,
- "n": 3
+ "p50": 4080,
+ "p95": 4089,
+ "mean": 4085,
+ "n": 2
},
"modelMs": {
- "p50": 3522,
- "p95": 3706,
- "mean": 3441,
- "n": 3
+ "p50": 2698,
+ "p95": 3006,
+ "mean": 2852,
+ "n": 2
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 3
+ "n": 2
},
"rounds": {
"mean": 2,
@@ -8062,73 +7645,75 @@
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 3136.8862910000025,
- "execMs": 0.015167000005021691
+ "atMs": 2672.9105000000272,
+ "execMs": 0.04100000002654269
}
],
"timings": {
- "gateMs": 1584.7591249999823,
+ "gateMs": 1074.154291999992,
"gateSkipped": false,
- "ttftMs": 4429.297875000018,
- "modelMs": 3096.0840000000026,
- "toolMs": 0.015167000005021691,
- "totalMs": 4680.85829199999,
+ "ttftMs": 3812.462542000023,
+ "modelMs": 3005.6591249999765,
+ "toolMs": 0.04100000002654269,
+ "totalMs": 4079.854416999995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1551.4524169999931,
+ "roundMs": 1598.5749999999825,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.015167000005021691
+ "execMs": 0.04100000002654269
}
]
},
{
- "roundMs": 1544.6315830000094,
- "firstDeltaMs": 1292.3201249999984,
+ "roundMs": 1407.084124999994,
+ "firstDeltaMs": 1139.453084000008,
"calls": []
}
]
}
},
{
- "pass": true,
+ "pass": false,
"intent": "work",
- "notes": [],
+ "notes": [
+ "reply did not match /voice|audio|narrat|welcome/i"
+ ],
"trace": [
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 3319.8856249999953,
- "execMs": 0.030209000018658116
+ "atMs": 2251.0040410000074,
+ "execMs": 0.007000000012340024
}
],
"timings": {
- "gateMs": 1288.542207999999,
+ "gateMs": 928.0902500000084,
"gateSkipped": false,
- "ttftMs": 4553.2693750000035,
- "modelMs": 3521.9472500000265,
- "toolMs": 0.030209000018658116,
- "totalMs": 4810.519667000044,
+ "ttftMs": 3436.292583000002,
+ "modelMs": 2811.5891249999986,
+ "toolMs": 0.007000000012340024,
+ "totalMs": 3739.6863750000193,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2030.8562500000116,
+ "roundMs": 1322.8132500000065,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.030209000018658116
+ "execMs": 0.007000000012340024
}
]
},
{
- "roundMs": 1491.091000000015,
- "firstDeltaMs": 1233.247583999997,
+ "roundMs": 1488.775874999992,
+ "firstDeltaMs": 1185.2578750000102,
"calls": []
}
]
@@ -8142,33 +7727,33 @@
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 2685.190583000018,
- "execMs": 0.0026249999937135726
+ "atMs": 2813.2490830000024,
+ "execMs": 0.010000000009313226
}
],
"timings": {
- "gateMs": 1121.4327079999784,
+ "gateMs": 1391.7804160000233,
"gateSkipped": false,
- "ttftMs": 4122.049458000023,
- "modelMs": 3705.6319169999915,
- "toolMs": 0.0026249999937135726,
- "totalMs": 4827.067249999964,
+ "ttftMs": 3864.5833750000165,
+ "modelMs": 2697.5791249999893,
+ "toolMs": 0.010000000009313226,
+ "totalMs": 4089.369541000022,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1563.506167000014,
+ "roundMs": 1421.324208999984,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.0026249999937135726
+ "execMs": 0.010000000009313226
}
]
},
{
- "roundMs": 2142.1257499999774,
- "firstDeltaMs": 1436.8285840000026,
+ "roundMs": 1276.2549160000053,
+ "firstDeltaMs": 1051.2846250000002,
"calls": []
}
]
@@ -8179,260 +7764,160 @@
{
"name": "music-ask-single-generate",
"bucket": "single-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1070,
+ "p95": 1087,
+ "mean": 1065,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 3679,
+ "p95": 4651,
+ "mean": 3990,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 4070,
+ "p95": 5061,
+ "mean": 4327,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 2982,
+ "p95": 3991,
+ "mean": 3262,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "slow: 5 tool calls (cap 3)",
- "mutating tools called: add_clip, update_audio, add_clip, add_clip"
- ],
+ "notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 2352.829874999996,
- "execMs": 0.024375000008149073
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 4430.645499999984,
- "execMs": 0.01570799999171868
- },
- {
- "name": "update_audio",
- "round": 2,
- "atMs": 5833.14158299999,
- "execMs": 0.012915999977849424
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 7860.4343749999825,
- "execMs": 0.006209000013768673
- },
- {
- "name": "add_clip",
- "round": 4,
- "atMs": 9272.745790999994,
- "execMs": 0.007958999980473891
+ "atMs": 3474.65395800001,
+ "execMs": 0.35604099999181926
}
],
"timings": {
- "gateMs": 909.80349999998,
+ "gateMs": 1069.5192919999827,
"gateSkipped": false,
- "ttftMs": 10488.381166000006,
- "modelMs": 9918.128708000004,
- "toolMs": 0.06716699997195974,
- "totalMs": 10827.999374999956,
+ "ttftMs": 4650.808500000014,
+ "modelMs": 3991.063374999998,
+ "toolMs": 0.35604099999181926,
+ "totalMs": 5060.938707999972,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1442.8079579999903,
+ "roundMs": 2404.728375000006,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.024375000008149073
+ "execMs": 0.35604099999181926
}
]
},
{
- "roundMs": 2077.7396660000086,
+ "roundMs": 1586.3349999999919,
+ "firstDeltaMs": 1175.7673749999958,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "work",
+ "notes": [],
+ "trace": [
+ {
+ "name": "generate_music",
+ "round": 0,
+ "atMs": 2370.9234580000048,
+ "execMs": 0.013124999997671694
+ }
+ ],
+ "timings": {
+ "gateMs": 1039.711209000001,
+ "gateSkipped": false,
+ "ttftMs": 3640.502750000014,
+ "modelMs": 2810.99787500003,
+ "toolMs": 0.013124999997671694,
+ "totalMs": 3850.7222090000287,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1331.0415000000212,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.01570799999171868
+ "name": "generate_music",
+ "execMs": 0.013124999997671694
}
]
},
{
- "roundMs": 1402.450291999994,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_audio",
- "execMs": 0.012915999977849424
- }
- ]
- },
- {
- "roundMs": 2027.2419170000067,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.006209000013768673
- }
- ]
- },
- {
- "roundMs": 1412.2362500000163,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.007958999980473891
- }
- ]
- },
- {
- "roundMs": 1555.652624999988,
- "firstDeltaMs": 1215.5909579999861,
+ "roundMs": 1479.9563750000088,
+ "firstDeltaMs": 1269.5314589999907,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: add_clip, update_audio"
- ],
+ "notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 2324.0570409999928,
- "execMs": 0.012709000002359971
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 4285.779165999993,
- "execMs": 0.04129100000136532
- },
- {
- "name": "update_audio",
- "round": 2,
- "atMs": 8096.656125000009,
- "execMs": 0.0175419999868609
+ "atMs": 2460.538916999998,
+ "execMs": 0.011624999984633178
}
],
"timings": {
- "gateMs": 966.6573330000101,
+ "gateMs": 1087.0885409999755,
"gateSkipped": false,
- "ttftMs": 9077.927625000011,
- "modelMs": 8459.698082000046,
- "toolMs": 0.07154199999058619,
- "totalMs": 9426.426957000047,
+ "ttftMs": 3678.6370830000087,
+ "modelMs": 2982.457874999993,
+ "toolMs": 0.011624999984633178,
+ "totalMs": 4069.558040999953,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1357.126042000018,
+ "roundMs": 1373.2132500000007,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.012709000002359971
- }
- ]
- },
- {
- "roundMs": 1961.639833000023,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.04129100000136532
- }
- ]
- },
- {
- "roundMs": 3810.5692909999925,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_audio",
- "execMs": 0.0175419999868609
+ "execMs": 0.011624999984633178
}
]
},
{
- "roundMs": 1330.3629160000128,
- "firstDeltaMs": 981.1439580000006,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": false,
- "intent": "work",
- "notes": [
- "mutating tools called: add_clip, update_audio"
- ],
- "trace": [
- {
- "name": "generate_music",
- "round": 0,
- "atMs": 2544.986917000002,
- "execMs": 0.01433299999916926
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 3832.2056250000023,
- "execMs": 0.03245899998000823
- },
- {
- "name": "update_audio",
- "round": 2,
- "atMs": 5347.859041999996,
- "execMs": 0.01662499998928979
- }
- ],
- "timings": {
- "gateMs": 1265.5523749999993,
- "gateSkipped": false,
- "ttftMs": 6887.767625000008,
- "modelMs": 6463.078042000037,
- "toolMs": 0.06341699996846728,
- "totalMs": 7728.693834000005,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1278.9673330000078,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_music",
- "execMs": 0.01433299999916926
- }
- ]
- },
- {
- "roundMs": 1286.986709000019,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.03245899998000823
- }
- ]
- },
- {
- "roundMs": 1515.5223330000008,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_audio",
- "execMs": 0.01662499998928979
- }
- ]
- },
- {
- "roundMs": 2381.60166700001,
- "firstDeltaMs": 1539.8420000000042,
+ "roundMs": 1609.244624999992,
+ "firstDeltaMs": 1218.0540420000034,
"calls": []
}
]
@@ -8447,27 +7932,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1188,
- "p95": 1203,
- "mean": 1140,
+ "p50": 1068,
+ "p95": 1852,
+ "mean": 1319,
"n": 3
},
"ttftMs": {
- "p50": 3447,
- "p95": 3459,
- "mean": 3411,
+ "p50": 4463,
+ "p95": 7354,
+ "mean": 5172,
"n": 3
},
"totalMs": {
- "p50": 3629,
- "p95": 3675,
- "mean": 3613,
+ "p50": 4579,
+ "p95": 7590,
+ "mean": 5506,
"n": 3
},
"modelMs": {
- "p50": 2486,
- "p95": 2508,
- "mean": 2473,
+ "p50": 3511,
+ "p95": 5738,
+ "mean": 4187,
"n": 3
},
"toolMs": {
@@ -8490,33 +7975,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 2352.953999999998,
- "execMs": 0.034500000008847564
+ "atMs": 2317.528624999977,
+ "execMs": 0.048584000003756955
}
],
"timings": {
- "gateMs": 1028.4529589999875,
+ "gateMs": 1036.3330840000126,
"gateSkipped": false,
- "ttftMs": 3327.1377499999944,
- "modelMs": 2508.2444159999723,
- "toolMs": 0.034500000008847564,
- "totalMs": 3536.7318749999686,
+ "ttftMs": 3699.506374999997,
+ "modelMs": 3312.2827080000134,
+ "toolMs": 0.048584000003756955,
+ "totalMs": 4348.66437600003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1323.775082999986,
+ "roundMs": 1280.691542000015,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.034500000008847564
+ "execMs": 0.048584000003756955
}
]
},
{
- "roundMs": 1184.4693329999864,
- "firstDeltaMs": 974.091041000007,
+ "roundMs": 2031.5911659999983,
+ "firstDeltaMs": 1381.8953329999931,
"calls": []
}
]
@@ -8530,33 +8015,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 2477.612666000001,
- "execMs": 0.0009579999896232039
+ "atMs": 3186.908708999981,
+ "execMs": 0.0020420000073499978
}
],
"timings": {
- "gateMs": 1203.2707079999964,
+ "gateMs": 1067.761959000025,
"gateSkipped": false,
- "ttftMs": 3458.997541000019,
- "modelMs": 2425.6767490000057,
- "toolMs": 0.0009579999896232039,
- "totalMs": 3628.9484149999917,
+ "ttftMs": 4463.38274999999,
+ "modelMs": 3511.068708000006,
+ "toolMs": 0.0020420000073499978,
+ "totalMs": 4578.832709000038,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1274.1388330000045,
+ "roundMs": 2118.9568750000035,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.0009579999896232039
+ "execMs": 0.0020420000073499978
}
]
},
{
- "roundMs": 1151.5379160000011,
- "firstDeltaMs": 981.3740409999737,
+ "roundMs": 1392.1118330000027,
+ "firstDeltaMs": 1276.462124999991,
"calls": []
}
]
@@ -8570,33 +8055,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 2486.0007500000065,
- "execMs": 0.0024170000106096268
+ "atMs": 6177.778583999985,
+ "execMs": 0.0019580000080168247
}
],
"timings": {
- "gateMs": 1188.352499999979,
+ "gateMs": 1851.666499999992,
"gateSkipped": false,
- "ttftMs": 3446.765416999988,
- "modelMs": 2486.1710410000233,
- "toolMs": 0.0024170000106096268,
- "totalMs": 3674.525958000013,
+ "ttftMs": 7354.213292,
+ "modelMs": 5738.12858400002,
+ "toolMs": 0.0019580000080168247,
+ "totalMs": 7589.79704200002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1297.4126660000184,
+ "roundMs": 4325.887291999999,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.0024170000106096268
+ "execMs": 0.0019580000080168247
}
]
},
{
- "roundMs": 1188.758375000005,
- "firstDeltaMs": 960.7363750000077,
+ "roundMs": 1412.2412920000206,
+ "firstDeltaMs": 1176.4167080000043,
"calls": []
}
]
@@ -8611,27 +8096,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1012,
- "p95": 1549,
- "mean": 1169,
+ "p50": 906,
+ "p95": 1668,
+ "mean": 1149,
"n": 3
},
"ttftMs": {
- "p50": 3935,
- "p95": 4029,
- "mean": 3772,
+ "p50": 4513,
+ "p95": 5113,
+ "mean": 4665,
"n": 3
},
"totalMs": {
- "p50": 4212,
- "p95": 4259,
- "mean": 4073,
+ "p50": 4869,
+ "p95": 5398,
+ "mean": 4951,
"n": 3
},
"modelMs": {
- "p50": 2800,
- "p95": 3200,
- "mean": 2903,
+ "p50": 3730,
+ "p95": 3995,
+ "mean": 3802,
"n": 3
},
"toolMs": {
@@ -8654,33 +8139,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 2865.755416,
- "execMs": 0.20999999999185093
+ "atMs": 3296.9999160000007,
+ "execMs": 0.20070799998939037
}
],
"timings": {
- "gateMs": 1548.92637500001,
+ "gateMs": 873.8488329999964,
"gateSkipped": false,
- "ttftMs": 4029.4106249999895,
- "modelMs": 2710.3154580000264,
- "toolMs": 0.20999999999185093,
- "totalMs": 4259.451833000028,
+ "ttftMs": 4512.664625000005,
+ "modelMs": 3995.05479200001,
+ "toolMs": 0.20070799998939037,
+ "totalMs": 4869.104332999996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1316.145416000014,
+ "roundMs": 2422.380375000008,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.20999999999185093
+ "execMs": 0.20070799998939037
}
]
},
{
- "roundMs": 1394.1700420000125,
- "firstDeltaMs": 1163.4228750000184,
+ "roundMs": 1572.674417000002,
+ "firstDeltaMs": 1215.4407090000168,
"calls": []
}
]
@@ -8694,33 +8179,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 2346.6360830000194,
- "execMs": 0.0044169999891892076
+ "atMs": 3916.6136660000193,
+ "execMs": 0.007833000010577962
}
],
"timings": {
- "gateMs": 945.8653329999943,
+ "gateMs": 1667.9967499999912,
"gateSkipped": false,
- "ttftMs": 3352.89279100002,
- "modelMs": 2800.444165999972,
- "toolMs": 0.0044169999891892076,
- "totalMs": 3746.3139159999555,
+ "ttftMs": 5112.969666000019,
+ "modelMs": 3730.319958999986,
+ "toolMs": 0.007833000010577962,
+ "totalMs": 5398.324541999988,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1400.5318329999864,
+ "roundMs": 2248.4540839999972,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.0044169999891892076
+ "execMs": 0.007833000010577962
}
]
},
{
- "roundMs": 1399.9123329999857,
- "firstDeltaMs": 1006.2387910000107,
+ "roundMs": 1481.8658749999886,
+ "firstDeltaMs": 1196.3293330000015,
"calls": []
}
]
@@ -8734,33 +8219,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 2976.0412080000096,
- "execMs": 0.006334000005153939
+ "atMs": 3087.6502920000057,
+ "execMs": 0.006417000025976449
}
],
"timings": {
- "gateMs": 1012.3754170000029,
+ "gateMs": 906.0476669999771,
"gateSkipped": false,
- "ttftMs": 3934.9016660000198,
- "modelMs": 3199.6600409999955,
- "toolMs": 0.006334000005153939,
- "totalMs": 4212.041792000004,
+ "ttftMs": 4368.9250000000175,
+ "modelMs": 3680.642999999982,
+ "toolMs": 0.006417000025976449,
+ "totalMs": 4586.697083999985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1963.3644579999964,
+ "roundMs": 2181.3552909999853,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.006334000005153939
+ "execMs": 0.006417000025976449
}
]
},
{
- "roundMs": 1236.2955829999992,
- "firstDeltaMs": 958.8462080000027,
+ "roundMs": 1499.2877089999965,
+ "firstDeltaMs": 1281.2564999999886,
"calls": []
}
]
@@ -8804,182 +8289,160 @@
{
"name": "look-ask-single-tool",
"bucket": "single-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1139,
+ "p95": 1208,
+ "mean": 1138,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 3970,
+ "p95": 4430,
+ "mean": 4023,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 4330,
+ "p95": 5084,
+ "mean": 4474,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 3121,
+ "p95": 3945,
+ "mean": 3335,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, add_effect"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 2740.2225000000035,
- "execMs": 0.08262500001001172
- },
- {
- "name": "add_effect",
- "round": 1,
- "atMs": 4083.1454170000216,
- "execMs": 0.012457999982871115
+ "atMs": 2797.1249170000083,
+ "execMs": 0.33541599998716265
}
],
"timings": {
- "gateMs": 1204.6799170000013,
+ "gateMs": 1208.25991600001,
"gateSkipped": false,
- "ttftMs": 5566.313584000018,
- "modelMs": 5067.257290999987,
- "toolMs": 0.09508299999288283,
- "totalMs": 6272.032290999981,
+ "ttftMs": 3969.9852919999976,
+ "modelMs": 3121.335290999996,
+ "toolMs": 0.33541599998716265,
+ "totalMs": 4329.930622999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1534.9307919999992,
+ "roundMs": 1588.4767500000016,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.08262500001001172
+ "execMs": 0.33541599998716265
}
]
},
{
- "roundMs": 1342.562791000004,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_effect",
- "execMs": 0.012457999982871115
- }
- ]
- },
- {
- "roundMs": 2189.763707999984,
- "firstDeltaMs": 1483.1222919999855,
+ "roundMs": 1532.8585409999941,
+ "firstDeltaMs": 1172.4601660000044,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, add_effect"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 2999.3856249999953,
- "execMs": 0.011709000013070181
- },
- {
- "name": "add_effect",
- "round": 1,
- "atMs": 4456.715375,
- "execMs": 0.0035000000207219273
+ "atMs": 2442.377582999994,
+ "execMs": 0.021792000014102086
}
],
"timings": {
- "gateMs": 1415.677874999994,
+ "gateMs": 1068.0635840000177,
"gateSkipped": false,
- "ttftMs": 5903.853207999986,
- "modelMs": 5285.484331999993,
- "toolMs": 0.015209000033792108,
- "totalMs": 6701.1774160000205,
+ "ttftMs": 3670.006708000001,
+ "modelMs": 2939.4694590000145,
+ "toolMs": 0.021792000014102086,
+ "totalMs": 4007.5548350000463,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1583.5894580000022,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_effect",
- "execMs": 0.011709000013070181
- }
- ]
- },
- {
- "roundMs": 1457.278707999998,
+ "roundMs": 1374.189417000016,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.0035000000207219273
+ "execMs": 0.021792000014102086
}
]
},
{
- "roundMs": 2244.6161659999925,
- "firstDeltaMs": 1447.1242499999935,
+ "roundMs": 1565.2800419999985,
+ "firstDeltaMs": 1227.5685839999933,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, add_effect"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 2427.076000000001,
- "execMs": 0.011667000013403594
- },
- {
- "name": "add_effect",
- "round": 1,
- "atMs": 5527.0410409999895,
- "execMs": 0.003542000020388514
+ "atMs": 2819.118124999979,
+ "execMs": 0.035582999989856035
}
],
"timings": {
- "gateMs": 1022.199042000022,
+ "gateMs": 1138.7052919999987,
"gateSkipped": false,
- "ttftMs": 9346.567749999987,
- "modelMs": 8731.220084,
- "toolMs": 0.015209000033792108,
- "totalMs": 9753.434335000056,
+ "ttftMs": 4430.461749999988,
+ "modelMs": 3945.028874999989,
+ "toolMs": 0.035582999989856035,
+ "totalMs": 5083.769749999978,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1404.7456250000105,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_effect",
- "execMs": 0.011667000013403594
- }
- ]
- },
- {
- "roundMs": 3099.9072499999893,
+ "roundMs": 1680.133375000005,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.003542000020388514
+ "execMs": 0.035582999989856035
}
]
},
{
- "roundMs": 4226.567209000001,
- "firstDeltaMs": 3819.513666999992,
+ "roundMs": 2264.895499999984,
+ "firstDeltaMs": 1611.2174170000071,
"calls": []
}
]
@@ -8994,27 +8457,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1717,
- "p95": 1778,
- "mean": 1541,
+ "p50": 969,
+ "p95": 1133,
+ "mean": 976,
"n": 3
},
"ttftMs": {
- "p50": 4196,
- "p95": 4390,
- "mean": 4188,
+ "p50": 3888,
+ "p95": 3903,
+ "mean": 3734,
"n": 3
},
"totalMs": {
- "p50": 4489,
- "p95": 4642,
- "mean": 4525,
+ "p50": 4210,
+ "p95": 4635,
+ "mean": 4145,
"n": 3
},
"modelMs": {
- "p50": 2863,
- "p95": 3315,
- "mean": 2984,
+ "p50": 3077,
+ "p95": 3666,
+ "mean": 3169,
"n": 3
},
"toolMs": {
@@ -9037,43 +8500,93 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 3164.3826249999984,
- "execMs": 0.3890839999949094
+ "atMs": 2564.475959000003,
+ "execMs": 0.07949999999254942
+ },
+ {
+ "name": "set_animation",
+ "round": 0,
+ "atMs": 2564.564792000019,
+ "execMs": 0.0022920000192243606
+ }
+ ],
+ "timings": {
+ "gateMs": 1133.2511250000098,
+ "gateSkipped": false,
+ "ttftMs": 3903.35275000002,
+ "modelMs": 3076.7766249999986,
+ "toolMs": 0.08179200001177378,
+ "totalMs": 4210.10954200002,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1430.4916669999948,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_transition",
+ "execMs": 0.07949999999254942
+ },
+ {
+ "name": "set_animation",
+ "execMs": 0.0022920000192243606
+ }
+ ]
+ },
+ {
+ "roundMs": 1646.2849580000038,
+ "firstDeltaMs": 1338.7371660000063,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "work",
+ "notes": [],
+ "trace": [
+ {
+ "name": "set_transition",
+ "round": 0,
+ "atMs": 2313.433208000002,
+ "execMs": 0.005042000004323199
},
{
"name": "set_animation",
"round": 0,
- "atMs": 3164.80929200002,
- "execMs": 0.01204100000904873
+ "atMs": 2313.4406660000095,
+ "execMs": 0.001166999980341643
}
],
"timings": {
- "gateMs": 1778.3240840000217,
+ "gateMs": 968.6332080000138,
"gateSkipped": false,
- "ttftMs": 4390.291625000013,
- "modelMs": 2863.241167,
- "toolMs": 0.4011250000039581,
- "totalMs": 4641.966376000026,
+ "ttftMs": 3888.1846660000156,
+ "modelMs": 3666.215541999991,
+ "toolMs": 0.0062089999846648425,
+ "totalMs": 4634.854958999989,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1385.6523750000051,
+ "roundMs": 1344.532124999998,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.3890839999949094
+ "execMs": 0.005042000004323199
},
{
"name": "set_animation",
- "execMs": 0.01204100000904873
+ "execMs": 0.001166999980341643
}
]
},
{
- "roundMs": 1477.588791999995,
- "firstDeltaMs": 1225.4359169999952,
+ "roundMs": 2321.683416999993,
+ "firstDeltaMs": 1574.7257499999832,
"calls": []
}
]
@@ -9087,321 +8600,455 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 3068.179666000011,
- "execMs": 0.010708000016165897
+ "atMs": 2248.459582999989,
+ "execMs": 0.004082999977981672
},
{
"name": "set_animation",
"round": 0,
- "atMs": 3068.2015829999873,
- "execMs": 0.0068749999918509275
+ "atMs": 2248.4675829999906,
+ "execMs": 0.002625000022817403
}
],
"timings": {
- "gateMs": 1716.7468329999829,
+ "gateMs": 826.4775410000002,
"gateSkipped": false,
- "ttftMs": 4195.672500000015,
- "modelMs": 2772.1944169999915,
- "toolMs": 0.017583000008016825,
- "totalMs": 4488.958832999982,
+ "ttftMs": 3411.4535000000033,
+ "modelMs": 2763.461874999979,
+ "toolMs": 0.006708000000799075,
+ "totalMs": 3589.94612399998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1351.117041999998,
+ "roundMs": 1421.7795000000042,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_transition",
+ "execMs": 0.004082999977981672
+ },
+ {
+ "name": "set_animation",
+ "execMs": 0.002625000022817403
+ }
+ ]
+ },
+ {
+ "roundMs": 1341.6823749999749,
+ "firstDeltaMs": 1162.9731249999895,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "filler-words-cut-with-editor",
+ "bucket": "multi-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 1024,
+ "p95": 1107,
+ "mean": 1023,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 8102,
+ "p95": 13653,
+ "mean": 9908,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 8490,
+ "p95": 14116,
+ "mean": 10332,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 7452,
+ "p95": 13090,
+ "mean": 9308,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 1,
+ "p95": 2,
+ "mean": 1,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 5.3,
+ "max": 7
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "work",
+ "notes": [],
+ "trace": [
+ {
+ "name": "split_at",
+ "round": 0,
+ "atMs": 2657.5356670000183,
+ "execMs": 0.48112499999115244
+ },
+ {
+ "name": "split_at",
+ "round": 0,
+ "atMs": 2658.0206250000047,
+ "execMs": 0.010750000015832484
+ },
+ {
+ "name": "split_at",
+ "round": 0,
+ "atMs": 2658.0322500000184,
+ "execMs": 0.0057499999820720404
+ },
+ {
+ "name": "split_at",
+ "round": 0,
+ "atMs": 2658.0384170000034,
+ "execMs": 0.004584000009344891
+ },
+ {
+ "name": "split_at",
+ "round": 0,
+ "atMs": 2658.043417000008,
+ "execMs": 0.01870799998869188
+ },
+ {
+ "name": "split_at",
+ "round": 0,
+ "atMs": 2658.0628750000033,
+ "execMs": 0.00979199999710545
+ },
+ {
+ "name": "delete_item",
+ "round": 1,
+ "atMs": 3990.695792000013,
+ "execMs": 0.6002499999885913
+ },
+ {
+ "name": "delete_item",
+ "round": 1,
+ "atMs": 3991.310958999995,
+ "execMs": 0.022417000000132248
+ },
+ {
+ "name": "get_state",
+ "round": 2,
+ "atMs": 5399.907666999992,
+ "execMs": 0.5670000000100117
+ },
+ {
+ "name": "delete_item",
+ "round": 3,
+ "atMs": 7169.571458999999,
+ "execMs": 0.08008400001563132
+ },
+ {
+ "name": "delete_item",
+ "round": 3,
+ "atMs": 7169.658334000007,
+ "execMs": 0.012917000014567748
+ },
+ {
+ "name": "delete_cue",
+ "round": 4,
+ "atMs": 10867.987874999992,
+ "execMs": 0.05208300001686439
+ },
+ {
+ "name": "delete_cue",
+ "round": 4,
+ "atMs": 10868.045084000012,
+ "execMs": 0.013957999995909631
+ },
+ {
+ "name": "update_cue",
+ "round": 5,
+ "atMs": 12632.397250000009,
+ "execMs": 0.07875000001513399
+ },
+ {
+ "name": "update_cue",
+ "round": 5,
+ "atMs": 12632.508292000013,
+ "execMs": 0.010500000003958121
+ },
+ {
+ "name": "update_cue",
+ "round": 5,
+ "atMs": 12632.53675,
+ "execMs": 0.007457999978214502
+ }
+ ],
+ "timings": {
+ "gateMs": 1023.5210420000076,
+ "gateSkipped": false,
+ "ttftMs": 13652.64625000002,
+ "modelMs": 13090.095709000045,
+ "toolMs": 1.9761260000232141,
+ "totalMs": 14115.592877000076,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1633.2098750000005,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "split_at",
+ "execMs": 0.48112499999115244
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.010750000015832484
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.0057499999820720404
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.004584000009344891
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.01870799998869188
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.00979199999710545
+ }
+ ]
+ },
+ {
+ "roundMs": 1332.5786670000234,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "delete_item",
+ "execMs": 0.6002499999885913
+ },
+ {
+ "name": "delete_item",
+ "execMs": 0.022417000000132248
+ }
+ ]
+ },
+ {
+ "roundMs": 1408.4932090000075,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.5670000000100117
+ }
+ ]
+ },
+ {
+ "roundMs": 1769.0269580000022,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.010708000016165897
+ "name": "delete_item",
+ "execMs": 0.08008400001563132
},
{
- "name": "set_animation",
- "execMs": 0.0068749999918509275
+ "name": "delete_item",
+ "execMs": 0.012917000014567748
}
]
},
{
- "roundMs": 1421.0773749999935,
- "firstDeltaMs": 1127.4380419999943,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "work",
- "notes": [],
- "trace": [
- {
- "name": "set_transition",
- "round": 0,
- "atMs": 2611.4928749999963,
- "execMs": 0.004165999998804182
- },
- {
- "name": "set_animation",
- "round": 0,
- "atMs": 2611.501290999993,
- "execMs": 0.0032079999800771475
- }
- ],
- "timings": {
- "gateMs": 1127.3854580000043,
- "gateSkipped": false,
- "ttftMs": 3979.3432499999763,
- "modelMs": 3315.4693330000446,
- "toolMs": 0.007373999978881329,
- "totalMs": 4442.862165000028,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1483.910166000016,
+ "roundMs": 3698.278874999989,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.004165999998804182
+ "name": "delete_cue",
+ "execMs": 0.05208300001686439
},
{
- "name": "set_animation",
- "execMs": 0.0032079999800771475
+ "name": "delete_cue",
+ "execMs": 0.013957999995909631
}
]
},
{
- "roundMs": 1831.5591670000285,
- "firstDeltaMs": 1367.827917000017,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "filler-words-cut-with-editor",
- "bucket": "multi-tool",
- "passRate": 0.3333333333333333,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1941,
- "p95": 1941,
- "mean": 1941,
- "n": 1
- },
- "ttftMs": {
- "p50": 139632,
- "p95": 139632,
- "mean": 139632,
- "n": 1
- },
- "totalMs": {
- "p50": 19356,
- "p95": 19356,
- "mean": 19356,
- "n": 1
- },
- "modelMs": {
- "p50": 17414,
- "p95": 17414,
- "mean": 17414,
- "n": 1
- },
- "toolMs": {
- "p50": 1,
- "p95": 1,
- "mean": 1,
- "n": 1
- },
- "rounds": {
- "mean": 7,
- "max": 7
- }
- },
- "runs": [
- {
- "pass": false,
- "intent": "work",
- "notes": [
- "none of [split_at, delete_item, trim_clip] were called"
- ],
- "trace": [
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 4709.034624999971,
- "execMs": 0.1302909999503754
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 4709.170207999996,
- "execMs": 0.026167000003624707
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 4709.197374999989,
- "execMs": 0.013000000035390258
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 4709.2111249999725,
- "execMs": 0.004917000012937933
- }
- ],
- "timings": {
- "gateMs": 1848.3951250000391,
- "gateSkipped": false,
- "ttftMs": 5864.417665999965,
- "modelMs": 4306.871083000035,
- "toolMs": 0.1743750000023283,
- "totalMs": 6155.440583000076,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2860.0307080000057,
+ "roundMs": 1764.2055420000106,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.1302909999503754
+ "execMs": 0.07875000001513399
},
{
"name": "update_cue",
- "execMs": 0.026167000003624707
- },
- {
- "name": "delete_cue",
- "execMs": 0.013000000035390258
+ "execMs": 0.010500000003958121
},
{
"name": "update_cue",
- "execMs": 0.004917000012937933
+ "execMs": 0.007457999978214502
}
]
},
{
- "roundMs": 1446.8403750000289,
- "firstDeltaMs": 1155.1912909999955,
+ "roundMs": 1484.3025830000115,
+ "firstDeltaMs": 1020.0378330000094,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: remove_gap, remove_gap"
- ],
+ "notes": [],
"trace": [
{
"name": "split_at",
"round": 0,
- "atMs": 3124.453125,
- "execMs": 1.1742910000029951
+ "atMs": 2693.6664579999924,
+ "execMs": 0.2664999999979045
},
{
"name": "split_at",
"round": 0,
- "atMs": 3125.6399170000223,
- "execMs": 0.02441700000781566
+ "atMs": 2693.974542000011,
+ "execMs": 0.03541599999880418
},
{
"name": "split_at",
"round": 0,
- "atMs": 3125.665917000035,
- "execMs": 0.010833000007551163
+ "atMs": 2694.0368749999907,
+ "execMs": 0.017083999991882592
},
{
"name": "split_at",
"round": 0,
- "atMs": 3125.677708000003,
- "execMs": 0.010500000033061951
+ "atMs": 2694.057541999995,
+ "execMs": 0.02029200000106357
},
{
"name": "delete_item",
"round": 0,
- "atMs": 3125.6895830000285,
- "execMs": 0.2095829999889247
+ "atMs": 2694.0799169999955,
+ "execMs": 0.02341700001852587
+ },
+ {
+ "name": "get_state",
+ "round": 1,
+ "atMs": 3889.9535000000033,
+ "execMs": 0.04595900001004338
},
{
"name": "delete_item",
- "round": 0,
- "atMs": 3125.903083000041,
- "execMs": 0.009625000006053597
+ "round": 2,
+ "atMs": 5542.402916999999,
+ "execMs": 0.14004200001363643
},
{
- "name": "remove_gap",
- "round": 0,
- "atMs": 3125.9147499999963,
- "execMs": 0.06712500000139698
+ "name": "delete_item",
+ "round": 2,
+ "atMs": 5542.554082999995,
+ "execMs": 0.018958000000566244
},
{
- "name": "remove_gap",
- "round": 0,
- "atMs": 3125.983292000019,
- "execMs": 0.001500000013038516
+ "name": "delete_cue",
+ "round": 3,
+ "atMs": 6912.437000000005,
+ "execMs": 0.13883300000452437
+ },
+ {
+ "name": "delete_cue",
+ "round": 3,
+ "atMs": 6912.58808300001,
+ "execMs": 0.015333999996073544
}
],
"timings": {
- "gateMs": 1086.5155420000083,
+ "gateMs": 938.2515419999836,
"gateSkipped": false,
- "ttftMs": 4121.5769169999985,
- "modelMs": 3390.9299170000595,
- "toolMs": 1.5078740000608377,
- "totalMs": 4478.953333000129,
+ "ttftMs": 8102.156125000009,
+ "modelMs": 7452.272457999992,
+ "toolMs": 0.7218350000330247,
+ "totalMs": 8391.245835000009,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2037.5894170000101,
+ "roundMs": 1755.1447089999856,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 1.1742910000029951
+ "execMs": 0.2664999999979045
},
{
"name": "split_at",
- "execMs": 0.02441700000781566
+ "execMs": 0.03541599999880418
},
{
"name": "split_at",
- "execMs": 0.010833000007551163
+ "execMs": 0.017083999991882592
},
{
"name": "split_at",
- "execMs": 0.010500000033061951
+ "execMs": 0.02029200000106357
},
{
"name": "delete_item",
- "execMs": 0.2095829999889247
- },
+ "execMs": 0.02341700001852587
+ }
+ ]
+ },
+ {
+ "roundMs": 1195.7900419999787,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.04595900001004338
+ }
+ ]
+ },
+ {
+ "roundMs": 1652.3607910000137,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "delete_item",
- "execMs": 0.009625000006053597
+ "execMs": 0.14004200001363643
},
{
- "name": "remove_gap",
- "execMs": 0.06712500000139698
+ "name": "delete_item",
+ "execMs": 0.018958000000566244
+ }
+ ]
+ },
+ {
+ "roundMs": 1369.8024580000201,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "delete_cue",
+ "execMs": 0.13883300000452437
},
{
- "name": "remove_gap",
- "execMs": 0.001500000013038516
+ "name": "delete_cue",
+ "execMs": 0.015333999996073544
}
]
},
{
- "roundMs": 1353.3405000000494,
- "firstDeltaMs": 995.5709580000257,
+ "roundMs": 1479.174457999994,
+ "firstDeltaMs": 1189.5059579999943,
"calls": []
}
]
@@ -9415,183 +9062,135 @@
{
"name": "split_at",
"round": 0,
- "atMs": 127942.75891700003,
- "execMs": 0.0978330000070855
+ "atMs": 2619.8150000000314,
+ "execMs": 0.05754100001649931
},
{
"name": "split_at",
"round": 0,
- "atMs": 127942.87329200003,
- "execMs": 0.019333000003825873
+ "atMs": 2619.879749999993,
+ "execMs": 0.01900000002933666
},
{
"name": "split_at",
"round": 0,
- "atMs": 127942.904125,
- "execMs": 0.060999999986961484
+ "atMs": 2619.899875000032,
+ "execMs": 0.00375000003259629
},
{
"name": "split_at",
"round": 0,
- "atMs": 127942.96741700004,
- "execMs": 0.010249999992083758
+ "atMs": 2619.9041670000006,
+ "execMs": 0.0043329999898560345
},
{
"name": "delete_item",
- "round": 1,
- "atMs": 130003.78629200003,
- "execMs": 0.03841599996667355
+ "round": 0,
+ "atMs": 2619.9090419999848,
+ "execMs": 0.007083999982569367
},
{
- "name": "delete_item",
+ "name": "get_state",
"round": 1,
- "atMs": 130003.829959,
- "execMs": 0.0044169999891892076
+ "atMs": 4351.555292000034,
+ "execMs": 0.0735829999903217
},
{
- "name": "get_state",
+ "name": "delete_item",
"round": 2,
- "atMs": 131390.267209,
- "execMs": 0.20758299995213747
+ "atMs": 6448.605292000022,
+ "execMs": 0.04520799999590963
},
{
"name": "delete_item",
- "round": 3,
- "atMs": 133442.43425,
- "execMs": 0.08079199999338016
+ "round": 2,
+ "atMs": 6448.655584000022,
+ "execMs": 0.028542000043671578
},
{
"name": "delete_cue",
- "round": 4,
- "atMs": 135625.561375,
- "execMs": 0.024250000016763806
+ "round": 2,
+ "atMs": 6448.684749999986,
+ "execMs": 0.007999999972525984
},
{
"name": "delete_cue",
- "round": 4,
- "atMs": 135625.588,
- "execMs": 0.008167000021785498
- },
- {
- "name": "update_cue",
- "round": 5,
- "atMs": 138101.19133400003,
- "execMs": 0.09537499997531995
- },
- {
- "name": "update_cue",
- "round": 5,
- "atMs": 138101.295584,
- "execMs": 0.03529200004413724
- },
- {
- "name": "update_cue",
- "round": 5,
- "atMs": 138101.33258400002,
- "execMs": 0.01416699995752424
+ "round": 2,
+ "atMs": 6448.693042000028,
+ "execMs": 0.0024580000317655504
}
],
"timings": {
- "gateMs": 1941.3015000000014,
+ "gateMs": 1107.4632920000004,
"gateSkipped": false,
- "ttftMs": 139631.84025,
- "modelMs": 17414.17683099996,
- "toolMs": 0.6968749999068677,
- "totalMs": 19356.17520599987,
- "retryMs": 120873.644333,
- "retries": 1,
+ "ttftMs": 7968.713791999995,
+ "modelMs": 7382.378457999934,
+ "toolMs": 0.2494990000850521,
+ "totalMs": 8490.09124900002,
+ "retryMs": 0,
+ "retries": 0,
"rounds": [
{
- "roundMs": 5126.9930830000085,
+ "roundMs": 1512.1788750000123,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.0978330000070855
+ "execMs": 0.05754100001649931
},
{
"name": "split_at",
- "execMs": 0.019333000003825873
+ "execMs": 0.01900000002933666
},
{
"name": "split_at",
- "execMs": 0.060999999986961484
+ "execMs": 0.00375000003259629
},
{
"name": "split_at",
- "execMs": 0.010249999992083758
- }
- ]
- },
- {
- "roundMs": 2060.733666999964,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.03841599996667355
+ "execMs": 0.0043329999898560345
},
{
"name": "delete_item",
- "execMs": 0.0044169999891892076
+ "execMs": 0.007083999982569367
}
]
},
{
- "roundMs": 1386.4014999999781,
+ "roundMs": 1731.5947499999893,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.20758299995213747
+ "execMs": 0.0735829999903217
}
]
},
{
- "roundMs": 2051.9301660000347,
+ "roundMs": 2096.9080409999588,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.08079199999338016
- }
- ]
- },
- {
- "roundMs": 2183.006915999984,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_cue",
- "execMs": 0.024250000016763806
+ "execMs": 0.04520799999590963
},
{
- "name": "delete_cue",
- "execMs": 0.008167000021785498
- }
- ]
- },
- {
- "roundMs": 2475.506915999984,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.09537499997531995
+ "name": "delete_item",
+ "execMs": 0.028542000043671578
},
{
- "name": "update_cue",
- "execMs": 0.03529200004413724
+ "name": "delete_cue",
+ "execMs": 0.007999999972525984
},
{
- "name": "update_cue",
- "execMs": 0.01416699995752424
+ "name": "delete_cue",
+ "execMs": 0.0024580000317655504
}
]
},
{
- "roundMs": 2129.6045830000076,
- "firstDeltaMs": 1530.4494580000173,
+ "roundMs": 2041.6967919999734,
+ "firstDeltaMs": 1520.005999999994,
"calls": []
}
]
@@ -9606,27 +9205,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1222,
- "p95": 1263,
- "mean": 1234,
+ "p50": 1226,
+ "p95": 1270,
+ "mean": 1208,
"n": 3
},
"ttftMs": {
- "p50": 3718,
- "p95": 4279,
- "mean": 3902,
+ "p50": 3982,
+ "p95": 4627,
+ "mean": 4148,
"n": 3
},
"totalMs": {
- "p50": 4088,
- "p95": 5089,
- "mean": 4416,
+ "p50": 4354,
+ "p95": 4829,
+ "mean": 4510,
"n": 3
},
"modelMs": {
- "p50": 2871,
- "p95": 3866,
- "mean": 3182,
+ "p50": 3128,
+ "p95": 3699,
+ "mean": 3301,
"n": 3
},
"toolMs": {
@@ -9649,33 +9248,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 2738.052416999999,
- "execMs": 0.014499999990221113
+ "atMs": 3146.262000000046,
+ "execMs": 0.0025000000023283064
}
],
"timings": {
- "gateMs": 1222.2030000000377,
+ "gateMs": 1129.1259580000187,
"gateSkipped": false,
- "ttftMs": 4279.260500000033,
- "modelMs": 3866.45304199995,
- "toolMs": 0.014499999990221113,
- "totalMs": 5088.670541999978,
+ "ttftMs": 4626.675208000001,
+ "modelMs": 3699.4816259999643,
+ "toolMs": 0.0025000000023283064,
+ "totalMs": 4828.610083999985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1515.0747499999707,
+ "roundMs": 2016.396166999999,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.014499999990221113
+ "execMs": 0.0025000000023283064
}
]
},
{
- "roundMs": 2351.3782919999794,
- "firstDeltaMs": 1541.0854590000235,
+ "roundMs": 1683.0854589999653,
+ "firstDeltaMs": 1480.393333999964,
"calls": []
}
]
@@ -9689,33 +9288,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 2521.474291999999,
- "execMs": 0.006540999980643392
+ "atMs": 2586.470082999964,
+ "execMs": 0.005541999998968095
}
],
"timings": {
- "gateMs": 1216.2460420000134,
+ "gateMs": 1270.4610840000096,
"gateSkipped": false,
- "ttftMs": 3717.61920799996,
- "modelMs": 2871.4205830000574,
- "toolMs": 0.006540999980643392,
- "totalMs": 4087.6731660000514,
+ "ttftMs": 3982.256915999984,
+ "modelMs": 3075.5390829999815,
+ "toolMs": 0.005541999998968095,
+ "totalMs": 4346.00570899999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1305.0804580000113,
+ "roundMs": 1315.6999579999829,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.006540999980643392
+ "execMs": 0.005541999998968095
}
]
},
{
- "roundMs": 1566.340125000046,
- "firstDeltaMs": 1196.1032500000438,
+ "roundMs": 1759.8391249999986,
+ "firstDeltaMs": 1395.7600840000086,
"calls": []
}
]
@@ -9729,33 +9328,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 2702.161000000022,
- "execMs": 0.012667000002693385
+ "atMs": 2574.200542000006,
+ "execMs": 0.003666000033263117
}
],
"timings": {
- "gateMs": 1263.2502920000115,
+ "gateMs": 1225.7804999999935,
"gateSkipped": false,
- "ttftMs": 3709.422041999991,
- "modelMs": 2808.866207999992,
- "toolMs": 0.012667000002693385,
- "totalMs": 4072.1291670000064,
+ "ttftMs": 3833.9175830000313,
+ "modelMs": 3128.2572919999366,
+ "toolMs": 0.003666000033263117,
+ "totalMs": 4354.041457999963,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1438.6697919999715,
+ "roundMs": 1348.2962499999558,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.012667000002693385
+ "execMs": 0.003666000033263117
}
]
},
{
- "roundMs": 1370.1964160000207,
- "firstDeltaMs": 1007.2123750000028,
+ "roundMs": 1779.9610419999808,
+ "firstDeltaMs": 1259.6974170000176,
"calls": []
}
]
@@ -9770,27 +9369,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1275,
- "p95": 1710,
- "mean": 1380,
+ "p50": 1058,
+ "p95": 1133,
+ "mean": 1015,
"n": 3
},
"ttftMs": {
- "p50": 4223,
- "p95": 4715,
- "mean": 4305,
+ "p50": 3759,
+ "p95": 3800,
+ "mean": 3695,
"n": 3
},
"totalMs": {
- "p50": 4934,
- "p95": 4999,
- "mean": 4760,
+ "p50": 4197,
+ "p95": 4517,
+ "mean": 4191,
"n": 3
},
"modelMs": {
- "p50": 3288,
- "p95": 3658,
- "mean": 3381,
+ "p50": 3064,
+ "p95": 3459,
+ "mean": 3176,
"n": 3
},
"toolMs": {
@@ -9813,33 +9412,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 2800.2281250000233,
- "execMs": 0.47370800003409386
+ "atMs": 2407.7798750000075,
+ "execMs": 0.44795899995369837
}
],
"timings": {
- "gateMs": 1275.309459000011,
+ "gateMs": 1132.6619590000482,
"gateSkipped": false,
- "ttftMs": 4223.014250000007,
- "modelMs": 3657.950957999972,
- "toolMs": 0.47370800003409386,
- "totalMs": 4933.734125000017,
+ "ttftMs": 3799.8184170000022,
+ "modelMs": 3063.7203330000048,
+ "toolMs": 0.44795899995369837,
+ "totalMs": 4196.830251000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1524.282332999981,
+ "roundMs": 1274.5728329999838,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.47370800003409386
+ "execMs": 0.44795899995369837
}
]
},
{
- "roundMs": 2133.668624999991,
- "firstDeltaMs": 1422.2709580000374,
+ "roundMs": 1789.147500000021,
+ "firstDeltaMs": 1391.5482920000213,
"calls": []
}
]
@@ -9853,33 +9452,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 2735.6334999999963,
- "execMs": 0.022999999986495823
+ "atMs": 2388.5682919999817,
+ "execMs": 0.012208000000100583
}
],
"timings": {
- "gateMs": 1153.646666000015,
+ "gateMs": 853.3947920000064,
"gateSkipped": false,
- "ttftMs": 3977.8254580000066,
- "modelMs": 3195.305000000051,
- "toolMs": 0.022999999986495823,
- "totalMs": 4348.974666000053,
+ "ttftMs": 3525.601666999981,
+ "modelMs": 3006.267292000004,
+ "toolMs": 0.012208000000100583,
+ "totalMs": 3859.6742920000106,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1581.8688750000438,
+ "roundMs": 1534.9357500000042,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.022999999986495823
+ "execMs": 0.012208000000100583
}
]
},
{
- "roundMs": 1613.4361250000075,
- "firstDeltaMs": 1242.130833000003,
+ "roundMs": 1471.331542,
+ "firstDeltaMs": 1137.0009590000263,
"calls": []
}
]
@@ -9893,33 +9492,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 3578.6099169999943,
- "execMs": 0.027916999999433756
+ "atMs": 2278.6639999999898,
+ "execMs": 0.03162500000325963
}
],
"timings": {
- "gateMs": 1710.2793330000131,
+ "gateMs": 1058.1494170000078,
"gateSkipped": false,
- "ttftMs": 4715.430874999962,
- "modelMs": 3288.3803330000374,
- "toolMs": 0.027916999999433756,
- "totalMs": 4998.68758300005,
+ "ttftMs": 3759.379957999976,
+ "modelMs": 3459.31570799998,
+ "toolMs": 0.03162500000325963,
+ "totalMs": 4517.496749999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1868.044750000001,
+ "roundMs": 1220.2312500000116,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.027916999999433756
+ "execMs": 0.03162500000325963
}
]
},
{
- "roundMs": 1420.3355830000364,
- "firstDeltaMs": 1136.758833000029,
+ "roundMs": 2239.0844579999684,
+ "firstDeltaMs": 1480.6524999999674,
"calls": []
}
]
@@ -9934,32 +9533,32 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1228,
- "p95": 2092,
- "mean": 1466,
+ "p50": 1151,
+ "p95": 1230,
+ "mean": 1169,
"n": 3
},
"ttftMs": {
- "p50": 3923,
- "p95": 4965,
- "mean": 4244,
+ "p50": 3578,
+ "p95": 6704,
+ "mean": 4616,
"n": 3
},
"totalMs": {
- "p50": 4360,
- "p95": 5765,
- "mean": 4792,
+ "p50": 3872,
+ "p95": 6990,
+ "mean": 4893,
"n": 3
},
"modelMs": {
- "p50": 3173,
- "p95": 3673,
- "mean": 3326,
+ "p50": 2745,
+ "p95": 5839,
+ "mean": 3724,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
@@ -9977,33 +9576,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 2811.5718750000233,
- "execMs": 0.517416000016965
+ "atMs": 2452.62729199999,
+ "execMs": 0.3697080000420101
}
],
"timings": {
- "gateMs": 1077.4886249999981,
+ "gateMs": 1126.492750000034,
"gateSkipped": false,
- "ttftMs": 3845.3752920000115,
- "modelMs": 3172.8542080000043,
- "toolMs": 0.517416000016965,
- "totalMs": 4250.860249000019,
+ "ttftMs": 3578.487458000018,
+ "modelMs": 2744.7206659999792,
+ "toolMs": 0.3697080000420101,
+ "totalMs": 3871.5831240000552,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1733.9121670000022,
+ "roundMs": 1325.836707999988,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.517416000016965
+ "execMs": 0.3697080000420101
}
]
},
{
- "roundMs": 1438.942041000002,
- "firstDeltaMs": 1033.2462909999886,
+ "roundMs": 1418.8839579999913,
+ "firstDeltaMs": 1125.3990830000257,
"calls": []
}
]
@@ -10017,33 +9616,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 3485.4127089999965,
- "execMs": 0.021874999976716936
+ "atMs": 2540.9425829999964,
+ "execMs": 0.03516700002364814
}
],
"timings": {
- "gateMs": 2091.5800000000163,
+ "gateMs": 1229.783167000045,
"gateSkipped": false,
- "ttftMs": 4964.911500000046,
- "modelMs": 3673.3112909999327,
- "toolMs": 0.021874999976716936,
- "totalMs": 5764.913165999926,
+ "ttftMs": 3564.264082999958,
+ "modelMs": 2589.12166600005,
+ "toolMs": 0.03516700002364814,
+ "totalMs": 3818.9400000001187,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1393.5784579999745,
+ "roundMs": 1310.755416000029,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.021874999976716936
+ "execMs": 0.03516700002364814
}
]
},
{
- "roundMs": 2279.732832999958,
- "firstDeltaMs": 1479.4573749999981,
+ "roundMs": 1278.366250000021,
+ "firstDeltaMs": 1023.2098750000005,
"calls": []
}
]
@@ -10057,33 +9656,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 2750.2452500000363,
- "execMs": 0.02474999998230487
+ "atMs": 2579.7813750000205,
+ "execMs": 0.02558299998054281
}
],
"timings": {
- "gateMs": 1227.678957999975,
+ "gateMs": 1151.0154169999878,
"gateSkipped": false,
- "ttftMs": 3922.9837919999845,
- "modelMs": 3132.792084000015,
- "toolMs": 0.02474999998230487,
- "totalMs": 4360.495791999972,
+ "ttftMs": 6704.3187920000055,
+ "modelMs": 5838.878582999983,
+ "toolMs": 0.02558299998054281,
+ "totalMs": 6989.919582999952,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1522.4153340000194,
+ "roundMs": 1428.5572919999831,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.02474999998230487
+ "execMs": 0.02558299998054281
}
]
},
{
- "roundMs": 1610.3767499999958,
- "firstDeltaMs": 1172.6839170000167,
+ "roundMs": 4410.321291,
+ "firstDeltaMs": 4124.470541000017,
"calls": []
}
]
@@ -10104,21 +9703,21 @@
"n": 3
},
"ttftMs": {
- "p50": 3506,
- "p95": 5245,
- "mean": 4039,
+ "p50": 2623,
+ "p95": 2757,
+ "mean": 2666,
"n": 3
},
"totalMs": {
- "p50": 3922,
- "p95": 5594,
- "mean": 4396,
+ "p50": 3036,
+ "p95": 3149,
+ "mean": 3066,
"n": 3
},
"modelMs": {
- "p50": 3922,
- "p95": 5593,
- "mean": 4396,
+ "p50": 3036,
+ "p95": 3149,
+ "mean": 3066,
"n": 3
},
"toolMs": {
@@ -10141,33 +9740,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 1540.5227500000037,
- "execMs": 0.07766700000502169
+ "atMs": 1579.401332999987,
+ "execMs": 0.2748330000322312
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5244.572667,
- "modelMs": 5593.429791999981,
- "toolMs": 0.07766700000502169,
- "totalMs": 5593.507458999986,
+ "ttftMs": 2757.4247910000267,
+ "modelMs": 3148.8273749999935,
+ "toolMs": 0.2748330000322312,
+ "totalMs": 3149.1022080000257,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1539.6792499999865,
+ "roundMs": 1578.8140830000048,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.07766700000502169
+ "execMs": 0.2748330000322312
}
]
},
{
- "roundMs": 4053.7505419999943,
- "firstDeltaMs": 3703.9650830000173,
+ "roundMs": 1570.0132919999887,
+ "firstDeltaMs": 1177.7047089999542,
"calls": []
}
]
@@ -10181,33 +9780,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 2293.839042000007,
- "execMs": 0.0324589999509044
+ "atMs": 1429.995791000023,
+ "execMs": 0.019582999986596406
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3505.9300839999923,
- "modelMs": 3921.7987079999875,
- "toolMs": 0.0324589999509044,
- "totalMs": 3921.8311669999384,
+ "ttftMs": 2617.7266250000102,
+ "modelMs": 3036.2304589999258,
+ "toolMs": 0.019582999986596406,
+ "totalMs": 3036.2500419999124,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2293.7691670000204,
+ "roundMs": 1429.830666999973,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.0324589999509044
+ "execMs": 0.019582999986596406
}
]
},
{
- "roundMs": 1628.0295409999671,
- "firstDeltaMs": 1212.0424580000108,
+ "roundMs": 1606.3997919999529,
+ "firstDeltaMs": 1187.6895420000073,
"calls": []
}
]
@@ -10221,33 +9820,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 2078.6249170000083,
- "execMs": 0.017583000008016825
+ "atMs": 1504.8534169999766,
+ "execMs": 0.03237500000977889
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3365.9664169999887,
- "modelMs": 3672.673373999947,
- "toolMs": 0.017583000008016825,
- "totalMs": 3672.6909569999552,
+ "ttftMs": 2623.2734579999815,
+ "modelMs": 3012.5639159999555,
+ "toolMs": 0.03237500000977889,
+ "totalMs": 3012.5962909999653,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2078.5644159999792,
+ "roundMs": 1504.6570409999695,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.017583000008016825
+ "execMs": 0.03237500000977889
}
]
},
{
- "roundMs": 1594.108957999968,
- "firstDeltaMs": 1287.309957999969,
+ "roundMs": 1507.906874999986,
+ "firstDeltaMs": 1118.3301670000073,
"calls": []
}
]
@@ -10262,27 +9861,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1154,
- "p95": 1222,
- "mean": 1166,
+ "p50": 1625,
+ "p95": 2027,
+ "mean": 1641,
"n": 3
},
"ttftMs": {
- "p50": 2592,
- "p95": 2651,
- "mean": 2603,
+ "p50": 2611,
+ "p95": 3212,
+ "mean": 2743,
"n": 3
},
"totalMs": {
- "p50": 3479,
- "p95": 3792,
- "mean": 3520,
+ "p50": 3463,
+ "p95": 4036,
+ "mean": 3519,
"n": 3
},
"modelMs": {
- "p50": 2357,
- "p95": 2639,
- "mean": 2354,
+ "p50": 1838,
+ "p95": 2009,
+ "mean": 1878,
"n": 3
},
"toolMs": {
@@ -10303,18 +9902,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1121.4748329999857,
+ "gateMs": 2027.432582999987,
"gateSkipped": false,
- "ttftMs": 2565.4744580000406,
- "modelMs": 2357.466000000015,
+ "ttftMs": 3212.3651660000323,
+ "modelMs": 2008.8742499999935,
"toolMs": 0,
- "totalMs": 3478.9408330000006,
+ "totalMs": 4036.3068329999805,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2357.466000000015,
- "firstDeltaMs": 1443.8289999999688,
+ "roundMs": 2008.8742499999935,
+ "firstDeltaMs": 1184.3671249999898,
"calls": []
}
]
@@ -10326,18 +9925,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1222.1102079999982,
+ "gateMs": 1270.2302080000518,
"gateSkipped": false,
- "ttftMs": 2650.661000000022,
- "modelMs": 2065.5825830000103,
+ "ttftMs": 2404.8939170000376,
+ "modelMs": 1788.33391700004,
"toolMs": 0,
- "totalMs": 3287.6927910000086,
+ "totalMs": 3058.5641250000917,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2065.5825830000103,
- "firstDeltaMs": 1428.3015420000302,
+ "roundMs": 1788.33391700004,
+ "firstDeltaMs": 1134.5162080000155,
"calls": []
}
]
@@ -10349,18 +9948,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1153.676374999981,
+ "gateMs": 1625.1815829999978,
"gateSkipped": false,
- "ttftMs": 2591.906583999982,
- "modelMs": 2638.724583000003,
+ "ttftMs": 2611.471375000023,
+ "modelMs": 1838.106415999995,
"toolMs": 0,
- "totalMs": 3792.400957999984,
+ "totalMs": 3463.287998999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2638.724583000003,
- "firstDeltaMs": 1438.1468749999767,
+ "roundMs": 1838.106415999995,
+ "firstDeltaMs": 986.1514579999493,
"calls": []
}
]
@@ -10381,21 +9980,21 @@
"n": 3
},
"ttftMs": {
- "p50": 7335,
- "p95": 8556,
- "mean": 7152,
+ "p50": 7357,
+ "p95": 10141,
+ "mean": 7931,
"n": 3
},
"totalMs": {
- "p50": 8106,
- "p95": 8909,
- "mean": 7807,
+ "p50": 8132,
+ "p95": 10646,
+ "mean": 8618,
"n": 3
},
"modelMs": {
- "p50": 8106,
- "p95": 8909,
- "mean": 7807,
+ "p50": 8132,
+ "p95": 10645,
+ "mean": 8618,
"n": 3
},
"toolMs": {
@@ -10405,8 +10004,8 @@
"n": 3
},
"rounds": {
- "mean": 4.3,
- "max": 5
+ "mean": 5,
+ "max": 6
}
},
"runs": [
@@ -10416,81 +10015,99 @@
"notes": [],
"trace": [
{
- "name": "watch_video",
+ "name": "listen_audio",
"round": 0,
- "atMs": 1742.0103750000126,
- "execMs": 0.1292919999687001
+ "atMs": 1637.2396250000456,
+ "execMs": 0.26524999999674037
},
{
"name": "watch_video",
- "round": 0,
- "atMs": 1742.1444999999949,
- "execMs": 0.0025000000023283064
+ "round": 1,
+ "atMs": 4417.872250000015,
+ "execMs": 0.040250000020023435
},
{
- "name": "listen_audio",
- "round": 0,
- "atMs": 1742.1486250000307,
- "execMs": 0.00033300003269687295
+ "name": "watch_video",
+ "round": 2,
+ "atMs": 5928.738750000019,
+ "execMs": 0.019458999973721802
},
{
"name": "regenerate_shot",
- "round": 1,
- "atMs": 4036.195417000039,
- "execMs": 0.05766699998639524
+ "round": 3,
+ "atMs": 7538.1191250000265,
+ "execMs": 0.09020899998722598
},
{
"name": "regenerate_shot",
- "round": 1,
- "atMs": 4036.2608750000363,
- "execMs": 0.009750000026542693
+ "round": 4,
+ "atMs": 9085.19425,
+ "execMs": 0.05100000003585592
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5564.469917000039,
- "modelMs": 6405.381166000036,
- "toolMs": 0.19954200001666322,
- "totalMs": 6405.580708000052,
+ "ttftMs": 10141.050083000038,
+ "modelMs": 10645.140709000058,
+ "toolMs": 0.4661680000135675,
+ "totalMs": 10645.606877000071,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1741.4082920000073,
+ "roundMs": 1636.9305000000168,
"firstDeltaMs": null,
"calls": [
{
- "name": "watch_video",
- "execMs": 0.1292919999687001
- },
+ "name": "listen_audio",
+ "execMs": 0.26524999999674037
+ }
+ ]
+ },
+ {
+ "roundMs": 2780.303083000006,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "watch_video",
- "execMs": 0.0025000000023283064
- },
+ "execMs": 0.040250000020023435
+ }
+ ]
+ },
+ {
+ "roundMs": 1510.7522089999984,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "listen_audio",
- "execMs": 0.00033300003269687295
+ "name": "watch_video",
+ "execMs": 0.019458999973721802
}
]
},
{
- "roundMs": 2294.0134160000016,
+ "roundMs": 1609.2836249999818,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.05766699998639524
- },
+ "execMs": 0.09020899998722598
+ }
+ ]
+ },
+ {
+ "roundMs": 1546.7366250000196,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "regenerate_shot",
- "execMs": 0.009750000026542693
+ "execMs": 0.05100000003585592
}
]
},
{
- "roundMs": 2369.9594580000266,
- "firstDeltaMs": 1528.1773750000284,
+ "roundMs": 1561.1346670000348,
+ "firstDeltaMs": 1055.7701670000097,
"calls": []
}
]
@@ -10504,91 +10121,85 @@
{
"name": "listen_audio",
"round": 0,
- "atMs": 1356.0478749999893,
- "execMs": 0.030666999984532595
+ "atMs": 1452.551999999967,
+ "execMs": 0.05729199998313561
},
{
"name": "watch_video",
"round": 1,
- "atMs": 3478.067000000039,
- "execMs": 0.006209000013768673
+ "atMs": 3020.791582999984,
+ "execMs": 0.03345899999840185
},
{
"name": "watch_video",
- "round": 2,
- "atMs": 4899.847999999998,
- "execMs": 0.024374999979045242
+ "round": 1,
+ "atMs": 3020.8743749999558,
+ "execMs": 0.003999999957159162
},
{
"name": "regenerate_shot",
- "round": 3,
- "atMs": 7243.12395800004,
- "execMs": 0.016833000001497567
+ "round": 2,
+ "atMs": 4724.119832999946,
+ "execMs": 0.07166700001107529
},
{
"name": "regenerate_shot",
- "round": 3,
- "atMs": 7243.1482500000275,
- "execMs": 0.0009589999681338668
+ "round": 2,
+ "atMs": 4724.2021249999525,
+ "execMs": 0.01324999995995313
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 8555.538082999992,
- "modelMs": 8908.51354200003,
- "toolMs": 0.07904299994697794,
- "totalMs": 8908.592584999977,
+ "ttftMs": 6295.329749999975,
+ "modelMs": 7075.407583999971,
+ "toolMs": 0.17966799990972504,
+ "totalMs": 7075.587251999881,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1355.626667000004,
+ "roundMs": 1452.367333000002,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.030666999984532595
+ "execMs": 0.05729199998313561
}
]
},
{
- "roundMs": 2121.897667000012,
+ "roundMs": 1568.027624999988,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.006209000013768673
- }
- ]
- },
- {
- "roundMs": 1421.7260000000242,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.03345899999840185
+ },
{
"name": "watch_video",
- "execMs": 0.024374999979045242
+ "execMs": 0.003999999957159162
}
]
},
{
- "roundMs": 2343.1867920000223,
+ "roundMs": 1703.1698340000003,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.016833000001497567
+ "execMs": 0.07166700001107529
},
{
"name": "regenerate_shot",
- "execMs": 0.0009589999681338668
+ "execMs": 0.01324999995995313
}
]
},
{
- "roundMs": 1666.0764159999671,
- "firstDeltaMs": 1312.3645829999587,
+ "roundMs": 2351.8427919999813,
+ "firstDeltaMs": 1571.0677920000162,
"calls": []
}
]
@@ -10602,91 +10213,91 @@
{
"name": "listen_audio",
"round": 0,
- "atMs": 1350.0536250000005,
- "execMs": 0.09116700000595301
+ "atMs": 1382.5452499999665,
+ "execMs": 0.028833999997004867
},
{
"name": "watch_video",
"round": 1,
- "atMs": 2742.7724160000216,
- "execMs": 0.011499999964144081
+ "atMs": 2885.655207999982,
+ "execMs": 0.01945799996610731
},
{
"name": "watch_video",
"round": 2,
- "atMs": 4134.867957999988,
- "execMs": 0.02312500000698492
+ "atMs": 4155.999415999977,
+ "execMs": 0.02016600000206381
},
{
"name": "regenerate_shot",
"round": 3,
- "atMs": 6012.662041000032,
- "execMs": 0.011832999996840954
+ "atMs": 5784.7925410000025,
+ "execMs": 0.02916699997149408
},
{
"name": "regenerate_shot",
"round": 3,
- "atMs": 6012.676375000039,
- "execMs": 0.002957999997306615
+ "atMs": 5784.830499999982,
+ "execMs": 0.01158299995586276
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 7334.541833000025,
- "modelMs": 8105.932832000079,
- "toolMs": 0.14058299997122958,
- "totalMs": 8106.07341500005,
+ "ttftMs": 7357.489165999985,
+ "modelMs": 8132.320333999989,
+ "toolMs": 0.10920799989253283,
+ "totalMs": 8132.429541999882,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1349.5902920000372,
+ "roundMs": 1382.1439169999794,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.09116700000595301
+ "execMs": 0.028833999997004867
}
]
},
{
- "roundMs": 1392.555541000038,
+ "roundMs": 1503.0383750000037,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.011499999964144081
+ "execMs": 0.01945799996610731
}
]
},
{
- "roundMs": 1391.9615829999675,
+ "roundMs": 1270.060959000024,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.02312500000698492
+ "execMs": 0.02016600000206381
}
]
},
{
- "roundMs": 1877.7025410000351,
+ "roundMs": 1628.7085829999996,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.011832999996840954
+ "execMs": 0.02916699997149408
},
{
"name": "regenerate_shot",
- "execMs": 0.002957999997306615
+ "execMs": 0.01158299995586276
}
]
},
{
- "roundMs": 2094.122875000001,
- "firstDeltaMs": 1321.8512089999858,
+ "roundMs": 2348.3684999999823,
+ "firstDeltaMs": 1572.6222499999567,
"calls": []
}
]
@@ -10703,70 +10314,70 @@
"buckets": {
"chat": {
"cases": 7,
- "passRate": 0.95,
+ "passRate": 1,
"gateMs": {
- "p50": 1941,
- "p95": 2620,
- "mean": 1558,
+ "p50": 1830,
+ "p95": 2967,
+ "mean": 1481,
"n": 7
},
"ttftMs": {
- "p50": 4371,
- "p95": 9547,
- "mean": 5350,
+ "p50": 4435,
+ "p95": 10574,
+ "mean": 5792,
"n": 7
},
"totalMs": {
- "p50": 4646,
- "p95": 10087,
- "mean": 5732,
+ "p50": 5437,
+ "p95": 11469,
+ "mean": 6298,
"n": 7
},
- "meanRounds": 1.1
+ "meanRounds": 1
},
"multi-tool": {
"cases": 4,
- "passRate": 0.5,
+ "passRate": 0.92,
"gateMs": {
"p50": 0,
- "p95": 2418,
- "mean": 1209,
- "n": 2
+ "p95": 2565,
+ "mean": 1114,
+ "n": 4
},
"ttftMs": {
- "p50": 10428,
- "p95": 16714,
- "mean": 13571,
- "n": 2
+ "p50": 17631,
+ "p95": 71405,
+ "mean": 30075,
+ "n": 4
},
"totalMs": {
- "p50": 10758,
- "p95": 17076,
- "mean": 13917,
- "n": 2
+ "p50": 18028,
+ "p95": 72780,
+ "mean": 30742,
+ "n": 4
},
- "meanRounds": 3
+ "meanRounds": 8.3
},
"single-tool": {
"cases": 15,
- "passRate": 0.78,
+ "passRate": 0.96,
"gateMs": {
- "p50": 1711,
- "p95": 2198,
- "mean": 1640,
- "n": 12
+ "p50": 1759,
+ "p95": 2052,
+ "mean": 1642,
+ "n": 15
},
"ttftMs": {
- "p50": 8126,
- "p95": 10902,
- "mean": 8236,
- "n": 12
+ "p50": 8781,
+ "p95": 12213,
+ "mean": 8600,
+ "n": 15
},
"totalMs": {
- "p50": 8516,
- "p95": 11150,
- "mean": 8590,
- "n": 12
+ "p50": 9118,
+ "p95": 12587,
+ "mean": 8965,
+ "n": 15
},
"meanRounds": 2.1
}
@@ -10785,21 +10396,21 @@
"n": 3
},
"ttftMs": {
- "p50": 3666,
- "p95": 4744,
- "mean": 3947,
+ "p50": 7164,
+ "p95": 8230,
+ "mean": 6534,
"n": 3
},
"totalMs": {
- "p50": 3958,
- "p95": 4988,
- "mean": 4218,
+ "p50": 7446,
+ "p95": 8309,
+ "mean": 6767,
"n": 3
},
"modelMs": {
- "p50": 3958,
- "p95": 4988,
- "mean": 4218,
+ "p50": 7446,
+ "p95": 8309,
+ "mean": 6767,
"n": 3
},
"toolMs": {
@@ -10809,8 +10420,8 @@
"n": 3
},
"rounds": {
- "mean": 1,
- "max": 1
+ "mean": 1.3,
+ "max": 2
}
},
"runs": [
@@ -10822,16 +10433,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4744.343333,
- "modelMs": 4987.743334,
+ "ttftMs": 8230.176541,
+ "modelMs": 8308.520875,
"toolMs": 0,
- "totalMs": 4987.743334,
+ "totalMs": 8308.520875,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4987.743334,
- "firstDeltaMs": 4743.9743339999995,
+ "roundMs": 8308.520875,
+ "firstDeltaMs": 8229.798916,
"calls": []
}
]
@@ -10841,20 +10452,37 @@
"pass": true,
"intent": "work",
"notes": [],
- "trace": [],
+ "trace": [
+ {
+ "name": "listen_audio",
+ "round": 0,
+ "atMs": 4499.476042,
+ "execMs": 0.2617920000011509
+ }
+ ],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3431.793375,
- "modelMs": 3708.2671250000003,
- "toolMs": 0,
- "totalMs": 3708.2671250000003,
+ "ttftMs": 7163.535667,
+ "modelMs": 7445.9930840000015,
+ "toolMs": 0.2617920000011509,
+ "totalMs": 7446.254876000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3708.2671250000003,
- "firstDeltaMs": 3431.758834,
+ "roundMs": 4499.228167000001,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "listen_audio",
+ "execMs": 0.2617920000011509
+ }
+ ]
+ },
+ {
+ "roundMs": 2946.7649170000004,
+ "firstDeltaMs": 2663.7235,
"calls": []
}
]
@@ -10868,16 +10496,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 3666.0775840000006,
- "modelMs": 3957.879083,
+ "ttftMs": 4208.736457999999,
+ "modelMs": 4546.944083000002,
"toolMs": 0,
- "totalMs": 3957.879083,
+ "totalMs": 4546.944083000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3957.879083,
- "firstDeltaMs": 3666.0268750000014,
+ "roundMs": 4546.944083000002,
+ "firstDeltaMs": 4208.119167000001,
"calls": []
}
]
@@ -10888,38 +10516,38 @@
{
"name": "greeting-is-gated",
"bucket": "chat",
- "passRate": 0.6666666666666666,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1654,
- "p95": 2390,
- "mean": 2022,
- "n": 2
+ "p50": 1830,
+ "p95": 1931,
+ "mean": 1773,
+ "n": 3
},
"ttftMs": {
- "p50": 3379,
- "p95": 4434,
- "mean": 3906,
- "n": 2
+ "p50": 3406,
+ "p95": 3589,
+ "mean": 3391,
+ "n": 3
},
"totalMs": {
- "p50": 3553,
- "p95": 4454,
- "mean": 4003,
- "n": 2
+ "p50": 3543,
+ "p95": 4017,
+ "mean": 3663,
+ "n": 3
},
"modelMs": {
- "p50": 1899,
- "p95": 2063,
- "mean": 1981,
- "n": 2
+ "p50": 1871,
+ "p95": 2086,
+ "mean": 1890,
+ "n": 3
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 3
},
"rounds": {
"mean": 1,
@@ -10928,25 +10556,23 @@
},
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "chat",
- "notes": [
- "reply did not match /help|what would you/i"
- ],
+ "notes": [],
"trace": [],
"timings": {
- "gateMs": 1748.0051249999997,
+ "gateMs": 1558.810540999999,
"gateSkipped": false,
- "ttftMs": 3658.778124999999,
- "modelMs": 2131.064583999998,
+ "ttftMs": 3176.8667910000004,
+ "modelMs": 1870.7128329999978,
"toolMs": 0,
- "totalMs": 3879.0697089999976,
+ "totalMs": 3429.5233739999967,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2131.064583999998,
- "firstDeltaMs": 1909.439124999999,
+ "roundMs": 1870.7128329999978,
+ "firstDeltaMs": 1617.9331669999992,
"calls": []
}
]
@@ -10958,18 +10584,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2390.3030830000025,
+ "gateMs": 1830.4391670000005,
"gateSkipped": false,
- "ttftMs": 4433.934041,
- "modelMs": 2063.2042089999995,
+ "ttftMs": 3406.016707999999,
+ "modelMs": 1712.287292000001,
"toolMs": 0,
- "totalMs": 4453.507292000002,
+ "totalMs": 3542.7264590000013,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2063.2042089999995,
- "firstDeltaMs": 2043.5508340000015,
+ "roundMs": 1712.287292000001,
+ "firstDeltaMs": 1575.3843749999978,
"calls": []
}
]
@@ -10981,18 +10607,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1653.9071250000015,
+ "gateMs": 1930.9875839999986,
"gateSkipped": false,
- "ttftMs": 3378.6929999999993,
- "modelMs": 1898.8390839999993,
+ "ttftMs": 3589.4067080000023,
+ "modelMs": 2086.4661669999987,
"toolMs": 0,
- "totalMs": 3552.746209000001,
+ "totalMs": 4017.4537509999973,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1898.8390839999993,
- "firstDeltaMs": 1724.6963340000002,
+ "roundMs": 2086.4661669999987,
+ "firstDeltaMs": 1658.2551669999993,
"calls": []
}
]
@@ -11007,27 +10633,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 2294,
- "p95": 2775,
- "mean": 2446,
+ "p50": 1903,
+ "p95": 2018,
+ "mean": 1873,
"n": 3
},
"ttftMs": {
- "p50": 4371,
- "p95": 5000,
- "mean": 4438,
+ "p50": 3613,
+ "p95": 3700,
+ "mean": 3550,
"n": 3
},
"totalMs": {
- "p50": 4444,
- "p95": 5201,
- "mean": 4654,
+ "p50": 3749,
+ "p95": 3849,
+ "mean": 3776,
"n": 3
},
"modelMs": {
- "p50": 2151,
- "p95": 2427,
- "mean": 2208,
+ "p50": 1846,
+ "p95": 2032,
+ "mean": 1903,
"n": 3
},
"toolMs": {
@@ -11048,18 +10674,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2293.5802919999987,
+ "gateMs": 1697.9578330000004,
"gateSkipped": false,
- "ttftMs": 4371.015457999998,
- "modelMs": 2150.84,
+ "ttftMs": 3334.9988749999975,
+ "modelMs": 2032.4882080000025,
"toolMs": 0,
- "totalMs": 4444.420291999999,
+ "totalMs": 3730.446041000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2150.84,
- "firstDeltaMs": 2077.3201660000013,
+ "roundMs": 2032.4882080000025,
+ "firstDeltaMs": 1636.8671660000036,
"calls": []
}
]
@@ -11071,18 +10697,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2270.123707999999,
+ "gateMs": 1902.8229999999967,
"gateSkipped": false,
- "ttftMs": 3942.6062079999974,
- "modelMs": 2045.7784999999967,
+ "ttftMs": 3613.298167000001,
+ "modelMs": 1846.0233329999974,
"toolMs": 0,
- "totalMs": 4315.902207999996,
+ "totalMs": 3748.846332999994,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2045.7784999999967,
- "firstDeltaMs": 1672.4342920000017,
+ "roundMs": 1846.0233329999974,
+ "firstDeltaMs": 1710.1575000000012,
"calls": []
}
]
@@ -11094,18 +10720,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2774.6500000000015,
+ "gateMs": 2018.3472500000062,
"gateSkipped": false,
- "ttftMs": 5000.252667000001,
- "modelMs": 2426.747083000002,
+ "ttftMs": 3700.368457999997,
+ "modelMs": 1831.0479160000032,
"toolMs": 0,
- "totalMs": 5201.3970830000035,
+ "totalMs": 3849.3951660000093,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2426.747083000002,
- "firstDeltaMs": 2225.3144170000014,
+ "roundMs": 1831.0479160000032,
+ "firstDeltaMs": 1681.7177080000038,
"calls": []
}
]
@@ -11120,38 +10746,38 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 2418,
- "p95": 2657,
- "mean": 2326,
+ "p50": 2565,
+ "p95": 2739,
+ "mean": 2488,
"n": 3
},
"ttftMs": {
- "p50": 10428,
- "p95": 25765,
- "mean": 14961,
+ "p50": 9982,
+ "p95": 15299,
+ "mean": 11698,
"n": 3
},
"totalMs": {
- "p50": 10758,
- "p95": 26018,
- "mean": 15285,
+ "p50": 10338,
+ "p95": 15770,
+ "mean": 12125,
"n": 3
},
"modelMs": {
- "p50": 8101,
- "p95": 24112,
- "mean": 12958,
+ "p50": 8107,
+ "p95": 13031,
+ "mean": 9637,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 2,
- "mean": 1,
+ "p95": 1,
+ "mean": 0,
"n": 3
},
"rounds": {
- "mean": 2.7,
- "max": 4
+ "mean": 2,
+ "max": 2
}
},
"runs": [
@@ -11163,75 +10789,43 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 14759.699125,
- "execMs": 0.620375000005879
+ "atMs": 8399.787125000003,
+ "execMs": 1.1936669999995502
},
{
"name": "update_cue",
"round": 0,
- "atMs": 14760.336166999994,
- "execMs": 0.006667000001471024
- },
- {
- "name": "merge_cue",
- "round": 1,
- "atMs": 20822.619957999996,
- "execMs": 0.4822500000009313
- },
- {
- "name": "get_state",
- "round": 2,
- "atMs": 22993.945874999998,
- "execMs": 0.8225420000017039
+ "atMs": 8401.019708,
+ "execMs": 0.015124999998079147
}
],
"timings": {
- "gateMs": 1903.8512499999997,
+ "gateMs": 2158.7650839999988,
"gateSkipped": false,
- "ttftMs": 25765.089249999997,
- "modelMs": 24112.23275000001,
- "toolMs": 1.9318340000099852,
- "totalMs": 26018.01583400002,
+ "ttftMs": 9982.407625,
+ "modelMs": 8107.402499000003,
+ "toolMs": 1.2087919999976293,
+ "totalMs": 10267.376375,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 12854.415375000004,
+ "roundMs": 6239.549458000001,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.620375000005879
+ "execMs": 1.1936669999995502
},
{
"name": "update_cue",
- "execMs": 0.006667000001471024
- }
- ]
- },
- {
- "roundMs": 6062.187209000003,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "merge_cue",
- "execMs": 0.4822500000009313
- }
- ]
- },
- {
- "roundMs": 2170.7620830000014,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.8225420000017039
+ "execMs": 0.015124999998079147
}
]
},
{
- "roundMs": 3024.868083000001,
- "firstDeltaMs": 2770.258625000002,
+ "roundMs": 1867.8530410000021,
+ "firstDeltaMs": 1581.3340830000016,
"calls": []
}
]
@@ -11245,43 +10839,43 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 8383.66816600001,
- "execMs": 0.009250000002793968
+ "atMs": 10290.023834,
+ "execMs": 0.027792000000772532
},
{
"name": "update_cue",
"round": 0,
- "atMs": 8383.67983300001,
- "execMs": 0.0037499999889405444
+ "atMs": 10290.057667000001,
+ "execMs": 0.004791999999724794
}
],
"timings": {
- "gateMs": 2657.1563329999917,
+ "gateMs": 2738.764624999996,
"gateSkipped": false,
- "ttftMs": 10427.855541000004,
- "modelMs": 8100.864082999993,
- "toolMs": 0.012999999991734512,
- "totalMs": 10758.033415999977,
+ "ttftMs": 15299.170958999995,
+ "modelMs": 13030.80225,
+ "toolMs": 0.032584000000497326,
+ "totalMs": 15769.599458999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5726.312875000003,
+ "roundMs": 7551.0825419999965,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.009250000002793968
+ "execMs": 0.027792000000772532
},
{
"name": "update_cue",
- "execMs": 0.0037499999889405444
+ "execMs": 0.004791999999724794
}
]
},
{
- "roundMs": 2374.55120799999,
- "firstDeltaMs": 2044.162249999994,
+ "roundMs": 5479.719708000004,
+ "firstDeltaMs": 5009.085875000004,
"calls": []
}
]
@@ -11295,43 +10889,43 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 7240.606665999992,
- "execMs": 0.014792000001762062
+ "atMs": 8100.620833000008,
+ "execMs": 0.09054200000537094
},
{
"name": "update_cue",
"round": 0,
- "atMs": 7240.627540999994,
- "execMs": 0.004666000008000992
+ "atMs": 8100.72866600001,
+ "execMs": 0.005791999996290542
}
],
"timings": {
- "gateMs": 2417.678832999998,
+ "gateMs": 2565.289875000002,
"gateSkipped": false,
- "ttftMs": 8690.902749999994,
- "modelMs": 6661.837458000009,
- "toolMs": 0.019458000009763055,
- "totalMs": 9079.535749000017,
+ "ttftMs": 9812.183458,
+ "modelMs": 7772.530624000006,
+ "toolMs": 0.09633400000166148,
+ "totalMs": 10337.91683300001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4822.3369999999995,
+ "roundMs": 5534.760332999998,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.014792000001762062
+ "execMs": 0.09054200000537094
},
{
"name": "update_cue",
- "execMs": 0.004666000008000992
+ "execMs": 0.005791999996290542
}
]
},
{
- "roundMs": 1839.5004580000095,
- "firstDeltaMs": 1450.2505000000092,
+ "roundMs": 2237.770291000008,
+ "firstDeltaMs": 1711.3884580000013,
"calls": []
}
]
@@ -11346,27 +10940,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1941,
- "p95": 2192,
- "mean": 2019,
+ "p50": 1779,
+ "p95": 2124,
+ "mean": 1854,
"n": 3
},
"ttftMs": {
- "p50": 4244,
- "p95": 4807,
- "mean": 4352,
+ "p50": 4435,
+ "p95": 4498,
+ "mean": 4243,
"n": 3
},
"totalMs": {
- "p50": 4646,
- "p95": 5048,
- "mean": 4679,
+ "p50": 4699,
+ "p95": 4715,
+ "mean": 4479,
"n": 3
},
"modelMs": {
- "p50": 2705,
- "p95": 2857,
- "mean": 2661,
+ "p50": 2591,
+ "p95": 2921,
+ "mean": 2625,
"n": 3
},
"toolMs": {
@@ -11387,18 +10981,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2191.587167000005,
+ "gateMs": 2124.4604170000093,
"gateSkipped": false,
- "ttftMs": 4806.903041000012,
- "modelMs": 2856.6628330000094,
+ "ttftMs": 4498.413459000003,
+ "modelMs": 2590.600874999989,
"toolMs": 0,
- "totalMs": 5048.250000000015,
+ "totalMs": 4715.0612919999985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2856.6628330000094,
- "firstDeltaMs": 2615.013750000013,
+ "roundMs": 2590.600874999989,
+ "firstDeltaMs": 2373.797458000001,
"calls": []
}
]
@@ -11410,18 +11004,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1940.697874999998,
+ "gateMs": 1658.2093750000058,
"gateSkipped": false,
- "ttftMs": 4244.376999999993,
- "modelMs": 2705.1033749999915,
+ "ttftMs": 3793.8133750000125,
+ "modelMs": 2364.2338750000054,
"toolMs": 0,
- "totalMs": 4645.8012499999895,
+ "totalMs": 4022.443250000011,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2705.1033749999915,
- "firstDeltaMs": 2303.3529160000035,
+ "roundMs": 2364.2338750000054,
+ "firstDeltaMs": 2135.4769170000072,
"calls": []
}
]
@@ -11433,18 +11027,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1923.645291000008,
+ "gateMs": 1778.734958000001,
"gateSkipped": false,
- "ttftMs": 4003.7998329999973,
- "modelMs": 2420.0521249999874,
+ "ttftMs": 4435.282999999996,
+ "modelMs": 2920.6068340000056,
"toolMs": 0,
- "totalMs": 4343.697415999995,
+ "totalMs": 4699.3417920000065,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2420.0521249999874,
- "firstDeltaMs": 2079.8929999999964,
+ "roundMs": 2920.6068340000056,
+ "firstDeltaMs": 2656.3195000000123,
"calls": []
}
]
@@ -11465,21 +11059,21 @@
"n": 3
},
"ttftMs": {
- "p50": 16714,
- "p95": 26102,
- "mean": 18063,
+ "p50": 21282,
+ "p95": 32718,
+ "mean": 21713,
"n": 3
},
"totalMs": {
- "p50": 17076,
- "p95": 26801,
- "mean": 18560,
+ "p50": 21821,
+ "p95": 33062,
+ "mean": 22130,
"n": 3
},
"modelMs": {
- "p50": 17076,
- "p95": 26801,
- "mean": 18560,
+ "p50": 21821,
+ "p95": 33062,
+ "mean": 22130,
"n": 3
},
"toolMs": {
@@ -11489,8 +11083,8 @@
"n": 3
},
"rounds": {
- "mean": 3.3,
- "max": 5
+ "mean": 4.7,
+ "max": 7
}
},
"runs": [
@@ -11502,59 +11096,75 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 5608.572958999997,
- "execMs": 0.011207999996258877
+ "atMs": 7059.552333,
+ "execMs": 0.0030830000032437965
},
{
"name": "add_clip",
"round": 0,
- "atMs": 5608.677041999996,
- "execMs": 0.02316699999209959
+ "atMs": 7059.5713749999995,
+ "execMs": 0.004832999999052845
},
{
"name": "get_state",
"round": 1,
- "atMs": 8334.503833999988,
- "execMs": 0.14633299999695737
+ "atMs": 10831.793708000012,
+ "execMs": 0.18204200000036508
+ },
+ {
+ "name": "get_state",
+ "round": 2,
+ "atMs": 15679.588917000001,
+ "execMs": 0.00474999999278225
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 11373.874249999993,
- "modelMs": 11804.687500000015,
- "toolMs": 0.18070799998531584,
- "totalMs": 11804.868208,
+ "ttftMs": 21281.922000000006,
+ "modelMs": 21821.046833,
+ "toolMs": 0.19470799999544397,
+ "totalMs": 21821.241540999996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5607.8315,
+ "roundMs": 7059.277125000008,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.011207999996258877
+ "execMs": 0.0030830000032437965
},
{
"name": "add_clip",
- "execMs": 0.02316699999209959
+ "execMs": 0.004832999999052845
+ }
+ ]
+ },
+ {
+ "roundMs": 3772.1297080000077,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.18204200000036508
}
]
},
{
- "roundMs": 2725.6907920000085,
+ "roundMs": 4847.506249999991,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.14633299999695737
+ "execMs": 0.00474999999278225
}
]
},
{
- "roundMs": 3471.165208000006,
- "firstDeltaMs": 3039.198666000011,
+ "roundMs": 6142.133749999994,
+ "firstDeltaMs": 5602.305374999996,
"calls": []
}
]
@@ -11568,91 +11178,59 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 6218.977208000011,
- "execMs": 0.008332999990670942
+ "atMs": 5138.336209000001,
+ "execMs": 0.005208000002312474
},
{
"name": "add_clip",
"round": 0,
- "atMs": 6219.022416000007,
- "execMs": 0.012541999996756203
+ "atMs": 5138.3614589999925,
+ "execMs": 0.007375000001047738
},
{
"name": "get_state",
"round": 1,
- "atMs": 11980.477500000008,
- "execMs": 0.027333000005455688
- },
- {
- "name": "get_state",
- "round": 2,
- "atMs": 18924.785500000013,
- "execMs": 0.006750000000465661
- },
- {
- "name": "get_state",
- "round": 3,
- "atMs": 23123.154875000007,
- "execMs": 0.01304099999833852
+ "atMs": 8319.152625000002,
+ "execMs": 0.061499999996158294
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 26102.39387500001,
- "modelMs": 26800.66958300001,
- "toolMs": 0.06799899999168701,
- "totalMs": 26800.737582,
+ "ttftMs": 11139.835083999991,
+ "modelMs": 11506.445499000009,
+ "toolMs": 0.0740829999995185,
+ "totalMs": 11506.519582000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6218.251749999996,
+ "roundMs": 5137.974583000003,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.008332999990670942
+ "execMs": 0.005208000002312474
},
{
"name": "add_clip",
- "execMs": 0.012541999996756203
- }
- ]
- },
- {
- "roundMs": 5761.278833000004,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.027333000005455688
- }
- ]
- },
- {
- "roundMs": 6944.222834,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.006750000000465661
+ "execMs": 0.007375000001047738
}
]
},
{
- "roundMs": 4198.323499999999,
+ "roundMs": 3180.6267080000107,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.01304099999833852
+ "execMs": 0.061499999996158294
}
]
},
{
- "roundMs": 3678.5926660000114,
- "firstDeltaMs": 2979.1698749999923,
+ "roundMs": 3187.844207999995,
+ "firstDeltaMs": 2820.5461249999935,
"calls": []
}
]
@@ -11666,43 +11244,133 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 7617.488333999994,
- "execMs": 0.0017909999878611416
+ "atMs": 9651.163916999998,
+ "execMs": 0.002666999993380159
},
{
"name": "add_clip",
"round": 0,
- "atMs": 7617.498458999995,
- "execMs": 0.0028750000055879354
+ "atMs": 9651.172374999995,
+ "execMs": 0.00012500002048909664
+ },
+ {
+ "name": "get_state",
+ "round": 1,
+ "atMs": 11954.252792000014,
+ "execMs": 0.020542000012937933
+ },
+ {
+ "name": "get_state",
+ "round": 2,
+ "atMs": 16043.883166999993,
+ "execMs": 0.014667000010376796
+ },
+ {
+ "name": "add_clip",
+ "round": 3,
+ "atMs": 19393.187124999997,
+ "execMs": 0.004125000006752089
+ },
+ {
+ "name": "add_clip",
+ "round": 3,
+ "atMs": 19393.199125000014,
+ "execMs": 0.00020899999071843922
+ },
+ {
+ "name": "get_state",
+ "round": 4,
+ "atMs": 22209.543333999987,
+ "execMs": 0.003208999987691641
+ },
+ {
+ "name": "add_clip",
+ "round": 5,
+ "atMs": 28621.03920900001,
+ "execMs": 0.0021659999911207706
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 16714.204459,
- "modelMs": 17075.876792000025,
- "toolMs": 0.004665999993449077,
- "totalMs": 17075.881458000018,
+ "ttftMs": 32717.8125,
+ "modelMs": 33061.617998999995,
+ "toolMs": 0.047710000013466924,
+ "totalMs": 33061.66570900001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7617.415000000008,
+ "roundMs": 9651.012082999994,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "add_clip",
+ "execMs": 0.002666999993380159
+ },
+ {
+ "name": "add_clip",
+ "execMs": 0.00012500002048909664
+ }
+ ]
+ },
+ {
+ "roundMs": 2303.0405419999734,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.020542000012937933
+ }
+ ]
+ },
+ {
+ "roundMs": 4089.5564160000067,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.014667000010376796
+ }
+ ]
+ },
+ {
+ "roundMs": 3349.201417000004,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.0017909999878611416
+ "execMs": 0.004125000006752089
},
{
"name": "add_clip",
- "execMs": 0.0028750000055879354
+ "execMs": 0.00020899999071843922
+ }
+ ]
+ },
+ {
+ "roundMs": 2816.294790999993,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.003208999987691641
+ }
+ ]
+ },
+ {
+ "roundMs": 6411.464541000023,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "add_clip",
+ "execMs": 0.0021659999911207706
}
]
},
{
- "roundMs": 9458.461792000016,
- "firstDeltaMs": 9096.683833000017,
+ "roundMs": 4441.0482090000005,
+ "firstDeltaMs": 4096.736124999996,
"calls": []
}
]
@@ -11723,21 +11391,21 @@
"n": 3
},
"ttftMs": {
- "p50": 5451,
- "p95": 5764,
- "mean": 5351,
+ "p50": 4332,
+ "p95": 6327,
+ "mean": 4908,
"n": 3
},
"totalMs": {
- "p50": 6075,
- "p95": 6302,
- "mean": 6058,
+ "p50": 5437,
+ "p95": 6782,
+ "mean": 5668,
"n": 3
},
"modelMs": {
- "p50": 6075,
- "p95": 6302,
- "mean": 6058,
+ "p50": 5437,
+ "p95": 6782,
+ "mean": 5668,
"n": 3
},
"toolMs": {
@@ -11760,16 +11428,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5450.720375000004,
- "modelMs": 5797.565415999998,
+ "ttftMs": 4331.698040999996,
+ "modelMs": 4785.104665999999,
"toolMs": 0,
- "totalMs": 5797.565415999998,
+ "totalMs": 4785.104665999999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5797.565415999998,
- "firstDeltaMs": 5450.2189160000125,
+ "roundMs": 4785.104665999999,
+ "firstDeltaMs": 4331.1287909999955,
"calls": []
}
]
@@ -11783,16 +11451,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5763.543209000025,
- "modelMs": 6074.971250000002,
+ "ttftMs": 6327.472167,
+ "modelMs": 6781.7623329999915,
"toolMs": 0,
- "totalMs": 6074.971250000002,
+ "totalMs": 6781.7623329999915,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6074.971250000002,
- "firstDeltaMs": 5763.332750000001,
+ "roundMs": 6781.7623329999915,
+ "firstDeltaMs": 6327.428707999992,
"calls": []
}
]
@@ -11806,16 +11474,16 @@
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 4837.468166000006,
- "modelMs": 6301.538582999987,
+ "ttftMs": 4064.7711249999993,
+ "modelMs": 5437.052333,
"toolMs": 0,
- "totalMs": 6301.538582999987,
+ "totalMs": 5437.052333,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6301.538582999987,
- "firstDeltaMs": 4837.302874999994,
+ "roundMs": 5437.052333,
+ "firstDeltaMs": 4064.595000000001,
"calls": []
}
]
@@ -11830,27 +11498,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 2620,
- "p95": 2669,
- "mean": 2606,
+ "p50": 2967,
+ "p95": 3070,
+ "mean": 2872,
"n": 3
},
"ttftMs": {
- "p50": 6793,
- "p95": 9525,
- "mean": 7653,
+ "p50": 7023,
+ "p95": 11867,
+ "mean": 8568,
"n": 3
},
"totalMs": {
- "p50": 7360,
- "p95": 10020,
- "mean": 8187,
+ "p50": 7741,
+ "p95": 12619,
+ "mean": 9313,
"n": 3
},
"modelMs": {
- "p50": 4740,
- "p95": 7490,
- "mean": 5581,
+ "p50": 5162,
+ "p95": 9652,
+ "mean": 6441,
"n": 3
},
"toolMs": {
@@ -11871,18 +11539,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2669.3064580000064,
+ "gateMs": 3069.776125000004,
"gateSkipped": false,
- "ttftMs": 6641.666667000012,
- "modelMs": 4512.294667000009,
+ "ttftMs": 6815.050958000007,
+ "modelMs": 4510.2124170000025,
"toolMs": 0,
- "totalMs": 7181.601125000016,
+ "totalMs": 7579.988542000006,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4512.294667000009,
- "firstDeltaMs": 3971.727542000008,
+ "roundMs": 4510.2124170000025,
+ "firstDeltaMs": 3745.0743749999965,
"calls": []
}
]
@@ -11894,18 +11562,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2529.628874999995,
+ "gateMs": 2966.7963330000057,
"gateSkipped": false,
- "ttftMs": 9524.604624999978,
- "modelMs": 7490.423249999993,
+ "ttftMs": 11866.744542,
+ "modelMs": 9652.23887500001,
"toolMs": 0,
- "totalMs": 10020.052124999987,
+ "totalMs": 12619.035208000016,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7490.423249999993,
- "firstDeltaMs": 6994.677374999999,
+ "roundMs": 9652.23887500001,
+ "firstDeltaMs": 8899.718499999988,
"calls": []
}
]
@@ -11917,18 +11585,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2619.939458000008,
+ "gateMs": 2579.066166000004,
"gateSkipped": false,
- "ttftMs": 6792.8969579999975,
- "modelMs": 4739.683667000005,
+ "ttftMs": 7023.087958999997,
+ "modelMs": 5161.990458999993,
"toolMs": 0,
- "totalMs": 7359.623125000013,
+ "totalMs": 7741.056624999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4739.683667000005,
- "firstDeltaMs": 4172.721374999994,
+ "roundMs": 5161.990458999993,
+ "firstDeltaMs": 4443.916499999992,
"calls": []
}
]
@@ -11943,27 +11611,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1711,
- "p95": 1749,
- "mean": 1703,
+ "p50": 1902,
+ "p95": 2170,
+ "mean": 1909,
"n": 3
},
"ttftMs": {
- "p50": 8126,
- "p95": 9345,
- "mean": 8434,
+ "p50": 9725,
+ "p95": 10736,
+ "mean": 10055,
"n": 3
},
"totalMs": {
- "p50": 8516,
- "p95": 10236,
- "mean": 9059,
+ "p50": 10245,
+ "p95": 11324,
+ "mean": 10569,
"n": 3
},
"modelMs": {
- "p50": 6805,
- "p95": 8487,
- "mean": 7355,
+ "p50": 8236,
+ "p95": 9670,
+ "mean": 8660,
"n": 3
},
"toolMs": {
@@ -11986,33 +11654,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 6787.063582999981,
- "execMs": 0.010458000004291534
+ "atMs": 7757.044500000018,
+ "execMs": 0.01425000000745058
}
],
"timings": {
- "gateMs": 1710.854083000013,
+ "gateMs": 2170.419957999984,
"gateSkipped": false,
- "ttftMs": 8126.472749999986,
- "modelMs": 6804.692582999996,
- "toolMs": 0.010458000004291534,
- "totalMs": 8515.557124000014,
+ "ttftMs": 9724.847542000003,
+ "modelMs": 8074.775792,
+ "toolMs": 0.01425000000745058,
+ "totalMs": 10245.209999999992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5075.349374999991,
+ "roundMs": 5586.437666999991,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.010458000004291534
+ "execMs": 0.01425000000745058
}
]
},
{
- "roundMs": 1729.3432080000057,
- "firstDeltaMs": 1339.330249999999,
+ "roundMs": 2488.3381250000093,
+ "firstDeltaMs": 1967.77554100001,
"calls": []
}
]
@@ -12026,33 +11694,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 7552.78479200002,
- "execMs": 0.0061249999853316694
+ "atMs": 9214.446582999983,
+ "execMs": 0.007750000018859282
}
],
"timings": {
- "gateMs": 1749.242166000011,
+ "gateMs": 1654.2320419999887,
"gateSkipped": false,
- "ttftMs": 9345.158334000007,
- "modelMs": 8486.969623999961,
- "toolMs": 0.0061249999853316694,
- "totalMs": 10236.217914999957,
+ "ttftMs": 10735.596624999976,
+ "modelMs": 9669.64400000003,
+ "toolMs": 0.007750000018859282,
+ "totalMs": 11323.883792000037,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5803.268457999977,
+ "roundMs": 7560.004417000018,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.0061249999853316694
+ "execMs": 0.007750000018859282
}
]
},
{
- "roundMs": 2683.7011659999844,
- "firstDeltaMs": 1792.3415830000013,
+ "roundMs": 2109.639583000011,
+ "firstDeltaMs": 1521.1150410000118,
"calls": []
}
]
@@ -12066,33 +11734,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 5418.948583999998,
- "execMs": 0.002082999999402091
+ "atMs": 7988.357457999984,
+ "execMs": 0.0027500000142026693
}
],
"timings": {
- "gateMs": 1650.0637500000012,
+ "gateMs": 1901.8656250000058,
"gateSkipped": false,
- "ttftMs": 7831.61712499999,
- "modelMs": 6774.139750000002,
- "toolMs": 0.002082999999402091,
- "totalMs": 8424.205583000003,
+ "ttftMs": 9705.276500000007,
+ "modelMs": 8235.69312499999,
+ "toolMs": 0.0027500000142026693,
+ "totalMs": 10137.56150000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3768.6321250000037,
+ "roundMs": 6086.3697499999835,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.002082999999402091
+ "execMs": 0.0027500000142026693
}
]
},
{
- "roundMs": 3005.5076249999984,
- "firstDeltaMs": 2412.6396250000107,
+ "roundMs": 2149.323375000007,
+ "firstDeltaMs": 1716.9022500000137,
"calls": []
}
]
@@ -12103,38 +11771,38 @@
{
"name": "iterate-keeps-existing-clip",
"bucket": "single-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 0.3333333333333333,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1815,
- "p95": 1876,
- "mean": 1845,
- "n": 2
+ "p50": 1884,
+ "p95": 1884,
+ "mean": 1884,
+ "n": 1
},
"ttftMs": {
- "p50": 10902,
- "p95": 12593,
- "mean": 11748,
- "n": 2
+ "p50": 8915,
+ "p95": 8915,
+ "mean": 8915,
+ "n": 1
},
"totalMs": {
- "p50": 11150,
- "p95": 12935,
- "mean": 12043,
- "n": 2
+ "p50": 9216,
+ "p95": 9216,
+ "mean": 9216,
+ "n": 1
},
"modelMs": {
- "p50": 9335,
- "p95": 11060,
- "mean": 10197,
- "n": 2
+ "p50": 7331,
+ "p95": 7331,
+ "mean": 7331,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -12143,40 +11811,74 @@
},
"runs": [
{
- "pass": true,
+ "pass": false,
"intent": "work",
- "notes": [],
+ "notes": [
+ "mutating tools called: set_color_grade"
+ ],
"trace": [
{
- "name": "generate_video",
+ "name": "set_color_grade",
"round": 0,
- "atMs": 8802.809041,
- "execMs": 0.20737499999813735
+ "atMs": 9222.825749999989,
+ "execMs": 0.45837499998742715
+ },
+ {
+ "name": "stock_search",
+ "round": 1,
+ "atMs": 12179.930958000012,
+ "execMs": 0.024875000002793968
+ },
+ {
+ "name": "generate_video",
+ "round": 2,
+ "atMs": 19872.032458,
+ "execMs": 0.03966599999694154
}
],
"timings": {
- "gateMs": 1814.6333330000052,
+ "gateMs": 1848.1655840000021,
"gateSkipped": false,
- "ttftMs": 10902.352583,
- "modelMs": 9334.910708999989,
- "toolMs": 0.20737499999813735,
- "totalMs": 11149.751416999992,
+ "ttftMs": 21926.977958000003,
+ "modelMs": 20523.27079099996,
+ "toolMs": 0.5229159999871626,
+ "totalMs": 22371.95929099995,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6987.326541999995,
+ "roundMs": 7374.347791999986,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_color_grade",
+ "execMs": 0.45837499998742715
+ }
+ ]
+ },
+ {
+ "roundMs": 2956.5989579999878,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "stock_search",
+ "execMs": 0.024875000002793968
+ }
+ ]
+ },
+ {
+ "roundMs": 7692.033790999994,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.20737499999813735
+ "execMs": 0.03966599999694154
}
]
},
{
- "roundMs": 2347.5841669999936,
- "firstDeltaMs": 2099.2792079999927,
+ "roundMs": 2500.290249999991,
+ "firstDeltaMs": 2054.871374999988,
"calls": []
}
]
@@ -12190,33 +11892,33 @@
{
"name": "generate_video",
"round": 0,
- "atMs": 7905.053249999997,
- "execMs": 0.03908300001057796
+ "atMs": 7053.0055410000205,
+ "execMs": 0.032374999951571226
}
],
"timings": {
- "gateMs": 1875.7818329999864,
+ "gateMs": 1884.338791999995,
"gateSkipped": false,
- "ttftMs": 12592.706791000004,
- "modelMs": 11059.657125000027,
- "toolMs": 0.03908300001057796,
- "totalMs": 12935.478041000024,
+ "ttftMs": 8914.617416000023,
+ "modelMs": 7331.323207999958,
+ "toolMs": 0.032374999951571226,
+ "totalMs": 9215.694374999905,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6028.86387500001,
+ "roundMs": 5168.172249999974,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.03908300001057796
+ "execMs": 0.032374999951571226
}
]
},
{
- "roundMs": 5030.793250000017,
- "firstDeltaMs": 4687.5375420000055,
+ "roundMs": 2163.150957999984,
+ "firstDeltaMs": 1861.543458,
"calls": []
}
]
@@ -12230,83 +11932,67 @@
],
"trace": [
{
- "name": "set_color_grade",
+ "name": "capture_frame",
"round": 0,
- "atMs": 7673.009250000003,
- "execMs": 0.04520799999590963
+ "atMs": 11492.753917000024,
+ "execMs": 0.022500000020954758
},
{
- "name": "stock_search",
+ "name": "set_color_grade",
"round": 1,
- "atMs": 13725.98216700001,
- "execMs": 0.09454199997708201
- },
- {
- "name": "stock_search",
- "round": 2,
- "atMs": 15935.895250000001,
- "execMs": 0.01554199995007366
+ "atMs": 16820.78474999999,
+ "execMs": 0.017416000016964972
},
{
"name": "generate_video",
- "round": 3,
- "atMs": 19572.82704200002,
- "execMs": 0.017709000036120415
+ "round": 2,
+ "atMs": 22191.509917000018,
+ "execMs": 0.012458999990485609
}
],
"timings": {
- "gateMs": 1558.9444580000127,
+ "gateMs": 1721.1451670000097,
"gateSkipped": false,
- "ttftMs": 21432.069582999975,
- "modelMs": 20062.652458000026,
- "toolMs": 0.17300099995918572,
- "totalMs": 21621.769916999998,
+ "ttftMs": 23898.07166700001,
+ "modelMs": 22619.06091500004,
+ "toolMs": 0.05237500002840534,
+ "totalMs": 24340.258457000076,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6113.789832999988,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.04520799999590963
- }
- ]
- },
- {
- "roundMs": 6052.850583000021,
+ "roundMs": 9771.500791000028,
"firstDeltaMs": null,
"calls": [
{
- "name": "stock_search",
- "execMs": 0.09454199997708201
+ "name": "capture_frame",
+ "execMs": 0.022500000020954758
}
]
},
{
- "roundMs": 2209.7534999999916,
+ "roundMs": 5327.968416000018,
"firstDeltaMs": null,
"calls": [
{
- "name": "stock_search",
- "execMs": 0.01554199995007366
+ "name": "set_color_grade",
+ "execMs": 0.017416000016964972
}
]
},
{
- "roundMs": 3636.8678749999963,
+ "roundMs": 5370.675958000007,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.017709000036120415
+ "execMs": 0.012458999990485609
}
]
},
{
- "roundMs": 2049.3906670000288,
- "firstDeltaMs": 1859.169292000006,
+ "roundMs": 2148.9157499999856,
+ "firstDeltaMs": 1706.5085410000174,
"calls": []
}
]
@@ -12317,211 +12003,186 @@
{
"name": "mute-all-does-not-redo-prior-turn",
"bucket": "single-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1567,
+ "p95": 1573,
+ "mean": 1549,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 9656,
+ "p95": 12849,
+ "mean": 9927,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 9981,
+ "p95": 13169,
+ "mean": 10149,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 8472,
+ "p95": 11602,
+ "mean": 8600,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "slow: 4 tool calls (cap 3)",
- "mutating tools called: set_track_muted"
- ],
+ "notes": [],
"trace": [
- {
- "name": "set_track_muted",
- "round": 0,
- "atMs": 4958.056749999989,
- "execMs": 0.1578330000047572
- },
{
"name": "set_clip_muted",
- "round": 1,
- "atMs": 6983.1316669999505,
- "execMs": 0.011999999987892807
+ "round": 0,
+ "atMs": 7971.766541999998,
+ "execMs": 0.2541670000064187
},
{
"name": "set_clip_muted",
- "round": 1,
- "atMs": 6983.152457999997,
- "execMs": 0.0013329999637790024
+ "round": 0,
+ "atMs": 7972.033249999979,
+ "execMs": 0.0021249999990686774
},
{
"name": "set_clip_muted",
- "round": 1,
- "atMs": 6983.154666999995,
- "execMs": 0.0005000000237487257
+ "round": 0,
+ "atMs": 7972.036458999966,
+ "execMs": 0.0006670000148005784
}
],
"timings": {
- "gateMs": 1651.113207999966,
+ "gateMs": 1508.3286250000237,
"gateSkipped": false,
- "ttftMs": 8427.122082999966,
- "modelMs": 7124.663749999949,
- "toolMs": 0.17166599998017773,
- "totalMs": 8775.948623999895,
+ "ttftMs": 9656.367958999996,
+ "modelMs": 8471.917707999994,
+ "toolMs": 0.25695900002028793,
+ "totalMs": 9980.503292000038,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3304.7224999999744,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_track_muted",
- "execMs": 0.1578330000047572
- }
- ]
- },
- {
- "roundMs": 2024.8812499999767,
+ "roundMs": 6462.718707999971,
"firstDeltaMs": null,
"calls": [
{
"name": "set_clip_muted",
- "execMs": 0.011999999987892807
+ "execMs": 0.2541670000064187
},
{
"name": "set_clip_muted",
- "execMs": 0.0013329999637790024
+ "execMs": 0.0021249999990686774
},
{
"name": "set_clip_muted",
- "execMs": 0.0005000000237487257
+ "execMs": 0.0006670000148005784
}
]
},
{
- "roundMs": 1795.0599999999977,
- "firstDeltaMs": 1443.9532079999917,
+ "roundMs": 2009.1990000000224,
+ "firstDeltaMs": 1684.2884160000249,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "reply did not match /mut/i"
- ],
+ "notes": [],
"trace": [
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 6319.478834000009,
- "execMs": 0.016792000038549304
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 6319.505125000025,
- "execMs": 0.0007910000276751816
- },
- {
- "name": "set_clip_muted",
+ "name": "set_track_muted",
"round": 0,
- "atMs": 6319.507000000041,
- "execMs": 0.000417000032030046
+ "atMs": 5499.885084000009,
+ "execMs": 0.03920800000196323
}
],
"timings": {
- "gateMs": 1679.3181249999907,
+ "gateMs": 1572.7849580000038,
"gateSkipped": false,
- "ttftMs": 8054.802292000037,
- "modelMs": 6622.259749999968,
- "toolMs": 0.01800000009825453,
- "totalMs": 8301.595875000057,
+ "ttftMs": 7274.411416999996,
+ "modelMs": 5724.824040999985,
+ "toolMs": 0.03920800000196323,
+ "totalMs": 7297.648206999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4639.451082999993,
+ "roundMs": 3926.4868329999736,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.016792000038549304
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0007910000276751816
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.000417000032030046
+ "name": "set_track_muted",
+ "execMs": 0.03920800000196323
}
]
},
{
- "roundMs": 1982.8086669999757,
- "firstDeltaMs": 1735.2410420000087,
+ "roundMs": 1798.3372080000117,
+ "firstDeltaMs": 1774.4419580000103,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "slow: 4 tool calls (cap 3)",
- "mutating tools called: set_track_muted"
- ],
+ "notes": [],
"trace": [
- {
- "name": "set_track_muted",
- "round": 0,
- "atMs": 4558.957416000019,
- "execMs": 0.022624999983236194
- },
{
"name": "set_clip_muted",
- "round": 1,
- "atMs": 7951.654791000008,
- "execMs": 0.009541000006720424
+ "round": 0,
+ "atMs": 9950.98212499998,
+ "execMs": 0.01854199997615069
},
{
"name": "set_clip_muted",
- "round": 1,
- "atMs": 7951.668915999995,
+ "round": 0,
+ "atMs": 9951.007500000007,
"execMs": 0.0009160000481642783
},
{
"name": "set_clip_muted",
- "round": 1,
- "atMs": 7951.670582999999,
- "execMs": 0.0004159999662078917
+ "round": 0,
+ "atMs": 9951.009541000007,
+ "execMs": 0.0005830000154674053
}
],
"timings": {
- "gateMs": 1616.1199170000036,
+ "gateMs": 1566.9033339999733,
"gateSkipped": false,
- "ttftMs": 9798.946832999995,
- "modelMs": 8397.795832999924,
- "toolMs": 0.03349800000432879,
- "totalMs": 10013.949247999932,
+ "ttftMs": 12849.403041000012,
+ "modelMs": 11602.057040999993,
+ "toolMs": 0.020041000039782375,
+ "totalMs": 13168.980416000006,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2942.4423329999554,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_track_muted",
- "execMs": 0.022624999983236194
- }
- ]
- },
- {
- "roundMs": 3392.6211669999757,
+ "roundMs": 8383.778208000003,
"firstDeltaMs": null,
"calls": [
{
"name": "set_clip_muted",
- "execMs": 0.009541000006720424
+ "execMs": 0.01854199997615069
},
{
"name": "set_clip_muted",
@@ -12529,13 +12190,13 @@
},
{
"name": "set_clip_muted",
- "execMs": 0.0004159999662078917
+ "execMs": 0.0005830000154674053
}
]
},
{
- "roundMs": 2062.7323329999927,
- "firstDeltaMs": 1847.2634580000304,
+ "roundMs": 3218.2788329999894,
+ "firstDeltaMs": 2898.347749999957,
"calls": []
}
]
@@ -12550,32 +12211,32 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1553,
- "p95": 1594,
- "mean": 1529,
+ "p50": 1499,
+ "p95": 2301,
+ "mean": 1745,
"n": 3
},
"ttftMs": {
- "p50": 7841,
- "p95": 11119,
- "mean": 8542,
+ "p50": 7577,
+ "p95": 7610,
+ "mean": 7451,
"n": 3
},
"totalMs": {
- "p50": 8114,
- "p95": 11460,
- "mean": 8885,
+ "p50": 7818,
+ "p95": 7928,
+ "mean": 7717,
"n": 3
},
"modelMs": {
- "p50": 6675,
- "p95": 9866,
- "mean": 7356,
+ "p50": 6383,
+ "p95": 6429,
+ "mean": 5971,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
@@ -12593,33 +12254,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 6944.910542000027,
- "execMs": 0.5484169999836013
+ "atMs": 5768.789374999993,
+ "execMs": 0.12383400002727285
}
],
"timings": {
- "gateMs": 1593.7056250000023,
+ "gateMs": 1498.776957999973,
"gateSkipped": false,
- "ttftMs": 11119.13270800002,
- "modelMs": 9866.20233400003,
- "toolMs": 0.5484169999836013,
- "totalMs": 11460.456376000016,
+ "ttftMs": 7576.531416999991,
+ "modelMs": 6429.175874000066,
+ "toolMs": 0.12383400002727285,
+ "totalMs": 7928.076666000066,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5350.240500000014,
+ "roundMs": 4269.513916000025,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.5484169999836013
+ "execMs": 0.12383400002727285
}
]
},
{
- "roundMs": 4515.961834000016,
- "firstDeltaMs": 4173.6302090000245,
+ "roundMs": 2159.6619580000406,
+ "firstDeltaMs": 1807.59954200004,
"calls": []
}
]
@@ -12633,33 +12294,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 6213.116500000004,
- "execMs": 0.04287499998463318
+ "atMs": 5584.545374999987,
+ "execMs": 0.03604099998483434
}
],
"timings": {
- "gateMs": 1439.0314999999828,
+ "gateMs": 2301.129707999993,
"gateSkipped": false,
- "ttftMs": 7840.988875000039,
- "modelMs": 6674.980708000017,
- "toolMs": 0.04287499998463318,
- "totalMs": 8114.055082999985,
+ "ttftMs": 7166.601207999978,
+ "modelMs": 5102.265918000019,
+ "toolMs": 0.03604099998483434,
+ "totalMs": 7403.431666999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4773.840166000009,
+ "roundMs": 3283.320209000027,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.04287499998463318
+ "execMs": 0.03604099998483434
}
]
},
{
- "roundMs": 1901.1405420000083,
- "firstDeltaMs": 1627.7837090000394,
+ "roundMs": 1818.9457089999923,
+ "firstDeltaMs": 1581.9739169999957,
"calls": []
}
]
@@ -12673,33 +12334,33 @@
{
"name": "generate_image",
"round": 0,
- "atMs": 5463.046834000037,
- "execMs": 0.026459000015165657
+ "atMs": 6107.375,
+ "execMs": 0.018749999988358468
}
],
"timings": {
- "gateMs": 1552.8722079999861,
+ "gateMs": 1435.2664160000277,
"gateSkipped": false,
- "ttftMs": 6664.379042000044,
- "modelMs": 5526.197166999977,
- "toolMs": 0.026459000015165657,
- "totalMs": 7079.095833999978,
+ "ttftMs": 7609.921957999992,
+ "modelMs": 6382.799540999986,
+ "toolMs": 0.018749999988358468,
+ "totalMs": 7818.084707000002,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3909.925333000021,
+ "roundMs": 4671.871082999976,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_image",
- "execMs": 0.026459000015165657
+ "execMs": 0.018749999988358468
}
]
},
{
- "roundMs": 1616.2718339999556,
- "firstDeltaMs": 1201.267041999963,
+ "roundMs": 1710.9284580000094,
+ "firstDeltaMs": 1502.4896249999874,
"calls": []
}
]
@@ -12714,27 +12375,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1879,
- "p95": 2008,
- "mean": 1871,
+ "p50": 1568,
+ "p95": 1915,
+ "mean": 1624,
"n": 3
},
"ttftMs": {
- "p50": 9667,
- "p95": 11421,
- "mean": 10020,
+ "p50": 7851,
+ "p95": 7902,
+ "mean": 7769,
"n": 3
},
"totalMs": {
- "p50": 9864,
- "p95": 11880,
- "mean": 10308,
+ "p50": 8258,
+ "p95": 8308,
+ "mean": 8209,
"n": 3
},
"modelMs": {
- "p50": 8136,
- "p95": 10001,
- "mean": 8437,
+ "p50": 6673,
+ "p95": 6741,
+ "mean": 6585,
"n": 3
},
"toolMs": {
@@ -12757,33 +12418,33 @@
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 6443.113416999986,
- "execMs": 0.04879200004506856
+ "atMs": 6143.076584000024,
+ "execMs": 0.038416999974288046
}
],
"timings": {
- "gateMs": 1878.7037910000072,
+ "gateMs": 1915.4628329999978,
"gateSkipped": false,
- "ttftMs": 11420.62941699999,
- "modelMs": 10001.460708999948,
- "toolMs": 0.04879200004506856,
- "totalMs": 11880.213292,
+ "ttftMs": 7902.2184170000255,
+ "modelMs": 6342.338625000033,
+ "toolMs": 0.038416999974288046,
+ "totalMs": 8257.839875000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4563.928249999997,
+ "roundMs": 4227.1413750000065,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.04879200004506856
+ "execMs": 0.038416999974288046
}
]
},
{
- "roundMs": 5437.532458999951,
- "firstDeltaMs": 4977.3975419999915,
+ "roundMs": 2115.1972500000265,
+ "firstDeltaMs": 1759.081583000021,
"calls": []
}
]
@@ -12797,33 +12458,33 @@
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 7802.9912919999915,
- "execMs": 0.012624999973922968
+ "atMs": 6019.524708999961,
+ "execMs": 0.008999999961815774
}
],
"timings": {
- "gateMs": 1728.1299589999835,
+ "gateMs": 1387.4689580000122,
"gateSkipped": false,
- "ttftMs": 9667.48116700002,
- "modelMs": 8135.591416999989,
- "toolMs": 0.012624999973922968,
- "totalMs": 9863.734000999946,
+ "ttftMs": 7555.35712499998,
+ "modelMs": 6673.011667000072,
+ "toolMs": 0.008999999961815774,
+ "totalMs": 8060.489625000046,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6074.664874999959,
+ "roundMs": 4631.777417000034,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.012624999973922968
+ "execMs": 0.008999999961815774
}
]
},
{
- "roundMs": 2060.92654200003,
- "firstDeltaMs": 1864.4383340000059,
+ "roundMs": 2041.2342500000377,
+ "firstDeltaMs": 1535.774916000024,
"calls": []
}
]
@@ -12837,33 +12498,33 @@
{
"name": "voiceover_generate",
"round": 0,
- "atMs": 7087.76337499998,
- "execMs": 0.01308399997651577
+ "atMs": 6287.113582999969,
+ "execMs": 0.004124999977648258
}
],
"timings": {
- "gateMs": 2007.5591249999707,
+ "gateMs": 1567.7531670000171,
"gateSkipped": false,
- "ttftMs": 8971.396749999956,
- "modelMs": 7173.442749999987,
- "toolMs": 0.01308399997651577,
- "totalMs": 9181.014958999935,
+ "ttftMs": 7850.6337080000085,
+ "modelMs": 6740.612667999987,
+ "toolMs": 0.004124999977648258,
+ "totalMs": 8308.369959999982,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5080.052041999996,
+ "roundMs": 4719.155083999969,
"firstDeltaMs": null,
"calls": [
{
"name": "voiceover_generate",
- "execMs": 0.01308399997651577
+ "execMs": 0.004124999977648258
}
]
},
{
- "roundMs": 2093.3907079999917,
- "firstDeltaMs": 1883.5791660000104,
+ "roundMs": 2021.457584000018,
+ "firstDeltaMs": 1563.5040000000154,
"calls": []
}
]
@@ -12874,179 +12535,160 @@
{
"name": "music-ask-single-generate",
"bucket": "single-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1573,
+ "p95": 2383,
+ "mean": 1819,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 8205,
+ "p95": 10685,
+ "mean": 8952,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 8498,
+ "p95": 11093,
+ "mean": 9295,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 6925,
+ "p95": 9591,
+ "mean": 7476,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: add_clip"
- ],
+ "notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 6811.897624999983,
- "execMs": 0.005834000010509044
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 9894.981417000003,
- "execMs": 0.015499999979510903
+ "atMs": 8340.620791999972,
+ "execMs": 0.2873339999932796
}
],
"timings": {
- "gateMs": 2190.99429100001,
+ "gateMs": 1501.5215000000317,
"gateSkipped": false,
- "ttftMs": 17051.637999999977,
- "modelMs": 15272.051999999967,
- "toolMs": 0.021333999990019947,
- "totalMs": 17463.067624999967,
+ "ttftMs": 10685.084208999993,
+ "modelMs": 9591.199293000042,
+ "toolMs": 0.2873339999932796,
+ "totalMs": 11093.008127000066,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4620.505000000005,
+ "roundMs": 6838.6554590000305,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.005834000010509044
- }
- ]
- },
- {
- "roundMs": 3082.862124999985,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.015499999979510903
+ "execMs": 0.2873339999932796
}
]
},
{
- "roundMs": 7568.684874999977,
- "firstDeltaMs": 7156.58566599997,
+ "roundMs": 2752.543834000011,
+ "firstDeltaMs": 2344.137374999991,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: add_clip"
- ],
+ "notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 6414.52804200002,
- "execMs": 0.013124999997671694
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 9660.976083000016,
- "execMs": 0.012250000028871
+ "atMs": 6008.007458000036,
+ "execMs": 0.01170899998396635
}
],
"timings": {
- "gateMs": 1805.8030830000062,
+ "gateMs": 2382.6214170000167,
"gateSkipped": false,
- "ttftMs": 12554.750791999977,
- "modelMs": 11053.705875000043,
- "toolMs": 0.025375000026542693,
- "totalMs": 12859.534333000076,
+ "ttftMs": 7967.531666000024,
+ "modelMs": 5912.663334000041,
+ "toolMs": 0.01170899998396635,
+ "totalMs": 8295.296460000041,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4608.390707999992,
+ "roundMs": 3625.011125000019,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.013124999997671694
- }
- ]
- },
- {
- "roundMs": 3246.3592500000377,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.012250000028871
+ "execMs": 0.01170899998396635
}
]
},
{
- "roundMs": 3198.955917000014,
- "firstDeltaMs": 2893.7229579999694,
+ "roundMs": 2287.6522090000217,
+ "firstDeltaMs": 1959.4860839999747,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: add_clip"
- ],
+ "notes": [],
"trace": [
{
"name": "generate_music",
"round": 0,
- "atMs": 5764.396708000044,
- "execMs": 0.010167000000365078
- },
- {
- "name": "add_clip",
- "round": 1,
- "atMs": 9392.675083000038,
- "execMs": 0.012041999958455563
+ "atMs": 5849.776874999981,
+ "execMs": 0.011165999982040375
}
],
"timings": {
- "gateMs": 1582.3647919999785,
+ "gateMs": 1572.5699160000077,
"gateSkipped": false,
- "ttftMs": 13010.550042000017,
- "modelMs": 12032.76449999999,
- "toolMs": 0.02220899995882064,
- "totalMs": 13615.151500999928,
+ "ttftMs": 8204.594209000003,
+ "modelMs": 6925.2364580000285,
+ "toolMs": 0.011165999982040375,
+ "totalMs": 8497.817540000018,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4181.915041,
+ "roundMs": 4276.849500000011,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_music",
- "execMs": 0.010167000000365078
- }
- ]
- },
- {
- "roundMs": 3628.1900000000023,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.012041999958455563
+ "execMs": 0.011165999982040375
}
]
},
{
- "roundMs": 4222.659458999988,
- "firstDeltaMs": 3617.81820899999,
+ "roundMs": 2648.3869580000173,
+ "firstDeltaMs": 2354.7594999999856,
"calls": []
}
]
@@ -13061,27 +12703,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 2198,
- "p95": 2201,
- "mean": 1972,
+ "p50": 2011,
+ "p95": 2085,
+ "mean": 1949,
"n": 3
},
"ttftMs": {
- "p50": 5816,
- "p95": 6772,
- "mean": 6117,
+ "p50": 6766,
+ "p95": 6809,
+ "mean": 6670,
"n": 3
},
"totalMs": {
- "p50": 6084,
- "p95": 6920,
- "mean": 6299,
+ "p50": 7101,
+ "p95": 7102,
+ "mean": 6989,
"n": 3
},
"modelMs": {
- "p50": 4568,
- "p95": 4719,
- "mean": 4327,
+ "p50": 5016,
+ "p95": 5091,
+ "mean": 5040,
"n": 3
},
"toolMs": {
@@ -13104,33 +12746,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 4114.1577080000425,
- "execMs": 0.04175000003306195
+ "atMs": 4702.68191699998,
+ "execMs": 0.0626250000204891
}
],
"timings": {
- "gateMs": 1516.110541000031,
+ "gateMs": 2010.5444169999682,
"gateSkipped": false,
- "ttftMs": 5762.734792000032,
- "modelMs": 4567.809124000021,
- "toolMs": 0.04175000003306195,
- "totalMs": 6083.961415000085,
+ "ttftMs": 6765.78570899996,
+ "modelMs": 5091.0555830000085,
+ "toolMs": 0.0626250000204891,
+ "totalMs": 7101.662624999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2597.4999160000007,
+ "roundMs": 2691.98208300001,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.04175000003306195
+ "execMs": 0.0626250000204891
}
]
},
{
- "roundMs": 1970.3092080000206,
- "firstDeltaMs": 1648.497875000001,
+ "roundMs": 2399.0734999999986,
+ "firstDeltaMs": 2063.004249999998,
"calls": []
}
]
@@ -13144,33 +12786,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 5048.606832999969,
- "execMs": 0.009874999988824129
+ "atMs": 4828.055499999959,
+ "execMs": 0.0042910000192932785
}
],
"timings": {
- "gateMs": 2201.4619999999995,
+ "gateMs": 1751.5354579999694,
"gateSkipped": false,
- "ttftMs": 6771.528208000003,
- "modelMs": 4718.995167000045,
- "toolMs": 0.009874999988824129,
- "totalMs": 6920.467042000033,
+ "ttftMs": 6434.503415999992,
+ "modelMs": 5012.710541999957,
+ "toolMs": 0.0042910000192932785,
+ "totalMs": 6764.250290999946,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2846.9122080000234,
+ "roundMs": 3076.1674999999814,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.009874999988824129
+ "execMs": 0.0042910000192932785
}
]
},
{
- "roundMs": 1872.0829590000212,
- "firstDeltaMs": 1722.8735420000157,
+ "roundMs": 1936.5430419999757,
+ "firstDeltaMs": 1606.4115420000162,
"calls": []
}
]
@@ -13184,33 +12826,33 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 4231.794750000001,
- "execMs": 0.008333000005222857
+ "atMs": 4995.234750000003,
+ "execMs": 0.01079199998639524
}
],
"timings": {
- "gateMs": 2198.181125000003,
+ "gateMs": 2084.9662499999977,
"gateSkipped": false,
- "ttftMs": 5816.022500000021,
- "modelMs": 3693.2577069999534,
- "toolMs": 0.008333000005222857,
- "totalMs": 5891.447164999961,
+ "ttftMs": 6808.679375000007,
+ "modelMs": 5015.833790999954,
+ "toolMs": 0.01079199998639524,
+ "totalMs": 7100.810832999938,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2033.3794159999816,
+ "roundMs": 2909.870790999965,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.008333000005222857
+ "execMs": 0.01079199998639524
}
]
},
{
- "roundMs": 1659.8782909999718,
- "firstDeltaMs": 1584.1882079999777,
+ "roundMs": 2105.962999999989,
+ "firstDeltaMs": 1813.3401670000167,
"calls": []
}
]
@@ -13225,27 +12867,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1544,
- "p95": 1946,
- "mean": 1663,
+ "p50": 1850,
+ "p95": 1999,
+ "mean": 1729,
"n": 3
},
"ttftMs": {
- "p50": 6735,
- "p95": 9649,
- "mean": 7671,
+ "p50": 7046,
+ "p95": 9878,
+ "mean": 7954,
"n": 3
},
"totalMs": {
- "p50": 7062,
- "p95": 9802,
- "mean": 7929,
+ "p50": 7356,
+ "p95": 10387,
+ "mean": 8295,
"n": 3
},
"modelMs": {
- "p50": 5518,
- "p95": 7857,
- "mean": 6267,
+ "p50": 6018,
+ "p95": 8387,
+ "mean": 6565,
"n": 3
},
"toolMs": {
@@ -13268,33 +12910,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 5190.259499999986,
- "execMs": 0.309917000005953
+ "atMs": 5287.226250000007,
+ "execMs": 0.1401670000050217
}
],
"timings": {
- "gateMs": 1543.702249999973,
+ "gateMs": 1850.217500000028,
"gateSkipped": false,
- "ttftMs": 6735.142666,
- "modelMs": 5517.733125000028,
- "toolMs": 0.309917000005953,
- "totalMs": 7061.745292000007,
+ "ttftMs": 7045.658666000003,
+ "modelMs": 5290.768417000014,
+ "toolMs": 0.1401670000050217,
+ "totalMs": 7141.126084000047,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3645.944583000033,
+ "roundMs": 3436.4283339999965,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.309917000005953
+ "execMs": 0.1401670000050217
}
]
},
{
- "roundMs": 1871.7885419999948,
- "firstDeltaMs": 1544.4941249999683,
+ "roundMs": 1854.3400830000173,
+ "firstDeltaMs": 1758.2375000000466,
"calls": []
}
]
@@ -13308,33 +12950,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 7733.293458,
- "execMs": 0.017958000011276454
+ "atMs": 5518.360333999968,
+ "execMs": 0.012792000023182482
}
],
"timings": {
- "gateMs": 1945.5491659999825,
+ "gateMs": 1999.4189999999944,
"gateSkipped": false,
- "ttftMs": 9649.16745800001,
- "modelMs": 7856.76004199998,
- "toolMs": 0.017958000011276454,
- "totalMs": 9802.327165999974,
+ "ttftMs": 9878.161458999966,
+ "modelMs": 8387.49874900002,
+ "toolMs": 0.012792000023182482,
+ "totalMs": 10386.930541000038,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5787.349250000028,
+ "roundMs": 3518.642416000017,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.017958000011276454
+ "execMs": 0.012792000023182482
}
]
},
{
- "roundMs": 2069.4107919999515,
- "firstDeltaMs": 1915.7794169999543,
+ "roundMs": 4868.856333000003,
+ "firstDeltaMs": 4359.764040999988,
"calls": []
}
]
@@ -13348,33 +12990,33 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 5053.49887499999,
- "execMs": 0.0232079999987036
+ "atMs": 5410.97229200002,
+ "execMs": 0.015083000005688518
}
],
"timings": {
- "gateMs": 1499.232707999996,
+ "gateMs": 1338.5506670000032,
"gateSkipped": false,
- "ttftMs": 6627.9673340000445,
- "modelMs": 5425.037375000073,
- "toolMs": 0.0232079999987036,
- "totalMs": 6924.293291000067,
+ "ttftMs": 6936.894666999986,
+ "modelMs": 6017.781750000024,
+ "toolMs": 0.015083000005688518,
+ "totalMs": 7356.347500000033,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3554.017792000028,
+ "roundMs": 4072.29137500003,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.0232079999987036
+ "execMs": 0.015083000005688518
}
]
},
{
- "roundMs": 1871.0195830000448,
- "firstDeltaMs": 1574.4046660000458,
+ "roundMs": 1945.490374999994,
+ "firstDeltaMs": 1525.880625000049,
"calls": []
}
]
@@ -13389,27 +13031,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1730,
- "p95": 1869,
- "mean": 1709,
+ "p50": 1605,
+ "p95": 1832,
+ "mean": 1636,
"n": 3
},
"ttftMs": {
- "p50": 8591,
- "p95": 9376,
- "mean": 8724,
+ "p50": 12213,
+ "p95": 13673,
+ "mean": 11116,
"n": 3
},
"totalMs": {
- "p50": 8942,
- "p95": 9690,
- "mean": 9059,
+ "p50": 12587,
+ "p95": 13949,
+ "mean": 11460,
"n": 3
},
"modelMs": {
- "p50": 7073,
- "p95": 8164,
- "mean": 7350,
+ "p50": 10755,
+ "p95": 12345,
+ "mean": 9824,
"n": 3
},
"toolMs": {
@@ -13419,8 +13061,8 @@
"n": 3
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3.3,
+ "max": 4
}
},
"runs": [
@@ -13432,33 +13074,33 @@
{
"name": "set_animation",
"round": 0,
- "atMs": 6686.777333000035,
- "execMs": 0.4107499999809079
+ "atMs": 6044.951707999979,
+ "execMs": 0.28854199999477714
}
],
"timings": {
- "gateMs": 1730.2487079999992,
+ "gateMs": 1471.1530829999829,
"gateSkipped": false,
- "ttftMs": 8203.532583000022,
- "modelMs": 6813.846834000025,
- "toolMs": 0.4107499999809079,
- "totalMs": 8544.506292000005,
+ "ttftMs": 7463.295457999979,
+ "modelMs": 6372.324583000038,
+ "toolMs": 0.28854199999477714,
+ "totalMs": 7843.7662080000155,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4956.122875000001,
+ "roundMs": 4573.093083000043,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.4107499999809079
+ "execMs": 0.28854199999477714
}
]
},
{
- "roundMs": 1857.7239590000245,
- "firstDeltaMs": 1516.294458999997,
+ "roundMs": 1799.2314999999944,
+ "firstDeltaMs": 1418.0074579999782,
"calls": []
}
]
@@ -13470,35 +13112,67 @@
"notes": [],
"trace": [
{
- "name": "set_animation",
+ "name": "list_skills",
"round": 0,
- "atMs": 7018.512208,
- "execMs": 0.004625000001396984
+ "atMs": 5452.24420799996,
+ "execMs": 0.0921669999952428
+ },
+ {
+ "name": "read_skill",
+ "round": 1,
+ "atMs": 7099.714040999999,
+ "execMs": 0.02887500001816079
+ },
+ {
+ "name": "set_animation",
+ "round": 2,
+ "atMs": 10709.138832999975,
+ "execMs": 0.008042000001296401
}
],
"timings": {
- "gateMs": 1869.4467920000316,
+ "gateMs": 1831.5383750000037,
"gateSkipped": false,
- "ttftMs": 8590.86158299999,
- "modelMs": 7072.7267499999725,
- "toolMs": 0.004625000001396984,
- "totalMs": 8942.178167000005,
+ "ttftMs": 12212.506915999984,
+ "modelMs": 10754.917541999952,
+ "toolMs": 0.1290840000147,
+ "totalMs": 12586.58500099997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5148.795125000004,
+ "roundMs": 3620.26791699999,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "list_skills",
+ "execMs": 0.0921669999952428
+ }
+ ]
+ },
+ {
+ "roundMs": 1647.3079169999692,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "read_skill",
+ "execMs": 0.02887500001816079
+ }
+ ]
+ },
+ {
+ "roundMs": 3609.291790999996,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.004625000001396984
+ "execMs": 0.008042000001296401
}
]
},
{
- "roundMs": 1923.9316249999683,
- "firstDeltaMs": 1572.3321249999572,
+ "roundMs": 1878.0499169999966,
+ "firstDeltaMs": 1503.3265000000247,
"calls": []
}
]
@@ -13510,35 +13184,67 @@
"notes": [],
"trace": [
{
- "name": "set_animation",
+ "name": "list_skills",
"round": 0,
- "atMs": 7443.279042000009,
- "execMs": 0.0236670000012964
+ "atMs": 5873.595041000051,
+ "execMs": 0.04899999999906868
+ },
+ {
+ "name": "read_skill",
+ "round": 1,
+ "atMs": 8169.138708000013,
+ "execMs": 0.004167000006418675
+ },
+ {
+ "name": "set_animation",
+ "round": 2,
+ "atMs": 10713.883791,
+ "execMs": 0.03291700000409037
}
],
"timings": {
- "gateMs": 1525.8115830000024,
+ "gateMs": 1604.690500000026,
"gateSkipped": false,
- "ttftMs": 9376.176459000038,
- "modelMs": 8164.319625000004,
- "toolMs": 0.0236670000012964,
- "totalMs": 9690.154875000007,
+ "ttftMs": 13672.636666000006,
+ "modelMs": 12344.719748999923,
+ "toolMs": 0.08608400000957772,
+ "totalMs": 13949.49633299996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5917.1807499999995,
+ "roundMs": 4268.6708749999525,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "list_skills",
+ "execMs": 0.04899999999906868
+ }
+ ]
+ },
+ {
+ "roundMs": 2295.4100409999955,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "read_skill",
+ "execMs": 0.004167000006418675
+ }
+ ]
+ },
+ {
+ "roundMs": 2544.6917910000193,
"firstDeltaMs": null,
"calls": [
{
"name": "set_animation",
- "execMs": 0.0236670000012964
+ "execMs": 0.03291700000409037
}
]
},
{
- "roundMs": 2247.138875000004,
- "firstDeltaMs": 1932.8257919999887,
+ "roundMs": 3235.947041999956,
+ "firstDeltaMs": 2958.6602079999866,
"calls": []
}
]
@@ -13549,182 +13255,160 @@
{
"name": "look-ask-single-tool",
"bucket": "single-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1587,
+ "p95": 1851,
+ "mean": 1633,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 7217,
+ "p95": 9546,
+ "mean": 7854,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 7611,
+ "p95": 9901,
+ "mean": 8227,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 6148,
+ "p95": 8050,
+ "mean": 6593,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, set_color_grade"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 5235.285625000019,
- "execMs": 0.25333300000056624
- },
- {
- "name": "set_color_grade",
- "round": 1,
- "atMs": 10847.858292000019,
- "execMs": 0.0313750000204891
+ "atMs": 5451.605333000014,
+ "execMs": 0.2962090000510216
}
],
"timings": {
- "gateMs": 1609.7616660000058,
+ "gateMs": 1462.544166000036,
"gateSkipped": false,
- "ttftMs": 14901.011375000002,
- "modelMs": 13745.535166999965,
- "toolMs": 0.28470800002105534,
- "totalMs": 15355.581540999992,
+ "ttftMs": 7217.064124999975,
+ "modelMs": 6148.430374000047,
+ "toolMs": 0.2962090000510216,
+ "totalMs": 7611.270749000134,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3625.031666999974,
+ "roundMs": 3988.614166000043,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.25333300000056624
- }
- ]
- },
- {
- "roundMs": 5612.130000000005,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.0313750000204891
+ "execMs": 0.2962090000510216
}
]
},
{
- "roundMs": 4508.373499999987,
- "firstDeltaMs": 4053.026291999966,
+ "roundMs": 2159.816208000004,
+ "firstDeltaMs": 1765.0970419999794,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, set_color_grade"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 5653.95437500003,
- "execMs": 0.006624999979976565
- },
- {
- "name": "set_color_grade",
- "round": 1,
- "atMs": 14065.912791999988,
- "execMs": 0.011415999964810908
+ "atMs": 5300.147625000041,
+ "execMs": 0.027624999987892807
}
],
"timings": {
- "gateMs": 2371.093958000012,
+ "gateMs": 1586.6503329999978,
"gateSkipped": false,
- "ttftMs": 19430.05587500002,
- "modelMs": 17605.114082000044,
- "toolMs": 0.018040999944787472,
- "totalMs": 19976.226081,
+ "ttftMs": 6799.384041999991,
+ "modelMs": 5582.004083000007,
+ "toolMs": 0.027624999987892807,
+ "totalMs": 7168.682040999993,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3282.5496250000433,
+ "roundMs": 3713.059416000033,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.006624999979976565
+ "execMs": 0.027624999987892807
}
]
},
{
- "roundMs": 8411.913791000028,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.011415999964810908
- }
- ]
- },
- {
- "roundMs": 5910.650665999972,
- "firstDeltaMs": 5364.099208,
+ "roundMs": 1868.9446669999743,
+ "firstDeltaMs": 1499.1694170000264,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "required tool set_look was never called",
- "mutating tools called: add_effect, set_color_grade"
- ],
+ "notes": [],
"trace": [
{
"name": "add_effect",
"round": 0,
- "atMs": 5619.916749999975,
- "execMs": 0.0035829999833367765
- },
- {
- "name": "set_color_grade",
- "round": 1,
- "atMs": 12105.853499999968,
- "execMs": 0.0047500000218860805
+ "atMs": 7529.667957999976,
+ "execMs": 0.006499999959487468
}
],
"timings": {
- "gateMs": 1847.9119580000406,
+ "gateMs": 1850.8777920000139,
"gateSkipped": false,
- "ttftMs": 18327.751958999957,
- "modelMs": 17019.82362499996,
- "toolMs": 0.008333000005222857,
- "totalMs": 18867.743916000007,
+ "ttftMs": 9546.061499999953,
+ "modelMs": 8049.621584000066,
+ "toolMs": 0.006499999959487468,
+ "totalMs": 9900.50587600004,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3771.7212500000023,
+ "roundMs": 5678.517584000016,
"firstDeltaMs": null,
"calls": [
{
"name": "add_effect",
- "execMs": 0.0035829999833367765
- }
- ]
- },
- {
- "roundMs": 6485.910916999972,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.0047500000218860805
+ "execMs": 0.006499999959487468
}
]
},
{
- "roundMs": 6762.191457999987,
- "firstDeltaMs": 6221.872791000002,
+ "roundMs": 2371.1040000000503,
+ "firstDeltaMs": 2016.373750000028,
"calls": []
}
]
@@ -13739,27 +13423,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1844,
- "p95": 2362,
- "mean": 1923,
+ "p50": 1771,
+ "p95": 1857,
+ "mean": 1623,
"n": 3
},
"ttftMs": {
- "p50": 8685,
- "p95": 9943,
- "mean": 9099,
+ "p50": 8805,
+ "p95": 9647,
+ "mean": 8759,
"n": 3
},
"totalMs": {
- "p50": 9295,
- "p95": 10270,
- "mean": 9534,
+ "p50": 9129,
+ "p95": 10036,
+ "mean": 9143,
"n": 3
},
"modelMs": {
- "p50": 7452,
- "p95": 8707,
- "mean": 7611,
+ "p50": 7358,
+ "p95": 8179,
+ "mean": 7519,
"n": 3
},
"toolMs": {
@@ -13782,43 +13466,43 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 6726.766624999989,
- "execMs": 0.07050000003073364
+ "atMs": 7179.70187499997,
+ "execMs": 0.3107080000336282
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6726.842833000061,
- "execMs": 0.001999999978579581
+ "atMs": 7180.0509589999565,
+ "execMs": 0.010167000000365078
}
],
"timings": {
- "gateMs": 2362.365208000003,
+ "gateMs": 1856.5103339999914,
"gateSkipped": false,
- "ttftMs": 8669.907000000065,
- "modelMs": 6673.650333999889,
- "toolMs": 0.07250000000931323,
- "totalMs": 9036.088041999901,
+ "ttftMs": 9646.994459000009,
+ "modelMs": 8179.462999999989,
+ "toolMs": 0.3208750000339933,
+ "totalMs": 10036.294209000014,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4363.653416999965,
+ "roundMs": 5322.471791999997,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.07050000003073364
+ "execMs": 0.3107080000336282
},
{
"name": "set_animation",
- "execMs": 0.001999999978579581
+ "execMs": 0.010167000000365078
}
]
},
{
- "roundMs": 2309.996916999924,
- "firstDeltaMs": 1943.0552089998964,
+ "roundMs": 2856.991207999992,
+ "firstDeltaMs": 2466.906541000004,
"calls": []
}
]
@@ -13832,43 +13516,43 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 6918.318334000069,
- "execMs": 0.010540999937802553
+ "atMs": 6933.767999999982,
+ "execMs": 0.008583999995607883
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6918.340124999988,
- "execMs": 0.008625000016763806
+ "atMs": 6933.787666999968,
+ "execMs": 0.007000000041443855
}
],
"timings": {
- "gateMs": 1563.0101250000298,
+ "gateMs": 1770.7273339999956,
"gateSkipped": false,
- "ttftMs": 9942.531709000003,
- "modelMs": 8706.91412500001,
- "toolMs": 0.01916599995456636,
- "totalMs": 10269.943415999995,
+ "ttftMs": 8805.173416999984,
+ "modelMs": 7358.433624999947,
+ "toolMs": 0.015584000037051737,
+ "totalMs": 9129.17654299998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5355.059374999953,
+ "roundMs": 5162.858749999956,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.010540999937802553
+ "execMs": 0.008583999995607883
},
{
"name": "set_animation",
- "execMs": 0.008625000016763806
+ "execMs": 0.007000000041443855
}
]
},
{
- "roundMs": 3351.854750000057,
- "firstDeltaMs": 3024.152667000075,
+ "roundMs": 2195.574874999991,
+ "firstDeltaMs": 1871.3499999999767,
"calls": []
}
]
@@ -13882,43 +13566,43 @@
{
"name": "set_transition",
"round": 0,
- "atMs": 6772.769582999987,
- "execMs": 0.0033340000081807375
+ "atMs": 6182.254167000065,
+ "execMs": 0.0035409999545663595
},
{
"name": "set_animation",
"round": 0,
- "atMs": 6772.776332999929,
- "execMs": 0.00229199999012053
+ "atMs": 6182.260791999986,
+ "execMs": 0.002250000019557774
}
],
"timings": {
- "gateMs": 1843.6062079999829,
+ "gateMs": 1242.0237079999642,
"gateSkipped": false,
- "ttftMs": 8685.332791999914,
- "modelMs": 7451.57574999996,
- "toolMs": 0.005625999998301268,
- "totalMs": 9295.187583999941,
+ "ttftMs": 7824.448542000027,
+ "modelMs": 7020.532000000065,
+ "toolMs": 0.005790999974124134,
+ "totalMs": 8262.561499000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4928.900040999986,
+ "roundMs": 4940.135750000074,
"firstDeltaMs": null,
"calls": [
{
"name": "set_transition",
- "execMs": 0.0033340000081807375
+ "execMs": 0.0035409999545663595
},
{
"name": "set_animation",
- "execMs": 0.00229199999012053
+ "execMs": 0.002250000019557774
}
]
},
{
- "roundMs": 2522.6757089999737,
- "firstDeltaMs": 1912.544208999956,
+ "roundMs": 2080.3962499999907,
+ "firstDeltaMs": 1642.1745830000145,
"calls": []
}
]
@@ -13929,1011 +13613,1072 @@
{
"name": "filler-words-cut-with-editor",
"bucket": "multi-tool",
- "passRate": 0,
+ "passRate": 1,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 1890,
+ "p95": 2241,
+ "mean": 1905,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 71405,
+ "p95": 97593,
+ "mean": 79403,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 72780,
+ "p95": 98145,
+ "mean": 80257,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 70888,
+ "p95": 95902,
+ "mean": 78349,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 2,
+ "p95": 4,
+ "mean": 3,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 21,
+ "max": 22
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: refine_speech_cuts"
- ],
+ "notes": [],
"trace": [
{
"name": "list_skills",
"round": 0,
- "atMs": 7683.05795799999,
- "execMs": 0.09441699995659292
+ "atMs": 4709.540124999941,
+ "execMs": 0.3462089999811724
},
{
"name": "read_skill",
"round": 1,
- "atMs": 10682.909624999971,
- "execMs": 0.02795799996238202
+ "atMs": 6286.163624999928,
+ "execMs": 0.025249999947845936
},
{
- "name": "read_skill",
+ "name": "detect_silence",
"round": 2,
- "atMs": 12442.984999999986,
- "execMs": 0.0712500000372529
+ "atMs": 14548.63254199992,
+ "execMs": 0.08125000004656613
},
{
- "name": "detect_silence",
+ "name": "listen_audio",
"round": 3,
- "atMs": 15394.043042000034,
- "execMs": 0.06374999997206032
+ "atMs": 18015.219583999948,
+ "execMs": 0.019709000014699996
},
{
- "name": "listen_audio",
+ "name": "detect_silence",
"round": 4,
- "atMs": 19364.84866700007,
- "execMs": 0.0320830000564456
+ "atMs": 21327.76579199999,
+ "execMs": 0.032832999946549535
},
{
"name": "split_at",
"round": 5,
- "atMs": 34495.08345799998,
- "execMs": 1.5357920000096783
+ "atMs": 29899.176833999925,
+ "execMs": 1.1913340000901371
+ },
+ {
+ "name": "get_state",
+ "round": 6,
+ "atMs": 34276.21520899993,
+ "execMs": 0.2707500000251457
},
{
"name": "split_at",
- "round": 5,
- "atMs": 34496.634500000044,
- "execMs": 0.043500000028871
+ "round": 7,
+ "atMs": 35986.92954199994,
+ "execMs": 0.11195799999404699
+ },
+ {
+ "name": "get_state",
+ "round": 8,
+ "atMs": 37537.965084000025,
+ "execMs": 0.6955409999936819
},
{
"name": "split_at",
- "round": 5,
- "atMs": 34496.68062500004,
- "execMs": 0.024458999978378415
+ "round": 9,
+ "atMs": 39836.15445899998,
+ "execMs": 0.03600000008009374
},
{
"name": "get_state",
- "round": 6,
- "atMs": 36663.799875000026,
- "execMs": 0.04266700008884072
+ "round": 10,
+ "atMs": 41468.20566700003,
+ "execMs": 0.0623749999795109
},
{
"name": "delete_item",
- "round": 7,
- "atMs": 41229.59454199998,
- "execMs": 0.5931250000139698
+ "round": 11,
+ "atMs": 46939.39495899994,
+ "execMs": 0.5352499999571592
},
{
"name": "delete_item",
- "round": 7,
- "atMs": 41230.200333000044,
- "execMs": 0.038791999919340014
+ "round": 12,
+ "atMs": 48909.406624999945,
+ "execMs": 0.08924999996088445
},
{
"name": "get_state",
- "round": 8,
- "atMs": 42797.80383300001,
- "execMs": 0.10358400002587587
+ "round": 13,
+ "atMs": 50468.75416699995,
+ "execMs": 0.3376660000067204
},
{
"name": "refine_speech_cuts",
- "round": 9,
- "atMs": 51944.85837500007,
- "execMs": 0.03445800009649247
- },
- {
- "name": "delete_cue",
- "round": 9,
- "atMs": 51944.903249999974,
- "execMs": 0.34058300009928644
+ "round": 14,
+ "atMs": 56088.408874999965,
+ "execMs": 0.29262500000186265
},
{
- "name": "update_cue",
- "round": 9,
- "atMs": 51945.24974999996,
- "execMs": 0.04091600002720952
+ "name": "subtitles_generate",
+ "round": 15,
+ "atMs": 58044.92395899992,
+ "execMs": 0.008999999961815774
},
{
- "name": "update_cue",
- "round": 9,
- "atMs": 51945.29362500005,
- "execMs": 0.018874999950639904
+ "name": "get_state",
+ "round": 16,
+ "atMs": 59837.69308400003,
+ "execMs": 0.11066699994262308
},
{
- "name": "update_cue",
- "round": 9,
- "atMs": 51945.31458300003,
- "execMs": 0.005750000011175871
+ "name": "subtitles_remove_track",
+ "round": 17,
+ "atMs": 63787.036208999925,
+ "execMs": 0.02304200001526624
},
{
- "name": "update_cue",
- "round": 9,
- "atMs": 51945.32245800004,
- "execMs": 0.006457999930717051
+ "name": "subtitles_generate",
+ "round": 18,
+ "atMs": 65595.47333399998,
+ "execMs": 0.008750000037252903
},
{
"name": "get_state",
- "round": 10,
- "atMs": 54474.194666999974,
- "execMs": 0.01816600002348423
+ "round": 19,
+ "atMs": 67367.14874999993,
+ "execMs": 0.03383400000166148
}
],
"timings": {
- "gateMs": 1455.7236249999842,
+ "gateMs": 1583.4500420000404,
"gateSkipped": false,
- "ttftMs": 56306.062000000034,
- "modelMs": 55558.090540999896,
- "toolMs": 3.1365830001886934,
- "totalMs": 57016.95074900007,
+ "ttftMs": 69212.36066699994,
+ "modelMs": 68257.65945700044,
+ "toolMs": 4.3132929999846965,
+ "totalMs": 69845.42279200046,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6226.593125000014,
+ "roundMs": 3125.7304579999764,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.09441699995659292
+ "execMs": 0.3462089999811724
}
]
},
{
- "roundMs": 2999.718583000009,
+ "roundMs": 1576.2286250001052,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.02795799996238202
+ "execMs": 0.025249999947845936
}
]
},
{
- "roundMs": 1759.9774999999208,
+ "roundMs": 8262.388000000035,
"firstDeltaMs": null,
"calls": [
{
- "name": "read_skill",
- "execMs": 0.0712500000372529
+ "name": "detect_silence",
+ "execMs": 0.08125000004656613
}
]
},
{
- "roundMs": 2950.9153749999823,
+ "roundMs": 3466.4622080000117,
"firstDeltaMs": null,
"calls": [
{
- "name": "detect_silence",
- "execMs": 0.06374999997206032
+ "name": "listen_audio",
+ "execMs": 0.019709000014699996
}
]
},
{
- "roundMs": 3970.671999999904,
+ "roundMs": 3312.4846670000115,
"firstDeltaMs": null,
"calls": [
{
- "name": "listen_audio",
- "execMs": 0.0320830000564456
+ "name": "detect_silence",
+ "execMs": 0.032832999946549535
}
]
},
{
- "roundMs": 15130.113250000053,
+ "roundMs": 8571.305625000037,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 1.5357920000096783
- },
+ "execMs": 1.1913340000901371
+ }
+ ]
+ },
+ {
+ "roundMs": 4375.798457999947,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.2707500000251457
+ }
+ ]
+ },
+ {
+ "roundMs": 1710.3841249999823,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "split_at",
- "execMs": 0.043500000028871
- },
+ "execMs": 0.11195799999404699
+ }
+ ]
+ },
+ {
+ "roundMs": 1550.826042000088,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.6955409999936819
+ }
+ ]
+ },
+ {
+ "roundMs": 2297.4393750000745,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "split_at",
- "execMs": 0.024458999978378415
+ "execMs": 0.03600000008009374
}
]
},
{
- "roundMs": 2167.039416000014,
+ "roundMs": 1631.9681669999845,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.04266700008884072
+ "execMs": 0.0623749999795109
}
]
},
{
- "roundMs": 4565.704665999976,
+ "roundMs": 5471.0619169999845,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.5931250000139698
- },
+ "execMs": 0.5352499999571592
+ }
+ ]
+ },
+ {
+ "roundMs": 1969.409915999975,
+ "firstDeltaMs": null,
+ "calls": [
{
"name": "delete_item",
- "execMs": 0.038791999919340014
+ "execMs": 0.08924999996088445
}
]
},
{
- "roundMs": 1567.4930419999873,
+ "roundMs": 1559.1823750000913,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.10358400002587587
+ "execMs": 0.3376660000067204
}
]
},
{
- "roundMs": 9146.876999999979,
+ "roundMs": 5619.19441700005,
"firstDeltaMs": null,
"calls": [
{
"name": "refine_speech_cuts",
- "execMs": 0.03445800009649247
- },
- {
- "name": "delete_cue",
- "execMs": 0.34058300009928644
- },
+ "execMs": 0.29262500000186265
+ }
+ ]
+ },
+ {
+ "roundMs": 1956.1869169999845,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "update_cue",
- "execMs": 0.04091600002720952
- },
+ "name": "subtitles_generate",
+ "execMs": 0.008999999961815774
+ }
+ ]
+ },
+ {
+ "roundMs": 1792.736416000058,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "update_cue",
- "execMs": 0.018874999950639904
- },
+ "name": "get_state",
+ "execMs": 0.11066699994262308
+ }
+ ]
+ },
+ {
+ "roundMs": 3949.1901249999646,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "update_cue",
- "execMs": 0.005750000011175871
- },
+ "name": "subtitles_remove_track",
+ "execMs": 0.02304200001526624
+ }
+ ]
+ },
+ {
+ "roundMs": 1808.3865000000224,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "update_cue",
- "execMs": 0.006457999930717051
+ "name": "subtitles_generate",
+ "execMs": 0.008750000037252903
}
]
},
{
- "roundMs": 2528.825625000056,
+ "roundMs": 1771.6349580000388,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.01816600002348423
+ "execMs": 0.03383400000166148
}
]
},
{
- "roundMs": 2544.1609590000007,
- "firstDeltaMs": 1831.838333999971,
+ "roundMs": 2479.660166000016,
+ "firstDeltaMs": 1845.1505829999223,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: refine_speech_cuts"
- ],
+ "notes": [],
"trace": [
{
"name": "list_skills",
"round": 0,
- "atMs": 4911.447374999989,
- "execMs": 0.008000000030733645
+ "atMs": 5643.351792000001,
+ "execMs": 0.04458300000987947
},
{
"name": "read_skill",
"round": 1,
- "atMs": 9256.017166999984,
- "execMs": 0.02250000007916242
+ "atMs": 7226.9671249999665,
+ "execMs": 0.021167000057175756
},
{
"name": "detect_silence",
"round": 2,
- "atMs": 13079.701833000057,
- "execMs": 0.006333999917842448
+ "atMs": 10362.47091699997,
+ "execMs": 0.05087500007357448
},
{
"name": "listen_audio",
"round": 3,
- "atMs": 15228.475749999983,
- "execMs": 0.005790999974124134
+ "atMs": 13501.942207999993,
+ "execMs": 0.04379199992399663
},
{
"name": "split_at",
"round": 4,
- "atMs": 24462.05816700007,
- "execMs": 0.1051669999724254
+ "atMs": 26430.813083000015,
+ "execMs": 0.05149999994318932
},
{
- "name": "get_state",
+ "name": "split_at",
"round": 5,
- "atMs": 26145.191833000048,
- "execMs": 0.12650000001303852
+ "atMs": 31244.123541999958,
+ "execMs": 0.07016699993982911
},
{
"name": "split_at",
"round": 6,
- "atMs": 28399.89716699999,
- "execMs": 0.03479100007098168
+ "atMs": 32800.950333000044,
+ "execMs": 0.08341699989978224
},
{
- "name": "split_at",
+ "name": "get_state",
"round": 7,
- "atMs": 29935.271750000073,
- "execMs": 0.17125000001396984
+ "atMs": 34480.02354199998,
+ "execMs": 0.2977080000564456
},
{
- "name": "get_state",
+ "name": "delete_item",
"round": 8,
- "atMs": 31559.44883300003,
- "execMs": 0.05987500003539026
+ "atMs": 40248.53370799997,
+ "execMs": 0.019416000111959875
},
{
"name": "delete_item",
"round": 9,
- "atMs": 35183.53812500008,
- "execMs": 0.08154200005810708
+ "atMs": 41860.26341699995,
+ "execMs": 0.065874999971129
},
{
"name": "get_state",
"round": 10,
- "atMs": 36698.171333000064,
- "execMs": 0.021583000081591308
+ "atMs": 43561.252791999956,
+ "execMs": 0.05291700002271682
},
{
- "name": "delete_item",
+ "name": "refine_speech_cuts",
"round": 11,
- "atMs": 39922.40220800007,
- "execMs": 0.14624999999068677
+ "atMs": 48903.77116699994,
+ "execMs": 0.060459000058472157
},
{
"name": "get_state",
"round": 12,
- "atMs": 41426.93541699997,
- "execMs": 0.2628329999279231
+ "atMs": 51908.44004200003,
+ "execMs": 0.014958000043407083
},
{
- "name": "refine_speech_cuts",
+ "name": "delete_cue",
"round": 13,
- "atMs": 44173.62070800003,
- "execMs": 0.043625000049360096
+ "atMs": 58190.999542000005,
+ "execMs": 0.4126660000765696
},
{
- "name": "delete_cue",
+ "name": "update_cue",
"round": 14,
- "atMs": 51567.14950000006,
- "execMs": 0.09087499999441206
+ "atMs": 60010.64129199996,
+ "execMs": 0.13916699995752424
},
{
"name": "update_cue",
"round": 15,
- "atMs": 53388.728458,
- "execMs": 0.06416700000409037
+ "atMs": 61743.22704200004,
+ "execMs": 0.094459000043571
},
{
"name": "update_cue",
"round": 16,
- "atMs": 55149.73920800001,
- "execMs": 0.06695800006855279
+ "atMs": 63421.286208000034,
+ "execMs": 0.06508299999404699
},
{
"name": "update_cue",
"round": 17,
- "atMs": 57019.41500000004,
- "execMs": 0.007541000028140843
- },
- {
- "name": "update_cue",
- "round": 18,
- "atMs": 58680.656000000075,
- "execMs": 0.03337499999906868
+ "atMs": 66149.14645799994,
+ "execMs": 0.029042000067420304
},
{
"name": "get_state",
- "round": 19,
- "atMs": 63427.42599999998,
- "execMs": 0.07466699997894466
+ "round": 18,
+ "atMs": 68980.80045800004,
+ "execMs": 0.0354580000275746
}
],
"timings": {
- "gateMs": 1363.9116249999497,
+ "gateMs": 1889.5865419999463,
"gateSkipped": false,
- "ttftMs": 65837.64591700002,
- "modelMs": 65191.24637399998,
- "toolMs": 1.433624000288546,
- "totalMs": 66556.59162300022,
+ "ttftMs": 71404.53208299994,
+ "modelMs": 70888.48383400007,
+ "toolMs": 1.6527090002782643,
+ "totalMs": 72779.7230850003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3547.177207999979,
+ "roundMs": 3753.502833000035,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.008000000030733645
+ "execMs": 0.04458300000987947
}
]
},
{
- "roundMs": 4344.529708000016,
+ "roundMs": 1583.5035420000786,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.02250000007916242
+ "execMs": 0.021167000057175756
}
]
},
{
- "roundMs": 3823.634874999989,
+ "roundMs": 3135.427875000052,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.006333999917842448
+ "execMs": 0.05087500007357448
}
]
},
{
- "roundMs": 2148.7535830000415,
+ "roundMs": 3139.340749999974,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.005790999974124134
+ "execMs": 0.04379199992399663
}
]
},
{
- "roundMs": 9233.559374999953,
+ "roundMs": 12928.753041999997,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.1051669999724254
- }
- ]
- },
- {
- "roundMs": 1682.9828330000164,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.12650000001303852
+ "execMs": 0.05149999994318932
}
]
},
{
- "roundMs": 2254.5310000000754,
+ "roundMs": 4813.203333999962,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.03479100007098168
+ "execMs": 0.07016699993982911
}
]
},
{
- "roundMs": 1535.295374999987,
+ "roundMs": 1556.6848340000724,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.17125000001396984
+ "execMs": 0.08341699989978224
}
]
},
{
- "roundMs": 1623.9257920000236,
+ "roundMs": 1678.9082919999491,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.05987500003539026
+ "execMs": 0.2977080000564456
}
]
},
{
- "roundMs": 3623.9565419999417,
+ "roundMs": 5768.166250000009,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.08154200005810708
+ "execMs": 0.019416000111959875
}
]
},
{
- "roundMs": 1514.4937500000233,
+ "roundMs": 1611.6839169999585,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.021583000081591308
+ "name": "delete_item",
+ "execMs": 0.065874999971129
}
]
},
{
- "roundMs": 3224.163791000028,
+ "roundMs": 1700.8737919999985,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.14624999999068677
+ "name": "get_state",
+ "execMs": 0.05291700002271682
}
]
},
{
- "roundMs": 1504.075375000015,
+ "roundMs": 5342.401249999995,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.2628329999279231
+ "name": "refine_speech_cuts",
+ "execMs": 0.060459000058472157
}
]
},
{
- "roundMs": 2746.218957999954,
+ "roundMs": 3004.5680410000496,
"firstDeltaMs": null,
"calls": [
{
- "name": "refine_speech_cuts",
- "execMs": 0.043625000049360096
+ "name": "get_state",
+ "execMs": 0.014958000043407083
}
]
},
{
- "roundMs": 7393.319708999945,
+ "roundMs": 6282.414499999955,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_cue",
- "execMs": 0.09087499999441206
+ "execMs": 0.4126660000765696
}
]
},
{
- "roundMs": 1821.402584000025,
+ "roundMs": 1819.1552909999155,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.06416700000409037
+ "execMs": 0.13916699995752424
}
]
},
{
- "roundMs": 1760.8830830000807,
+ "roundMs": 1732.3772500000196,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.06695800006855279
+ "execMs": 0.094459000043571
}
]
},
{
- "roundMs": 1869.5737499999814,
+ "roundMs": 1677.8888329999754,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.007541000028140843
+ "execMs": 0.06508299999404699
}
]
},
{
- "roundMs": 1661.2008749999804,
+ "roundMs": 2727.746999999974,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.03337499999906868
+ "execMs": 0.029042000067420304
}
]
},
{
- "roundMs": 4746.47412499995,
+ "roundMs": 2831.5766250001034,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.07466699997894466
+ "execMs": 0.0354580000275746
}
]
},
{
- "roundMs": 3131.0940829999745,
- "firstDeltaMs": 2410.0891660000198,
+ "roundMs": 3800.3065829999978,
+ "firstDeltaMs": 2423.6167499999283,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: refine_speech_cuts, subtitles_generate"
- ],
+ "notes": [],
"trace": [
{
"name": "list_skills",
"round": 0,
- "atMs": 8687.520625000005,
- "execMs": 0.11004199995659292
+ "atMs": 6081.855667000054,
+ "execMs": 0.09516699996311218
},
{
"name": "read_skill",
"round": 1,
- "atMs": 10654.37708399992,
- "execMs": 0.007707999902777374
+ "atMs": 7839.606541999965,
+ "execMs": 0.0216669999063015
},
{
"name": "detect_silence",
"round": 2,
- "atMs": 14716.376874999958,
- "execMs": 0.04895800002850592
+ "atMs": 14389.324957999983,
+ "execMs": 0.06387500010896474
},
{
"name": "listen_audio",
"round": 3,
- "atMs": 16853.913416999974,
- "execMs": 0.030124999932013452
+ "atMs": 22638.66441700002,
+ "execMs": 0.03791700000874698
},
{
"name": "split_at",
"round": 4,
- "atMs": 26794.15179199993,
- "execMs": 0.2713329999241978
+ "atMs": 38330.27916699997,
+ "execMs": 0.293582999962382
},
{
"name": "get_state",
"round": 5,
- "atMs": 28813.36366699997,
- "execMs": 0.22212500008754432
+ "atMs": 40730.30591700005,
+ "execMs": 0.28529200004413724
},
{
"name": "split_at",
"round": 6,
- "atMs": 30518.33029199997,
- "execMs": 0.02404199994634837
+ "atMs": 42608.86345800001,
+ "execMs": 0.15150000003632158
},
{
"name": "get_state",
"round": 7,
- "atMs": 32019.205624999944,
- "execMs": 0.09162500000093132
+ "atMs": 48689.862250000006,
+ "execMs": 0.0809159999480471
},
{
"name": "split_at",
"round": 8,
- "atMs": 33878.3395,
- "execMs": 0.07854200003203005
+ "atMs": 50919.13145800005,
+ "execMs": 0.03912500001024455
},
{
"name": "get_state",
"round": 9,
- "atMs": 35702.544125000015,
- "execMs": 0.05245899991132319
+ "atMs": 54287.60383299994,
+ "execMs": 0.0652919999556616
},
{
"name": "delete_item",
"round": 10,
- "atMs": 38282.80637499993,
- "execMs": 0.02941600000485778
+ "atMs": 59258.05833300005,
+ "execMs": 0.17741699994076043
},
{
- "name": "get_state",
+ "name": "delete_item",
"round": 11,
- "atMs": 39878.307042,
- "execMs": 0.10520900005940348
+ "atMs": 61617.07024999999,
+ "execMs": 0.10412500007078052
},
{
- "name": "delete_item",
+ "name": "get_state",
"round": 12,
- "atMs": 41941.65641699999,
- "execMs": 0.02966599992942065
+ "atMs": 67837.85783300002,
+ "execMs": 0.07524999999441206
},
{
- "name": "get_state",
+ "name": "refine_speech_cuts",
"round": 13,
- "atMs": 43384.38958399999,
- "execMs": 0.17895800003316253
+ "atMs": 70859.098,
+ "execMs": 0.10333299997728318
},
{
- "name": "refine_speech_cuts",
+ "name": "get_state",
"round": 14,
- "atMs": 45997.593292000005,
- "execMs": 0.0040420000441372395
+ "atMs": 74379.11095799995,
+ "execMs": 0.06295800011139363
},
{
- "name": "subtitles_generate",
+ "name": "delete_cue",
"round": 15,
- "atMs": 51881.96345899999,
- "execMs": 0.061834000051021576
+ "atMs": 80318.545208,
+ "execMs": 0.104583999956958
},
{
- "name": "delete_cue",
+ "name": "update_cue",
"round": 16,
- "atMs": 57801.716041999985,
- "execMs": 0.1644170000217855
+ "atMs": 84913.05741699995,
+ "execMs": 0.330333000048995
},
{
"name": "update_cue",
"round": 17,
- "atMs": 59647.65420899994,
- "execMs": 0.11291600007098168
+ "atMs": 87405.65720799996,
+ "execMs": 0.025957999983802438
},
{
"name": "update_cue",
"round": 18,
- "atMs": 61218.34054200002,
- "execMs": 0.22774999996181577
+ "atMs": 89288.55799999996,
+ "execMs": 0.06462499999906868
},
{
"name": "update_cue",
"round": 19,
- "atMs": 62819.64741700003,
- "execMs": 0.06595800002105534
- },
- {
- "name": "update_cue",
- "round": 20,
- "atMs": 67232.61174999992,
- "execMs": 0.037040999974124134
+ "atMs": 91285.93804200005,
+ "execMs": 0.02354099997319281
},
{
"name": "get_state",
- "round": 21,
- "atMs": 69300.00008399994,
- "execMs": 0.10891700000502169
+ "round": 20,
+ "atMs": 92937.84183299995,
+ "execMs": 0.05791699991095811
}
],
"timings": {
- "gateMs": 2364.752499999944,
+ "gateMs": 2240.584375000093,
"gateSkipped": false,
- "ttftMs": 73399.78579200001,
- "modelMs": 72038.76733499998,
- "toolMs": 2.063082999899052,
- "totalMs": 74405.58291799983,
+ "ttftMs": 97592.77120800002,
+ "modelMs": 95902.11512600014,
+ "toolMs": 2.2643749999115244,
+ "totalMs": 98144.96387600014,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6322.600084000034,
+ "roundMs": 3840.73987499997,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.11004199995659292
+ "execMs": 0.09516699996311218
}
]
},
{
- "roundMs": 1966.652417000034,
+ "roundMs": 1757.5490840000566,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.007707999902777374
+ "execMs": 0.0216669999063015
}
]
},
{
- "roundMs": 4061.9343330000993,
+ "roundMs": 6549.608165999991,
"firstDeltaMs": null,
"calls": [
{
"name": "detect_silence",
- "execMs": 0.04895800002850592
+ "execMs": 0.06387500010896474
}
]
},
{
- "roundMs": 2137.4172920000274,
+ "roundMs": 8249.217416999978,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.030124999932013452
+ "execMs": 0.03791700000874698
}
]
},
{
- "roundMs": 9940.129791999934,
+ "roundMs": 15691.462082999991,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.2713329999241978
+ "execMs": 0.293582999962382
}
]
},
{
- "roundMs": 2018.8656249999767,
+ "roundMs": 2399.661499999929,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.22212500008754432
+ "execMs": 0.28529200004413724
}
]
},
{
- "roundMs": 1704.7009160000598,
+ "roundMs": 1878.201417000033,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.02404199994634837
+ "execMs": 0.15150000003632158
}
]
},
{
- "roundMs": 1500.5970830000006,
+ "roundMs": 6080.371666999999,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.09162500000093132
+ "execMs": 0.0809159999480471
}
]
},
{
- "roundMs": 1858.933791999938,
+ "roundMs": 2229.107958000037,
"firstDeltaMs": null,
"calls": [
{
"name": "split_at",
- "execMs": 0.07854200003203005
+ "execMs": 0.03912500001024455
}
]
},
{
- "roundMs": 1824.061333999969,
+ "roundMs": 3368.388791999896,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.05245899991132319
+ "execMs": 0.0652919999556616
}
]
},
{
- "roundMs": 2580.1680000000633,
+ "roundMs": 4970.291958000045,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.02941600000485778
- }
- ]
- },
- {
- "roundMs": 1595.4269169999752,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.10520900005940348
+ "execMs": 0.17741699994076043
}
]
},
{
- "roundMs": 2063.2001669999445,
+ "roundMs": 2358.770583000034,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_item",
- "execMs": 0.02966599992942065
+ "execMs": 0.10412500007078052
}
]
},
{
- "roundMs": 1442.6621250000317,
+ "roundMs": 6220.634834000026,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.17895800003316253
+ "execMs": 0.07524999999441206
}
]
},
{
- "roundMs": 2612.9472909999313,
+ "roundMs": 3021.109584000078,
"firstDeltaMs": null,
"calls": [
{
"name": "refine_speech_cuts",
- "execMs": 0.0040420000441372395
+ "execMs": 0.10333299997728318
}
]
},
{
- "roundMs": 5884.324957999983,
+ "roundMs": 3519.8118339999346,
"firstDeltaMs": null,
"calls": [
{
- "name": "subtitles_generate",
- "execMs": 0.061834000051021576
+ "name": "get_state",
+ "execMs": 0.06295800011139363
}
]
},
{
- "roundMs": 5919.456041999976,
+ "roundMs": 5939.302375000087,
"firstDeltaMs": null,
"calls": [
{
"name": "delete_cue",
- "execMs": 0.1644170000217855
+ "execMs": 0.104583999956958
}
]
},
{
- "roundMs": 1845.4605000000447,
+ "roundMs": 4594.340709000011,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.11291600007098168
+ "execMs": 0.330333000048995
}
]
},
{
- "roundMs": 1570.4338330000173,
+ "roundMs": 2492.2208330000285,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.22774999996181577
+ "execMs": 0.025957999983802438
}
]
},
{
- "roundMs": 1600.9985410000663,
+ "roundMs": 1882.830541000003,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.06595800002105534
+ "execMs": 0.06462499999906868
}
]
},
{
- "roundMs": 4412.824791999999,
+ "roundMs": 1997.2639160000253,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.037040999974124134
+ "execMs": 0.02354099997319281
}
]
},
{
- "roundMs": 2067.297208999982,
+ "roundMs": 1651.8375829999568,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.10891700000502169
+ "execMs": 0.05791699991095811
}
]
},
{
- "roundMs": 5107.674291999894,
- "firstDeltaMs": 4099.646457999945,
+ "roundMs": 5209.392417000025,
+ "firstDeltaMs": 4654.8282500000205,
"calls": []
}
]
@@ -14948,27 +14693,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1643,
- "p95": 1829,
- "mean": 1655,
+ "p50": 2000,
+ "p95": 2159,
+ "mean": 1822,
"n": 3
},
"ttftMs": {
- "p50": 6080,
- "p95": 6432,
- "mean": 5813,
+ "p50": 7021,
+ "p95": 7114,
+ "mean": 6946,
"n": 3
},
"totalMs": {
- "p50": 6606,
- "p95": 6820,
- "mean": 6216,
+ "p50": 7379,
+ "p95": 7399,
+ "mean": 7263,
"n": 3
},
"modelMs": {
- "p50": 4963,
- "p95": 4991,
- "mean": 4561,
+ "p50": 5380,
+ "p95": 6093,
+ "mean": 5441,
"n": 3
},
"toolMs": {
@@ -14991,33 +14736,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 3545.4290830000537,
- "execMs": 0.0032499999506399035
+ "atMs": 5102.406416999991,
+ "execMs": 0.024457999970763922
}
],
"timings": {
- "gateMs": 1493.3881670000264,
+ "gateMs": 2158.632500000065,
"gateSkipped": false,
- "ttftMs": 4926.878999999957,
- "modelMs": 3729.693250000011,
- "toolMs": 0.0032499999506399035,
- "totalMs": 5223.084666999988,
+ "ttftMs": 6703.288541999995,
+ "modelMs": 4850.772417000029,
+ "toolMs": 0.024457999970763922,
+ "totalMs": 7009.429375000065,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2051.8648750000866,
+ "roundMs": 2943.4654169999994,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.0032499999506399035
+ "execMs": 0.024457999970763922
}
]
},
{
- "roundMs": 1677.8283749999246,
- "firstDeltaMs": 1381.3997909999453,
+ "roundMs": 1907.3070000000298,
+ "firstDeltaMs": 1600.8166669999482,
"calls": []
}
]
@@ -15031,33 +14776,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 4166.564458000008,
- "execMs": 0.02604100003372878
+ "atMs": 5369.308791999938,
+ "execMs": 0.04633300006389618
}
],
"timings": {
- "gateMs": 1642.5119580000173,
+ "gateMs": 1999.5517499999842,
"gateSkipped": false,
- "ttftMs": 6079.799916999997,
- "modelMs": 4962.982000000076,
- "toolMs": 0.02604100003372878,
- "totalMs": 6605.519999000127,
+ "ttftMs": 7113.576416999917,
+ "modelMs": 5379.699500000104,
+ "toolMs": 0.04633300006389618,
+ "totalMs": 7379.297583000152,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2523.623166000005,
+ "roundMs": 3369.5696250000037,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.02604100003372878
+ "execMs": 0.04633300006389618
}
]
},
{
- "roundMs": 2439.3588340000715,
- "firstDeltaMs": 1913.1263749999925,
+ "roundMs": 2010.1298750001006,
+ "firstDeltaMs": 1744.182958999998,
"calls": []
}
]
@@ -15071,33 +14816,33 @@
{
"name": "subtitles_generate",
"round": 0,
- "atMs": 4683.590915999957,
- "execMs": 0.013000000035390258
+ "atMs": 4477.277167000109,
+ "execMs": 0.003790999995544553
}
],
"timings": {
- "gateMs": 1829.252542000031,
+ "gateMs": 1306.5645419999491,
"gateSkipped": false,
- "ttftMs": 6432.002332999953,
- "modelMs": 4991.195873999968,
- "toolMs": 0.013000000035390258,
- "totalMs": 6820.461416000035,
+ "ttftMs": 7021.223875000025,
+ "modelMs": 6092.609167000046,
+ "toolMs": 0.003790999995544553,
+ "totalMs": 7399.177499999991,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2854.1554160000524,
+ "roundMs": 3170.496083999984,
"firstDeltaMs": null,
"calls": [
{
"name": "subtitles_generate",
- "execMs": 0.013000000035390258
+ "execMs": 0.003790999995544553
}
]
},
{
- "roundMs": 2137.040457999916,
- "firstDeltaMs": 1748.341415999923,
+ "roundMs": 2922.113083000062,
+ "firstDeltaMs": 2543.931541000027,
"calls": []
}
]
@@ -15112,27 +14857,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1640,
- "p95": 1698,
- "mean": 1581,
+ "p50": 1759,
+ "p95": 1879,
+ "mean": 1722,
"n": 3
},
"ttftMs": {
- "p50": 9487,
- "p95": 10579,
- "mean": 9328,
+ "p50": 9107,
+ "p95": 12315,
+ "mean": 9902,
"n": 3
},
"totalMs": {
- "p50": 9865,
- "p95": 10939,
- "mean": 9651,
+ "p50": 9671,
+ "p95": 12667,
+ "mean": 10362,
"n": 3
},
"modelMs": {
- "p50": 8224,
- "p95": 9240,
- "mean": 8070,
+ "p50": 8143,
+ "p95": 10908,
+ "mean": 8640,
"n": 3
},
"toolMs": {
@@ -15142,8 +14887,8 @@
"n": 3
},
"rounds": {
- "mean": 2.7,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
@@ -15155,33 +14900,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 6281.896582999965,
- "execMs": 0.46287499996833503
+ "atMs": 6532.421709000017,
+ "execMs": 0.34208299999590963
}
],
"timings": {
- "gateMs": 1405.0089170000283,
+ "gateMs": 1878.998499999987,
"gateSkipped": false,
- "ttftMs": 7918.723541999934,
- "modelMs": 6745.314666999853,
- "toolMs": 0.46287499996833503,
- "totalMs": 8150.78645899985,
+ "ttftMs": 8284.53912500001,
+ "modelMs": 6868.328126000008,
+ "toolMs": 0.34208299999590963,
+ "totalMs": 8747.66870899999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4876.516958999913,
+ "roundMs": 4652.895584000042,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.46287499996833503
+ "execMs": 0.34208299999590963
}
]
},
{
- "roundMs": 1868.79770799994,
- "firstDeltaMs": 1636.2407499999972,
+ "roundMs": 2215.432541999966,
+ "firstDeltaMs": 1751.7392080000136,
"calls": []
}
]
@@ -15192,52 +14937,36 @@
"intent": "work",
"notes": [],
"trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 5581.305583999958,
- "execMs": 0.049291999894194305
- },
{
"name": "generate_scene",
- "round": 1,
- "atMs": 7698.535749999923,
- "execMs": 0.005124999908730388
+ "round": 0,
+ "atMs": 7182.046875,
+ "execMs": 0.043082999996840954
}
],
"timings": {
- "gateMs": 1640.4949590000324,
+ "gateMs": 1527.6081669999985,
"gateSkipped": false,
- "ttftMs": 9487.270416999934,
- "modelMs": 8224.18033299991,
- "toolMs": 0.05441699980292469,
- "totalMs": 9864.729708999745,
+ "ttftMs": 9106.606499999994,
+ "modelMs": 8142.909542000038,
+ "toolMs": 0.043082999996840954,
+ "totalMs": 9670.560792000033,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3940.3855419999454,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.049291999894194305
- }
- ]
- },
- {
- "roundMs": 2117.097165999934,
+ "roundMs": 5654.235084000044,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.005124999908730388
+ "execMs": 0.043082999996840954
}
]
},
{
- "roundMs": 2166.69762500003,
- "firstDeltaMs": 1788.6699999999255,
+ "roundMs": 2488.674457999994,
+ "firstDeltaMs": 1924.4537919999566,
"calls": []
}
]
@@ -15248,52 +14977,36 @@
"intent": "work",
"notes": [],
"trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 5701.083542000037,
- "execMs": 0.05224999994970858
- },
{
"name": "generate_scene",
- "round": 1,
- "atMs": 8865.078291999991,
- "execMs": 0.011249999981373549
+ "round": 0,
+ "atMs": 7301.422917000018,
+ "execMs": 0.017125000013038516
}
],
"timings": {
- "gateMs": 1698.4170419999864,
+ "gateMs": 1759.0470000000205,
"gateSkipped": false,
- "ttftMs": 10578.973667000071,
- "modelMs": 9240.490957999951,
- "toolMs": 0.06349999993108213,
- "totalMs": 10938.971499999869,
+ "ttftMs": 12315.128167000017,
+ "modelMs": 10907.975875000004,
+ "toolMs": 0.017125000013038516,
+ "totalMs": 12667.040000000037,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4002.324666999979,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.05224999994970858
- }
- ]
- },
- {
- "roundMs": 3163.8819580000127,
+ "roundMs": 5542.0927920000395,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.011249999981373549
+ "execMs": 0.017125000013038516
}
]
},
{
- "roundMs": 2074.2843329999596,
- "firstDeltaMs": 1713.8528749999823,
+ "roundMs": 5365.883082999964,
+ "firstDeltaMs": 5013.638665999984,
"calls": []
}
]
@@ -15308,27 +15021,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 2124,
- "p95": 2557,
- "mean": 2223,
+ "p50": 2052,
+ "p95": 2324,
+ "mean": 2096,
"n": 3
},
"ttftMs": {
- "p50": 10732,
- "p95": 11175,
- "mean": 9943,
+ "p50": 10116,
+ "p95": 10283,
+ "mean": 9541,
"n": 3
},
"totalMs": {
- "p50": 11087,
- "p95": 11558,
- "mean": 10282,
+ "p50": 10500,
+ "p95": 10831,
+ "mean": 9950,
"n": 3
},
"modelMs": {
- "p50": 8530,
- "p95": 9433,
- "mean": 8059,
+ "p50": 8176,
+ "p95": 8779,
+ "mean": 7854,
"n": 3
},
"toolMs": {
@@ -15351,33 +15064,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 9283.591709000058,
- "execMs": 0.46299999998882413
+ "atMs": 6282.333791999961,
+ "execMs": 0.30579200002830476
}
],
"timings": {
- "gateMs": 2124.4372079999885,
+ "gateMs": 1911.434957999969,
"gateSkipped": false,
- "ttftMs": 11174.629209000035,
- "modelMs": 9433.022624999983,
- "toolMs": 0.46299999998882413,
- "totalMs": 11557.92283299996,
+ "ttftMs": 8224.63062499999,
+ "modelMs": 6607.8267079999205,
+ "toolMs": 0.30579200002830476,
+ "totalMs": 8519.567457999918,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7158.805666,
+ "roundMs": 4370.72329200001,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.46299999998882413
+ "execMs": 0.30579200002830476
}
]
},
{
- "roundMs": 2274.216958999983,
- "firstDeltaMs": 1890.4574169999687,
+ "roundMs": 2237.103415999911,
+ "firstDeltaMs": 1941.9606659999117,
"calls": []
}
]
@@ -15391,33 +15104,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 5878.165999999968,
- "execMs": 0.06966699997428805
+ "atMs": 8233.136708000093,
+ "execMs": 0.04620800004340708
}
],
"timings": {
- "gateMs": 1987.3846250000643,
+ "gateMs": 2052.016125000082,
"gateSkipped": false,
- "ttftMs": 7922.246665999992,
- "modelMs": 6213.0728330001,
- "toolMs": 0.06966699997428805,
- "totalMs": 8200.527125000139,
+ "ttftMs": 10282.509791000048,
+ "modelMs": 8779.38525000005,
+ "toolMs": 0.04620800004340708,
+ "totalMs": 10831.447583000176,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3890.4603330000537,
+ "roundMs": 6180.987458000076,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.06966699997428805
+ "execMs": 0.04620800004340708
}
]
},
{
- "roundMs": 2322.6125000000466,
- "firstDeltaMs": 2043.9044160000049,
+ "roundMs": 2598.3977919999743,
+ "firstDeltaMs": 2049.2595420000143,
"calls": []
}
]
@@ -15431,33 +15144,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 8621.808416999993,
- "execMs": 0.006917000049725175
+ "atMs": 8251.312250000075,
+ "execMs": 0.025166999897919595
}
],
"timings": {
- "gateMs": 2557.221290999907,
+ "gateMs": 2324.4123339999933,
"gateSkipped": false,
- "ttftMs": 10732.021916999947,
- "modelMs": 8530.2687919999,
- "toolMs": 0.006917000049725175,
- "totalMs": 11087.496999999858,
+ "ttftMs": 10116.229209000012,
+ "modelMs": 8176.038625000045,
+ "toolMs": 0.025166999897919595,
+ "totalMs": 10500.476125999936,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6064.284833999933,
+ "roundMs": 5926.610166999977,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.006917000049725175
+ "execMs": 0.025166999897919595
}
]
},
{
- "roundMs": 2465.983957999968,
- "firstDeltaMs": 2110.1872079999885,
+ "roundMs": 2249.4284580000676,
+ "firstDeltaMs": 1864.857917000074,
"calls": []
}
]
@@ -15478,21 +15191,21 @@
"n": 3
},
"ttftMs": {
- "p50": 6170,
- "p95": 7065,
- "mean": 6326,
+ "p50": 8781,
+ "p95": 8931,
+ "mean": 8450,
"n": 3
},
"totalMs": {
- "p50": 6500,
- "p95": 7459,
- "mean": 6659,
+ "p50": 9118,
+ "p95": 9198,
+ "mean": 8855,
"n": 3
},
"modelMs": {
- "p50": 6500,
- "p95": 7459,
- "mean": 6659,
+ "p50": 9118,
+ "p95": 9198,
+ "mean": 8855,
"n": 3
},
"toolMs": {
@@ -15515,33 +15228,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 5181.186250000028,
- "execMs": 0.07483299996238202
+ "atMs": 6766.483082999941,
+ "execMs": 0.1133329999865964
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 7064.621291999938,
- "modelMs": 7458.592375000007,
- "toolMs": 0.07483299996238202,
- "totalMs": 7458.66720799997,
+ "ttftMs": 8781.262666999944,
+ "modelMs": 9117.792250000057,
+ "toolMs": 0.1133329999865964,
+ "totalMs": 9117.905583000043,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5180.675832999987,
+ "roundMs": 6765.96420799999,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.07483299996238202
+ "execMs": 0.1133329999865964
}
]
},
{
- "roundMs": 2277.916542000021,
- "firstDeltaMs": 1883.3518750000512,
+ "roundMs": 2351.8280420000665,
+ "firstDeltaMs": 2014.654291999992,
"calls": []
}
]
@@ -15555,33 +15268,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 3973.357541999896,
- "execMs": 0.03345800004899502
+ "atMs": 6579.776625000057,
+ "execMs": 0.04783399996813387
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 5744.041417,
- "modelMs": 6018.576374999946,
- "toolMs": 0.03345800004899502,
- "totalMs": 6018.609832999995,
+ "ttftMs": 8931.158042000025,
+ "modelMs": 9198.32716799993,
+ "toolMs": 0.04783399996813387,
+ "totalMs": 9198.375001999899,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3973.1824999999953,
+ "roundMs": 6579.411958999932,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.03345800004899502
+ "execMs": 0.04783399996813387
}
]
},
{
- "roundMs": 2045.3938749999506,
- "firstDeltaMs": 1770.5964579998981,
+ "roundMs": 2618.915208999999,
+ "firstDeltaMs": 2351.2976250000065,
"calls": []
}
]
@@ -15595,33 +15308,33 @@
{
"name": "generate_scene",
"round": 0,
- "atMs": 4192.011375000002,
- "execMs": 0.0059590000892058015
+ "atMs": 5722.564124999917,
+ "execMs": 0.01708300004247576
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 6170.377041999949,
- "modelMs": 6499.516916000051,
- "toolMs": 0.0059590000892058015,
- "totalMs": 6499.522875000141,
+ "ttftMs": 7636.145625000005,
+ "modelMs": 8248.831917000003,
+ "toolMs": 0.01708300004247576,
+ "totalMs": 8248.849000000046,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4191.944332999992,
+ "roundMs": 5722.270207999973,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_scene",
- "execMs": 0.0059590000892058015
+ "execMs": 0.01708300004247576
}
]
},
{
- "roundMs": 2307.5725830000592,
- "firstDeltaMs": 1978.3437919999706,
+ "roundMs": 2526.5617090000305,
+ "firstDeltaMs": 1913.5407500000438,
"calls": []
}
]
@@ -15636,27 +15349,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 2395,
- "p95": 2593,
- "mean": 2186,
+ "p50": 1886,
+ "p95": 2274,
+ "mean": 2007,
"n": 3
},
"ttftMs": {
- "p50": 9547,
- "p95": 13713,
- "mean": 10369,
+ "p50": 10574,
+ "p95": 11794,
+ "mean": 10490,
"n": 3
},
"totalMs": {
- "p50": 10087,
- "p95": 14133,
- "mean": 10870,
+ "p50": 11469,
+ "p95": 12311,
+ "mean": 11129,
"n": 3
},
"modelMs": {
- "p50": 8518,
- "p95": 11738,
- "mean": 8685,
+ "p50": 9583,
+ "p95": 10037,
+ "mean": 9122,
"n": 3
},
"toolMs": {
@@ -15666,8 +15379,8 @@
"n": 3
},
"rounds": {
- "mean": 1.7,
- "max": 3
+ "mean": 1,
+ "max": 1
}
},
"runs": [
@@ -15675,53 +15388,20 @@
"pass": true,
"intent": "work",
"notes": [],
- "trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 7971.526415999979,
- "execMs": 0.06633299996610731
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 9891.030291000032,
- "execMs": 0.022083999938331544
- }
- ],
+ "trace": [],
"timings": {
- "gateMs": 2395.005249999929,
+ "gateMs": 2273.987000000081,
"gateSkipped": false,
- "ttftMs": 13712.674790999969,
- "modelMs": 11737.587249999982,
- "toolMs": 0.08841699990443885,
- "totalMs": 14132.680916999816,
+ "ttftMs": 11793.638167000026,
+ "modelMs": 10037.193791000056,
+ "toolMs": 0,
+ "totalMs": 12311.180791000137,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5576.013500000001,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.06633299996610731
- }
- ]
- },
- {
- "roundMs": 1919.358375000069,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.022083999938331544
- }
- ]
- },
- {
- "roundMs": 4242.2153749999125,
- "firstDeltaMs": 3821.561457999982,
+ "roundMs": 10037.193791000056,
+ "firstDeltaMs": 9519.495915999985,
"calls": []
}
]
@@ -15733,18 +15413,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 1568.329792000004,
+ "gateMs": 1860.4365410000319,
"gateSkipped": false,
- "ttftMs": 9547.130207999959,
- "modelMs": 8518.479374999995,
+ "ttftMs": 9101.572000000044,
+ "modelMs": 7746.5170420000795,
"toolMs": 0,
- "totalMs": 10086.809167,
+ "totalMs": 9606.953583000111,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8518.479374999995,
- "firstDeltaMs": 7978.281083000009,
+ "roundMs": 7746.5170420000795,
+ "firstDeltaMs": 7240.8749170000665,
"calls": []
}
]
@@ -15756,18 +15436,18 @@
"notes": [],
"trace": [],
"timings": {
- "gateMs": 2593.2591250000987,
+ "gateMs": 1886.1008340000408,
"gateSkipped": false,
- "ttftMs": 7848.587249999982,
- "modelMs": 5798.053209000034,
+ "ttftMs": 10574.146415999974,
+ "modelMs": 9583.133957999991,
"toolMs": 0,
- "totalMs": 8391.312334000133,
+ "totalMs": 11469.234792000032,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5798.053209000034,
- "firstDeltaMs": 5254.91650000005,
+ "roundMs": 9583.133957999991,
+ "firstDeltaMs": 8687.837540999986,
"calls": []
}
]
@@ -15778,115 +15458,164 @@
{
"name": "clip-mentions-redo-their-shots",
"bucket": "multi-tool",
- "passRate": 0,
+ "passRate": 0.6666666666666666,
"budgetBreaches": [],
- "latency": null,
+ "latency": {
+ "gateMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 2
+ },
+ "ttftMs": {
+ "p50": 17631,
+ "p95": 29941,
+ "mean": 23786,
+ "n": 2
+ },
+ "totalMs": {
+ "p50": 18028,
+ "p95": 30376,
+ "mean": 24202,
+ "n": 2
+ },
+ "modelMs": {
+ "p50": 18028,
+ "p95": 30375,
+ "mean": 24201,
+ "n": 2
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 1,
+ "mean": 0,
+ "n": 2
+ },
+ "rounds": {
+ "mean": 5.5,
+ "max": 6
+ }
+ },
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: wait_for_renders"
- ],
+ "notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 4174.626083999989,
- "execMs": 0.4835420000599697
+ "atMs": 3956.589125000057,
+ "execMs": 0.4098750000121072
},
{
"name": "watch_video",
"round": 1,
- "atMs": 7861.809499999974,
- "execMs": 0.037124999915249646
+ "atMs": 6528.679208999965,
+ "execMs": 0.011749999946914613
},
{
"name": "watch_video",
"round": 1,
- "atMs": 7861.906499999925,
- "execMs": 0.003999999957159162
+ "atMs": 6528.697000000044,
+ "execMs": 0.0005420000525191426
},
{
- "name": "regenerate_shot",
+ "name": "list_skills",
"round": 2,
- "atMs": 20588.930083999992,
- "execMs": 0.015500000095926225
+ "atMs": 17704.964250000077,
+ "execMs": 0.024624999961815774
+ },
+ {
+ "name": "read_skill",
+ "round": 3,
+ "atMs": 19667.447167000035,
+ "execMs": 0.021208000020124018
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 20588.94837499992,
- "execMs": 0.003792000003159046
+ "round": 4,
+ "atMs": 26929.335625000065,
+ "execMs": 0.07708299998193979
},
{
- "name": "wait_for_renders",
- "round": 3,
- "atMs": 22464.063958999934,
- "execMs": 0.024500000057742
+ "name": "regenerate_shot",
+ "round": 4,
+ "atMs": 26929.429958999972,
+ "execMs": 0.01587499992456287
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 24570.99450000003,
- "modelMs": 25211.532832999947,
- "toolMs": 0.5684590000892058,
- "totalMs": 25212.101292000036,
+ "ttftMs": 29941.019749999978,
+ "modelMs": 30374.96229200007,
+ "toolMs": 0.5609579998999834,
+ "totalMs": 30375.52324999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4173.895291000023,
+ "roundMs": 3955.917791999993,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.4835420000599697
+ "execMs": 0.4098750000121072
}
]
},
{
- "roundMs": 3686.5321249999106,
+ "roundMs": 2571.605375000043,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.037124999915249646
+ "execMs": 0.011749999946914613
},
{
"name": "watch_video",
- "execMs": 0.003999999957159162
+ "execMs": 0.0005420000525191426
}
]
},
{
- "roundMs": 12726.965375000029,
+ "roundMs": 11176.22912500007,
"firstDeltaMs": null,
"calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.015500000095926225
- },
+ "name": "list_skills",
+ "execMs": 0.024624999961815774
+ }
+ ]
+ },
+ {
+ "roundMs": 1962.2910840000259,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.003792000003159046
+ "name": "read_skill",
+ "execMs": 0.021208000020124018
}
]
},
{
- "roundMs": 1875.0709169999463,
+ "roundMs": 7261.75629099994,
"firstDeltaMs": null,
"calls": [
{
- "name": "wait_for_renders",
- "execMs": 0.024500000057742
+ "name": "regenerate_shot",
+ "execMs": 0.07708299998193979
+ },
+ {
+ "name": "regenerate_shot",
+ "execMs": 0.01587499992456287
}
]
},
{
- "roundMs": 2749.069125000038,
- "firstDeltaMs": 2106.857958000037,
+ "roundMs": 3447.162624999997,
+ "firstDeltaMs": 3011.5432080000173,
"calls": []
}
]
@@ -15896,249 +15625,227 @@
"pass": false,
"intent": "work",
"notes": [
- "mutating tools called: recut_scene, wait_for_renders"
+ "mutating tools called: subtitles_generate"
],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 4162.445542000001,
- "execMs": 0.024000000092200935
+ "atMs": 6821.244834000012,
+ "execMs": 0.024250000016763806
},
{
- "name": "watch_video",
+ "name": "subtitles_generate",
"round": 1,
- "atMs": 10797.955667000031,
- "execMs": 0.02000000001862645
+ "atMs": 12000.581542000058,
+ "execMs": 0.020249999943189323
},
{
"name": "watch_video",
- "round": 1,
- "atMs": 10797.986583000049,
- "execMs": 0.001416000071913004
- },
- {
- "name": "recut_scene",
"round": 2,
- "atMs": 18355.900041999994,
- "execMs": 0.040957999997772276
+ "atMs": 14058.70433400001,
+ "execMs": 0.006541999988257885
},
{
"name": "regenerate_shot",
"round": 3,
- "atMs": 23983.95229200006,
- "execMs": 0.018874999950639904
+ "atMs": 23041.45658400003,
+ "execMs": 0.029749999986961484
},
{
"name": "regenerate_shot",
"round": 3,
- "atMs": 23983.985208000056,
- "execMs": 0.0036250000121071935
+ "atMs": 23041.499583999976,
+ "execMs": 0.0037079999456182122
},
{
- "name": "wait_for_renders",
+ "name": "get_state",
"round": 4,
- "atMs": 26405.27575000003,
- "execMs": 0.05749999999534339
+ "atMs": 25768.92425000004,
+ "execMs": 0.014375000027939677
+ },
+ {
+ "name": "wait_for_renders",
+ "round": 5,
+ "atMs": 27363.703749999986,
+ "execMs": 0.03383400000166148
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 28766.016417000093,
- "modelMs": 29414.91041699995,
- "toolMs": 0.16637400013860315,
- "totalMs": 29415.076791000087,
+ "ttftMs": 29801.93633399997,
+ "modelMs": 30069.434252000065,
+ "toolMs": 0.13270899991039187,
+ "totalMs": 30069.566960999975,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4162.035499999998,
+ "roundMs": 6821.02345800004,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.024000000092200935
+ "execMs": 0.024250000016763806
}
]
},
{
- "roundMs": 6635.360457999981,
+ "roundMs": 5179.250041999971,
"firstDeltaMs": null,
"calls": [
{
- "name": "watch_video",
- "execMs": 0.02000000001862645
- },
- {
- "name": "watch_video",
- "execMs": 0.001416000071913004
+ "name": "subtitles_generate",
+ "execMs": 0.020249999943189323
}
]
},
{
- "roundMs": 7557.580792000052,
+ "roundMs": 2058.0623340000166,
"firstDeltaMs": null,
"calls": [
{
- "name": "recut_scene",
- "execMs": 0.040957999997772276
+ "name": "watch_video",
+ "execMs": 0.006541999988257885
}
]
},
{
- "roundMs": 5627.896375000011,
+ "roundMs": 8982.693458999973,
"firstDeltaMs": null,
"calls": [
{
"name": "regenerate_shot",
- "execMs": 0.018874999950639904
+ "execMs": 0.029749999986961484
},
{
"name": "regenerate_shot",
- "execMs": 0.0036250000121071935
+ "execMs": 0.0037079999456182122
+ }
+ ]
+ },
+ {
+ "roundMs": 2727.374291999964,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.014375000027939677
}
]
},
{
- "roundMs": 2421.2303749999264,
+ "roundMs": 1594.7038750000065,
"firstDeltaMs": null,
"calls": [
{
"name": "wait_for_renders",
- "execMs": 0.05749999999534339
+ "execMs": 0.03383400000166148
}
]
},
{
- "roundMs": 3010.80691699998,
- "firstDeltaMs": 2360.6197920000413,
+ "roundMs": 2706.3267920000944,
+ "firstDeltaMs": 2438.1530000000494,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "work",
- "notes": [
- "mutating tools called: recut_scene, wait_for_renders"
- ],
+ "notes": [],
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 4241.677999999956,
- "execMs": 0.08279199991375208
+ "atMs": 3737.972375000012,
+ "execMs": 0.0058750000316649675
},
{
"name": "watch_video",
"round": 1,
- "atMs": 10474.961791999987,
- "execMs": 0.008166000014171004
+ "atMs": 7410.7817499999655,
+ "execMs": 0.02466699993237853
},
{
"name": "watch_video",
"round": 1,
- "atMs": 10474.978707999922,
- "execMs": 0.0005829998990520835
+ "atMs": 7410.816624999978,
+ "execMs": 0.0017499999376013875
},
{
"name": "recut_scene",
"round": 2,
- "atMs": 16216.990791999968,
- "execMs": 0.018459000042639673
- },
- {
- "name": "regenerate_shot",
- "round": 3,
- "atMs": 18708.564582999912,
- "execMs": 0.1134580000070855
- },
- {
- "name": "regenerate_shot",
- "round": 3,
- "atMs": 18708.688916999963,
- "execMs": 0.015791999991051853
+ "atMs": 13667.431290999986,
+ "execMs": 0.01533299998845905
},
{
"name": "wait_for_renders",
- "round": 4,
- "atMs": 20455.379624999943,
- "execMs": 0.037250000052154064
+ "round": 3,
+ "atMs": 15767.950165999937,
+ "execMs": 0.004291999968700111
}
],
"timings": {
"gateMs": 0,
"gateSkipped": true,
- "ttftMs": 22883.16924999992,
- "modelMs": 23401.58379100007,
- "toolMs": 0.27649999991990626,
- "totalMs": 23401.86029099999,
+ "ttftMs": 17631.307124999934,
+ "modelMs": 18027.737208000035,
+ "toolMs": 0.05191699985880405,
+ "totalMs": 18027.789124999894,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4241.457167000044,
+ "roundMs": 3737.74916700006,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.08279199991375208
+ "execMs": 0.0058750000316649675
}
]
},
{
- "roundMs": 6233.110666000051,
+ "roundMs": 3672.7550830000546,
"firstDeltaMs": null,
"calls": [
{
"name": "watch_video",
- "execMs": 0.008166000014171004
+ "execMs": 0.02466699993237853
},
{
"name": "watch_video",
- "execMs": 0.0005829998990520835
+ "execMs": 0.0017499999376013875
}
]
},
{
- "roundMs": 5741.974083999987,
+ "roundMs": 6256.573832999915,
"firstDeltaMs": null,
"calls": [
{
"name": "recut_scene",
- "execMs": 0.018459000042639673
- }
- ]
- },
- {
- "roundMs": 2491.485708000022,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "regenerate_shot",
- "execMs": 0.1134580000070855
- },
- {
- "name": "regenerate_shot",
- "execMs": 0.015791999991051853
+ "execMs": 0.01533299998845905
}
]
},
{
- "roundMs": 1746.5988750000251,
+ "roundMs": 2100.4867089999607,
"firstDeltaMs": null,
"calls": [
{
"name": "wait_for_renders",
- "execMs": 0.037250000052154064
+ "execMs": 0.004291999968700111
}
]
},
{
- "roundMs": 2946.9572909999406,
- "firstDeltaMs": 2427.6852909999434,
+ "roundMs": 2260.172416000045,
+ "firstDeltaMs": 1863.342874999973,
"calls": []
}
]