Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

chore(backend): refresh the expert writing-style baseline (#14495)

Reinier van der Leer7a354df889c1
diff --git a/autogpt_platform/backend/backend/copilot/eval/style/baseline.json b/autogpt_platform/backend/backend/copilot/eval/style/baseline.json
index c2abd33fa8..c326611166 100644
--- a/autogpt_platform/backend/backend/copilot/eval/style/baseline.json
+++ b/autogpt_platform/backend/backend/copilot/eval/style/baseline.json
@@ -1,38 +1,37 @@
{
- "run_id": "style-eca9c14e",
- "ts": "2026-09-07T16:02:48.416336Z",
+ "run_id": "style-5784acf3",
+ "ts": "2026-09-10T13:24:29.920161Z",
"chat_model": "anthropic/claude-sonnet-5",
"judge_model": "anthropic/claude-haiku-4-5",
"delegation_enabled": true,
- "cost_usd": 5.1,
- "note": "One pass, 2026-09-07, on the 27 chat prompts per expert. Six turns that were still calling tools at the round cap are excluded as errors and two judge calls failed, so 73 of 81 responses are scored. The run predates both the tool-stub fix and dev's current prompts, which is why a dry run reports the harness and prompt components as changed: these are recorded scores, not current ones.",
- "fingerprint": "2dd3c1d36d6f4c6efc1fa6133f67a6e844f577937859e278d4b6cb3982743959",
+ "cost_usd": 5.2676,
+ "note": "One pass, 2026-09-10, on the 27 chat prompts per expert, measured on dev at dd71bda0df. Seven turns that were still calling tools at the round cap are excluded as errors (Frankie 5, Max 2), so 74 of 81 responses are scored. It replaces the 2026-09-07 run, which predated the tool-stub fix and ten dev commits to the prompt assembly; against it the paired per-prompt change is Maria +1.15 +/- 5.03, Max +4.57 +/- 7.69, Frankie -9.29 +/- 7.07, and -0.86 +/- 3.80 over all 70 shared prompts. No pass threshold is set: the per-prompt spread (sd 31.8) makes a single pass sensitive only to a shift of about 10 points per expert.",
+ "fingerprint": "ba94f8d3cff7bdf60fcfd21e51041b8c2584dfc7acbd10bd58ae58113006d525",
"parts": {
- "autopilot": "f129fbc05802de8b326772f2de7ed45e98b6ee7ef5827cf67829d9e2ba604798",
+ "autopilot": "4e4ab6206736428d334c26ff206644dfd89c9a79b03bf3960a4b1c04eeceebe5",
"rubric": "a86a379960470eaa04b2bdcbd4a42425e0ff9ea9b6aa184b0eb27721d2a65719",
- "models": "b14360fa8ada4853b029dba8f24c1e7d348aeb8c4f67f7777e7b90d1daac5f62",
- "context:Frankie": "2560405c14d7c61c99096d82c4681d6e0de351dbe463a15ed644829f1abe2345",
- "context:Maria": "d2950b58346906e23f2e572b95f1ac3403f9179cc642a07e70b177fb7727a9b1",
- "context:Max": "abaf3beec4c1729e426776156c15618ad02d98a19072844703fa7b3ba0f1f9cb",
- "fixture:Frankie": "6c20f9d4632e1dc1afdebcca76037bfd31b26989b42d59a4d3e4ea57ba6bed7b",
- "fixture:Maria": "069134a67b06569cc19d33f303fd39d3090a676595296fbc5730f68005f0f592",
- "fixture:Max": "eac0d0c897df7a80907365850e9442e949e0652f5bd047baf2a6eb0a9e43dbb7",
- "lede:Frankie": "813bdf7e411650cfbf2d837c35e59e3930cf808dde9303f746068440582578b1",
- "lede:Maria": "85c3dc3c153d73139d7f169c0ef3cd2003483fab9f80fbcb9fbe56fc677504b7",
- "lede:Max": "69cbbda2a9b7d8c1f2a752bbce46eee0e1a3f049c990053ad9139ee50ba0d61f",
- "prompt:Frankie": "325fb133bf7f8ffb8aa46ff186a408921148368cb4453423bedfc4711a522646",
- "prompt:Maria": "3bfab1c2f4611db7d318b74a21c69bb7634590390d86042418d6da833e33e354",
- "prompt:Max": "a7619e3de3abbbd606741f9cb04428441e2c013fe979944556426fb39104278d"
+ "models": "1e6729fadf146170eb9aaca3c2d65318e35cf5e883bd81fbc59a65f1c27561c8",
+ "delegation": "3cbc87c7681f34db4617feaa2c8801931bc5e42d8d0f560e756dd4cd92885f18",
+ "harness": "9d2e4c4247daf166c80b1ebba0ab3d8123fe632de0bc7f7cb417a2c10d55336f",
+ "context:Frankie": "6b1d175ec17ee91731aec98619ec5dc723d6dd90f84a730bfbc0ca6f0715196f",
+ "context:Maria": "3a5ace81d95923c24c5ed337d77286d9454fae845817c8c4bfcb2eabaaf801c6",
+ "context:Max": "a4b73b5fb381ba7e65fbcccf0d68ef553545bdf27dc12ec93890d3b455d09023",
+ "fixture:Frankie": "3bf40ca2039ac60d45394a4a420eadb89240509e3906a9947ef91fa834db00dd",
+ "fixture:Maria": "d6748e4ec5ad6c1851450684a7b63d7adad6f96f0b2ed845960419b2449eca0a",
+ "fixture:Max": "8e361cf6a0bbf2342dc86fcf7326c10cdc3f094a3253faa5fa9c18da4d225b19",
+ "prompt:Frankie": "c7b4291e8f094ba655e957ec616cdbf19ca7bef65ef8ee950f5a45bc43175568",
+ "prompt:Maria": "624a9b4a43dc780c3651c2fbd4b966fab2ef399b459362847972f23d3357741a",
+ "prompt:Max": "5516868051bc7892162ef1d0aa97596bcd84be93fcda9f7fb4ad8faa5c4c85ee"
},
"experts": [
{
"expert": "Maria",
"scores": {
- "n": 26,
- "mean": 85.77,
- "sd": 28.97,
+ "n": 27,
+ "mean": 87.41,
+ "sd": 26.03,
"min": 10.0,
- "p25": 91.25,
+ "p25": 95.0,
"median": 100.0,
"p75": 100.0,
"max": 100.0
@@ -40,249 +39,250 @@
"by_kind": {
"briefing": {
"n": 6,
- "mean": 100.0,
- "sd": 0.0,
- "min": 100.0,
- "p25": 100.0,
- "median": 100.0,
+ "mean": 97.5,
+ "sd": 2.74,
+ "min": 95.0,
+ "p25": 95.0,
+ "median": 97.5,
"p75": 100.0,
"max": 100.0
},
"escalation": {
- "n": 6,
- "mean": 100.0,
- "sd": 0.0,
- "min": 100.0,
- "p25": 100.0,
+ "n": 7,
+ "mean": 98.57,
+ "sd": 2.44,
+ "min": 95.0,
+ "p25": 97.5,
"median": 100.0,
"p75": 100.0,
"max": 100.0
},
"failure": {
"n": 6,
- "mean": 57.5,
- "sd": 39.84,
+ "mean": 50.0,
+ "sd": 36.47,
"min": 10.0,
- "p25": 26.25,
- "median": 60.0,
- "p75": 93.75,
- "max": 95.0
+ "p25": 21.25,
+ "median": 47.5,
+ "p75": 73.75,
+ "max": 100.0
},
"reply_draft": {
"n": 8,
- "mean": 85.62,
- "sd": 28.84,
- "min": 15.0,
- "p25": 90.0,
- "median": 95.0,
+ "mean": 98.12,
+ "sd": 3.72,
+ "min": 90.0,
+ "p25": 98.75,
+ "median": 100.0,
"p75": 100.0,
"max": 100.0
}
},
"by_prompt": {
- "maria-briefing-01": 100.0,
+ "maria-briefing-01": 95.0,
"maria-briefing-02": 100.0,
"maria-briefing-03": 100.0,
- "maria-briefing-04": 100.0,
- "maria-briefing-05": 100.0,
+ "maria-briefing-04": 95.0,
+ "maria-briefing-05": 95.0,
"maria-briefing-06": 100.0,
- "maria-reply-01": 95.0,
- "maria-reply-02": 90.0,
+ "maria-reply-01": 90.0,
+ "maria-reply-02": 100.0,
"maria-reply-03": 100.0,
"maria-reply-04": 95.0,
- "maria-reply-05": 15.0,
+ "maria-reply-05": 100.0,
"maria-reply-06": 100.0,
- "maria-reply-07": 90.0,
+ "maria-reply-07": 100.0,
"maria-reply-08": 100.0,
"maria-escalation-01": 100.0,
- "maria-escalation-02": 100.0,
+ "maria-escalation-02": 95.0,
+ "maria-escalation-03": 100.0,
"maria-escalation-04": 100.0,
"maria-escalation-05": 100.0,
- "maria-escalation-06": 100.0,
+ "maria-escalation-06": 95.0,
"maria-escalation-07": 100.0,
- "maria-failure-01": 30.0,
- "maria-failure-02": 90.0,
- "maria-failure-03": 95.0,
+ "maria-failure-01": 25.0,
+ "maria-failure-02": 100.0,
+ "maria-failure-03": 20.0,
"maria-failure-04": 10.0,
- "maria-failure-05": 95.0,
- "maria-failure-06": 25.0
+ "maria-failure-05": 70.0,
+ "maria-failure-06": 75.0
}
},
{
"expert": "Max",
"scores": {
"n": 25,
- "mean": 82.8,
- "sd": 29.05,
- "min": 20.0,
- "p25": 90.0,
- "median": 100.0,
+ "mean": 86.8,
+ "sd": 21.35,
+ "min": 25.0,
+ "p25": 85.0,
+ "median": 95.0,
"p75": 100.0,
"max": 100.0
},
"by_kind": {
"briefing": {
- "n": 5,
- "mean": 97.0,
- "sd": 2.74,
- "min": 95.0,
- "p25": 95.0,
- "median": 95.0,
- "p75": 100.0,
+ "n": 6,
+ "mean": 88.33,
+ "sd": 12.52,
+ "min": 65.0,
+ "p25": 86.25,
+ "median": 92.5,
+ "p75": 95.0,
"max": 100.0
},
"escalation": {
"n": 7,
- "mean": 89.29,
- "sd": 24.23,
- "min": 35.0,
- "p25": 95.0,
+ "mean": 83.57,
+ "sd": 28.83,
+ "min": 25.0,
+ "p25": 80.0,
"median": 100.0,
"p75": 100.0,
"max": 100.0
},
"failure": {
- "n": 6,
- "mean": 67.5,
- "sd": 38.44,
- "min": 20.0,
- "p25": 32.5,
- "median": 82.5,
- "p75": 98.75,
+ "n": 5,
+ "mean": 88.0,
+ "sd": 9.75,
+ "min": 75.0,
+ "p25": 85.0,
+ "median": 85.0,
+ "p75": 95.0,
"max": 100.0
},
"reply_draft": {
"n": 7,
- "mean": 79.29,
- "sd": 32.97,
- "min": 20.0,
- "p25": 67.5,
+ "mean": 87.86,
+ "sd": 27.97,
+ "min": 25.0,
+ "p25": 95.0,
"median": 100.0,
"p75": 100.0,
"max": 100.0
}
},
"by_prompt": {
- "max-briefing-01": 95.0,
+ "max-briefing-01": 65.0,
+ "max-briefing-02": 90.0,
"max-briefing-03": 100.0,
"max-briefing-04": 95.0,
- "max-briefing-05": 95.0,
- "max-briefing-06": 100.0,
- "max-reply-01": 90.0,
- "max-reply-02": 100.0,
- "max-reply-03": 20.0,
- "max-reply-04": 45.0,
- "max-reply-05": 100.0,
+ "max-briefing-05": 85.0,
+ "max-briefing-06": 95.0,
+ "max-reply-01": 100.0,
+ "max-reply-03": 90.0,
+ "max-reply-04": 100.0,
+ "max-reply-05": 25.0,
"max-reply-06": 100.0,
+ "max-reply-07": 100.0,
"max-reply-08": 100.0,
"max-escalation-01": 100.0,
"max-escalation-02": 100.0,
- "max-escalation-03": 35.0,
+ "max-escalation-03": 65.0,
"max-escalation-04": 100.0,
- "max-escalation-05": 100.0,
- "max-escalation-06": 100.0,
- "max-escalation-07": 90.0,
+ "max-escalation-05": 95.0,
+ "max-escalation-06": 25.0,
+ "max-escalation-07": 100.0,
"max-failure-01": 100.0,
- "max-failure-02": 95.0,
- "max-failure-03": 70.0,
- "max-failure-04": 20.0,
- "max-failure-05": 100.0,
- "max-failure-06": 20.0
+ "max-failure-03": 85.0,
+ "max-failure-04": 95.0,
+ "max-failure-05": 85.0,
+ "max-failure-06": 75.0
}
},
{
"expert": "Frankie",
"scores": {
"n": 22,
- "mean": 74.09,
- "sd": 30.3,
+ "mean": 67.5,
+ "sd": 30.74,
"min": 20.0,
- "p25": 45.0,
- "median": 87.5,
- "p75": 100.0,
+ "p25": 36.25,
+ "median": 75.0,
+ "p75": 95.0,
"max": 100.0
},
"by_kind": {
"briefing": {
- "n": 4,
- "mean": 45.0,
- "sd": 31.89,
- "min": 20.0,
- "p25": 23.75,
- "median": 35.0,
- "p75": 56.25,
- "max": 90.0
+ "n": 3,
+ "mean": 46.67,
+ "sd": 16.07,
+ "min": 35.0,
+ "p25": 37.5,
+ "median": 40.0,
+ "p75": 52.5,
+ "max": 65.0
},
"escalation": {
"n": 6,
- "mean": 95.0,
- "sd": 7.75,
- "min": 80.0,
- "p25": 95.0,
- "median": 97.5,
- "p75": 100.0,
- "max": 100.0
+ "mean": 81.67,
+ "sd": 27.87,
+ "min": 25.0,
+ "p25": 90.0,
+ "median": 92.5,
+ "p75": 95.0,
+ "max": 95.0
},
"failure": {
- "n": 4,
- "mean": 47.5,
- "sd": 33.29,
+ "n": 5,
+ "mean": 47.0,
+ "sd": 33.09,
"min": 20.0,
- "p25": 27.5,
- "median": 37.5,
- "p75": 57.5,
- "max": 95.0
+ "p25": 20.0,
+ "median": 30.0,
+ "p75": 75.0,
+ "max": 90.0
},
"reply_draft": {
"n": 8,
- "mean": 86.25,
- "sd": 19.23,
- "min": 45.0,
- "p25": 82.5,
- "median": 92.5,
+ "mean": 77.5,
+ "sd": 28.41,
+ "min": 20.0,
+ "p25": 70.0,
+ "median": 85.0,
"p75": 100.0,
"max": 100.0
}
},
"by_prompt": {
- "frankie-briefing-01": 25.0,
- "frankie-briefing-02": 45.0,
- "frankie-briefing-04": 90.0,
- "frankie-briefing-05": 20.0,
- "frankie-reply-01": 85.0,
+ "frankie-briefing-02": 35.0,
+ "frankie-briefing-04": 40.0,
+ "frankie-briefing-05": 65.0,
+ "frankie-reply-01": 20.0,
"frankie-reply-02": 100.0,
- "frankie-reply-03": 100.0,
+ "frankie-reply-03": 95.0,
"frankie-reply-04": 75.0,
- "frankie-reply-05": 100.0,
- "frankie-reply-06": 85.0,
+ "frankie-reply-05": 75.0,
+ "frankie-reply-06": 100.0,
"frankie-reply-07": 100.0,
- "frankie-reply-08": 45.0,
- "frankie-escalation-01": 95.0,
+ "frankie-reply-08": 55.0,
+ "frankie-escalation-01": 25.0,
"frankie-escalation-02": 95.0,
- "frankie-escalation-03": 80.0,
- "frankie-escalation-05": 100.0,
- "frankie-escalation-06": 100.0,
- "frankie-escalation-07": 100.0,
+ "frankie-escalation-03": 95.0,
+ "frankie-escalation-05": 90.0,
+ "frankie-escalation-06": 90.0,
+ "frankie-escalation-07": 95.0,
"frankie-failure-01": 30.0,
- "frankie-failure-02": 95.0,
- "frankie-failure-03": 45.0,
- "frankie-failure-04": 20.0
+ "frankie-failure-02": 20.0,
+ "frankie-failure-03": 90.0,
+ "frankie-failure-04": 20.0,
+ "frankie-failure-06": 75.0
}
}
],
"separation": {
- "right_spec_mean": 81.23,
- "right_spec_sd": 29.4,
- "wrong_spec_mean": 54.86,
- "no_suffix_mean": 13.57,
- "gap": 26.37,
+ "right_spec_mean": 81.28,
+ "right_spec_sd": 27.29,
+ "wrong_spec_mean": 51.66,
+ "no_suffix_mean": 16.67,
+ "gap": 29.63,
"paired": {
- "n": 73,
- "mean": 27.02,
- "sem": 3.7,
- "wins": 58,
- "ties": 9,
- "win_rate": 0.795
+ "n": 74,
+ "mean": 29.63,
+ "sem": 3.72,
+ "wins": 63,
+ "ties": 5,
+ "win_rate": 0.851
},
"separated": true
}