Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
{
"decision_id": "p2-attribution-dev",
"plan": "P2",
"stage": "dev",
"eligible_for_default": false,
"eligibility_note": "Dev verdicts come from development data and never set a default. A held-out verdict comes from the custodian after a committed preregistration.",
"verdict_type": "quality",
"overall": "pass",
"decided_at": "2026-10-05T02:38:54.705Z",
"sources": [
{
"source": "lme-s-dev-assistant-facts",
"verdict": "pass",
"arms": {
"baseline": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "7adf6c1f0d9df80f9bd2f5f6c75a32a83d330b9b8377ca86f11590a322dcc2c1",
"loaded_git_head": "5d5375d438da490ad4d865fa93e3218a57e6bbdd"
},
"candidate": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "7adf6c1f0d9df80f9bd2f5f6c75a32a83d330b9b8377ca86f11590a322dcc2c1",
"loaded_git_head": "5d5375d438da490ad4d865fa93e3218a57e6bbdd"
}
},
"family": {
"family_id": "p2-attribution-dev:lme-s-dev-assistant-facts",
"verdict": "pass",
"reasons": [],
"alpha": 0.05,
"holm_family": [
"assistant-qa-from-facts"
],
"comparisons": [
{
"id": "assistant-qa-from-facts",
"metric": "qa_score",
"gate": "superiority",
"status": "pass",
"reasons": [],
"n_pairs": 30,
"n_clusters": 30,
"n_excluded": 0,
"stats": {
"n_pairs": 30,
"n_clusters": 30,
"mean_a": 0.23333333333333334,
"mean_b": 0.6,
"delta": 0.36666666666666664,
"se_cluster": 0.08948554539839962,
"se_iid": 0.08948554539839962,
"design_effect": 1,
"effective_n": 30,
"ci95": [
0.2,
0.5333333333333333
],
"p_two_sided": 0.000999900009999,
"p_greater": 0.0005999400059994001,
"p_less": 1,
"p_method": "monte-carlo-sign-flip"
},
"mcnemar": {
"losses": 0,
"wins": 11,
"both_pass": 7,
"both_fail": 12,
"discordant": 11,
"p_two_sided": 0.0009765625
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": 0.25070138128860187,
"min_one_sided_discordant": 6,
"note": "with 30 pairs in 30 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 0.2507; the observed delta is at least that size; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
},
"direction": "higher",
"p_holm": 0.00009999000099990002,
"min_effect": 0,
"p_superiority": 0.00009999000099990002
}
]
}
},
{
"source": "lme-s-dev-user-facts",
"verdict": "pass",
"arms": {
"baseline": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "7adf6c1f0d9df80f9bd2f5f6c75a32a83d330b9b8377ca86f11590a322dcc2c1",
"loaded_git_head": "5d5375d438da490ad4d865fa93e3218a57e6bbdd"
},
"candidate": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "7adf6c1f0d9df80f9bd2f5f6c75a32a83d330b9b8377ca86f11590a322dcc2c1",
"loaded_git_head": "5d5375d438da490ad4d865fa93e3218a57e6bbdd"
}
},
"family": {
"family_id": "p2-attribution-dev:lme-s-dev-user-facts",
"verdict": "pass",
"reasons": [],
"alpha": 0.05,
"holm_family": [
"user-qa-from-facts"
],
"comparisons": [
{
"id": "user-qa-from-facts",
"metric": "qa_score",
"gate": "noninferiority",
"status": "pass",
"reasons": [],
"n_pairs": 19,
"n_clusters": 19,
"n_excluded": 1,
"stats": {
"n_pairs": 19,
"n_clusters": 19,
"mean_a": 0.7368421052631579,
"mean_b": 0.7368421052631579,
"delta": 0,
"se_cluster": 0,
"se_iid": 0,
"design_effect": null,
"effective_n": null,
"ci95": [
0,
0
],
"p_two_sided": 1,
"p_greater": 1,
"p_less": 1,
"p_method": "degenerate"
},
"mcnemar": {
"losses": 0,
"wins": 0,
"both_pass": 14,
"both_fail": 5,
"discordant": 0,
"p_two_sided": 1
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": null,
"min_one_sided_discordant": 6,
"note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
},
"direction": "higher",
"p_holm": 0,
"tolerance": 0.02,
"p_noninferiority": 0
}
]
}
}
]
}