Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
{
"decision_id": "p8-dev-20261004",
"plan": "P8",
"stage": "dev",
"eligible_for_default": false,
"eligibility_note": "Dev verdicts come from development data and never set a default. A held-out verdict comes from the custodian after a committed preregistration.",
"verdict_type": "correctness",
"overall": "pass",
"decided_at": "2026-10-04T23:07:12.616Z",
"sources": [
{
"source": "n5-forget-residue",
"verdict": "pass",
"arms": {
"baseline": "0.60.48.0",
"candidate": "0.60.48.0"
},
"contracts": [
{
"path": "verdict",
"op": "eq",
"value": "pass",
"candidate": "pass",
"baseline": "pass",
"ok": true
}
]
},
{
"source": "n1-knowledge-update",
"verdict": "pass",
"arms": {
"baseline": "0.60.48.0",
"candidate": "0.60.48.0"
},
"contracts": [
{
"path": "verdict",
"op": "eq",
"value": "pass",
"candidate": "pass",
"baseline": "pass",
"ok": true
}
]
},
{
"source": "lme-s-dev",
"verdict": "pass",
"arms": {
"baseline": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "fab1c3142270c58c8357f2298bb3f7bf9b59a1062f5954684e95bd1d1cc11c01",
"loaded_git_head": "6622a119e40ea09a7719233046aca24741863ed2"
},
"candidate": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "e1ee8f75b641083bfc321af009ef2316d709db4a9403c72591fde76525d8aa1a",
"loaded_git_head": "850e8ef3dfadcaf15e1547d1d754398caf0edd41"
}
},
"family": {
"family_id": "p8-dev-20261004:lme-s-dev",
"verdict": "pass",
"reasons": [],
"alpha": 0.05,
"holm_family": [
"recall-all-5"
],
"comparisons": [
{
"id": "recall-all-5",
"metric": "recall_all_at_5",
"gate": "noninferiority",
"status": "pass",
"reasons": [],
"n_pairs": 470,
"n_clusters": 470,
"n_excluded": 30,
"stats": {
"n_pairs": 470,
"n_clusters": 470,
"mean_a": 0.9276595744680851,
"mean_b": 0.9276595744680851,
"delta": 0,
"se_cluster": 0,
"se_iid": 0,
"design_effect": null,
"effective_n": null,
"ci95": [
0,
0
],
"p_two_sided": 1,
"p_greater": 1,
"p_less": 1,
"p_method": "degenerate"
},
"mcnemar": {
"losses": 0,
"wins": 0,
"both_pass": 436,
"both_fail": 34,
"discordant": 0,
"p_two_sided": 1
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": null,
"min_one_sided_discordant": 6,
"note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
},
"direction": "higher",
"p_holm": 0,
"tolerance": 0.01,
"p_noninferiority": 0
},
{
"id": "ndcg-10",
"metric": "ndcg_at_10",
"gate": "exploratory",
"status": "report_only",
"reasons": [],
"n_pairs": 470,
"n_clusters": 470,
"n_excluded": 30,
"stats": {
"n_pairs": 470,
"n_clusters": 470,
"mean_a": 0.9429294330024023,
"mean_b": 0.9429414225547724,
"delta": 0.0000119895523700806,
"se_cluster": 0.000011989552370080597,
"se_iid": 0.000011989552370080599,
"design_effect": 0.9999999999999998,
"effective_n": 470.0000000000001,
"ci95": [
0,
0.0000359686571102418
],
"p_two_sided": 1,
"p_greater": 0.49905009499050096,
"p_less": 1,
"p_method": "monte-carlo-sign-flip"
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": 0.000033589752700607585,
"min_one_sided_discordant": null,
"note": "with 470 pairs in 470 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 0.0000; the observed delta is below that size, so a null result here is not evidence of no effect"
},
"direction": "higher"
},
{
"id": "latency",
"metric": "latency_ms",
"gate": "exploratory",
"status": "report_only",
"reasons": [],
"n_pairs": 470,
"n_clusters": 470,
"n_excluded": 30,
"stats": {
"n_pairs": 470,
"n_clusters": 470,
"mean_a": 255.9219148936171,
"mean_b": 252.40595744680843,
"delta": -3.5159574468085055,
"se_cluster": 6.813703022819454,
"se_iid": 6.813703022819456,
"design_effect": 0.9999999999999996,
"effective_n": 470.0000000000002,
"ci95": [
-16.65276595744684,
9.602553191489367
],
"p_two_sided": 0.6028397160283971,
"p_greater": 0.6982301769823017,
"p_less": 0.30186981301869814,
"p_method": "monte-carlo-sign-flip"
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": 19.089169674342838,
"min_one_sided_discordant": null,
"note": "with 470 pairs in 470 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 19.0892; the observed delta is below that size, so a null result here is not evidence of no effect"
},
"direction": "lower"
}
]
}
}
]
}