Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
{
"decision_id": "p6-think-notes",
"plan": "P6",
"stage": "dev",
"eligible_for_default": false,
"eligibility_note": "Dev verdicts come from development data and never set a default. A held-out verdict comes from the custodian after a committed preregistration.",
"verdict_type": "quality",
"overall": "inconclusive",
"decided_at": "2026-10-05T01:39:20.209Z",
"sources": [
{
"source": "lme-s-dev-think",
"verdict": "inconclusive",
"arms": {
"baseline": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
"loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
},
"candidate": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
"loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
}
},
"family": {
"family_id": "p6-think-notes:lme-s-dev-think",
"verdict": "inconclusive",
"reasons": [
"qa-score: inconclusive"
],
"alpha": 0.05,
"holm_family": [
"qa-score"
],
"comparisons": [
{
"id": "qa-score",
"metric": "qa_score",
"gate": "superiority",
"status": "inconclusive",
"reasons": [
"an improvement larger than 0 is not shown (Holm p = 0.7486 > 0.05); this is not evidence of a loss"
],
"n_pairs": 150,
"n_clusters": 150,
"n_excluded": 0,
"stats": {
"n_pairs": 150,
"n_clusters": 150,
"mean_a": 0.8866666666666667,
"mean_b": 0.88,
"delta": -0.006666666666666667,
"se_cluster": 0.014947085390483146,
"se_iid": 0.014947085390483145,
"design_effect": 1.0000000000000004,
"effective_n": 149.99999999999994,
"ci95": [
-0.03333333333333333,
0.02
],
"p_two_sided": 1,
"p_greater": 0.8164183581641836,
"p_less": 0.5008499150084992,
"p_method": "monte-carlo-sign-flip"
},
"mcnemar": {
"losses": 3,
"wins": 2,
"both_pass": 130,
"both_fail": 15,
"discordant": 5,
"p_two_sided": 1
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": 0.04187553349481874,
"min_one_sided_discordant": 6,
"note": "with 150 pairs in 150 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 0.0419; the observed delta is below that size, so a null result here is not evidence of no effect; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
},
"direction": "higher",
"p_holm": 0.7486251374862514,
"min_effect": 0,
"p_superiority": 0.7486251374862514
},
{
"id": "recall-all-5",
"metric": "recall_all_at_5",
"gate": "exploratory",
"status": "report_only",
"reasons": [],
"n_pairs": 142,
"n_clusters": 142,
"n_excluded": 8,
"stats": {
"n_pairs": 142,
"n_clusters": 142,
"mean_a": 0.9436619718309859,
"mean_b": 0.9436619718309859,
"delta": 0,
"se_cluster": 0,
"se_iid": 0,
"design_effect": null,
"effective_n": null,
"ci95": [
0,
0
],
"p_two_sided": 1,
"p_greater": 1,
"p_less": 1,
"p_method": "degenerate"
},
"mcnemar": {
"losses": 0,
"wins": 0,
"both_pass": 134,
"both_fail": 8,
"discordant": 0,
"p_two_sided": 1
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": null,
"min_one_sided_discordant": 6,
"note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
},
"direction": "higher"
},
{
"id": "qa-input-tokens",
"metric": "qa_input_tokens",
"gate": "exploratory",
"status": "report_only",
"reasons": [],
"n_pairs": 150,
"n_clusters": 150,
"n_excluded": 0,
"stats": {
"n_pairs": 150,
"n_clusters": 150,
"mean_a": 23.213333333333335,
"mean_b": 23.213333333333335,
"delta": 0,
"se_cluster": 0,
"se_iid": 0,
"design_effect": null,
"effective_n": null,
"ci95": [
0,
0
],
"p_two_sided": 1,
"p_greater": 1,
"p_less": 1,
"p_method": "degenerate"
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": null,
"min_one_sided_discordant": null,
"note": "no paired variation was observed, so the detectable effect cannot be estimated from this data"
},
"direction": "lower"
}
]
}
},
{
"source": "locomo-dev-think",
"verdict": "inconclusive",
"arms": {
"baseline": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
"loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
},
"candidate": {
"package": "gbrain",
"declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
"version": "0.60.48.0",
"package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
"loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
}
},
"family": {
"family_id": "p6-think-notes:locomo-dev-think",
"verdict": "inconclusive",
"reasons": [
"qa-score: inconclusive"
],
"alpha": 0.05,
"holm_family": [
"qa-score"
],
"comparisons": [
{
"id": "qa-score",
"metric": "qa_score",
"gate": "superiority",
"status": "inconclusive",
"reasons": [
"McNemar treats items as independent; the clustered sign-flip p-value is the primary test here",
"an improvement larger than 0 is not shown (Holm p = 1.0000 > 0.05); this is not evidence of a loss"
],
"n_pairs": 587,
"n_clusters": 3,
"n_excluded": 0,
"stats": {
"n_pairs": 587,
"n_clusters": 3,
"mean_a": 0.8943781942078365,
"mean_b": 0.879045996592845,
"delta": -0.015332197614991482,
"se_cluster": 0.011104515918088198,
"se_iid": 0.009774154610923767,
"design_effect": 1.2907461931830273,
"effective_n": 454.7756972673586,
"ci95": [
-0.03347280334728033,
0
],
"p_two_sided": 0.5,
"p_greater": 1,
"p_less": 0.25,
"p_method": "exact-sign-flip"
},
"mcnemar": {
"losses": 21,
"wins": 12,
"both_pass": 504,
"both_fail": 50,
"discordant": 33,
"p_two_sided": 0.16275565745308995
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": 0.03111024765856505,
"min_one_sided_discordant": 6,
"note": "with 587 pairs in 3 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 0.0311; the observed delta is below that size, so a null result here is not evidence of no effect; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05; clustering inflates variance by 1.29x (effective n about 454.8 of 587)"
},
"direction": "higher",
"p_holm": 1,
"min_effect": 0,
"p_superiority": 1
},
{
"id": "recall-all-5",
"metric": "recall_all_at_5",
"gate": "exploratory",
"status": "report_only",
"reasons": [
"McNemar treats items as independent; the clustered sign-flip p-value is the primary test here"
],
"n_pairs": 464,
"n_clusters": 3,
"n_excluded": 123,
"stats": {
"n_pairs": 464,
"n_clusters": 3,
"mean_a": 0.7564655172413793,
"mean_b": 0.7564655172413793,
"delta": 0,
"se_cluster": 0,
"se_iid": 0,
"design_effect": null,
"effective_n": null,
"ci95": [
0,
0
],
"p_two_sided": 1,
"p_greater": 1,
"p_less": 1,
"p_method": "degenerate"
},
"mcnemar": {
"losses": 0,
"wins": 0,
"both_pass": 351,
"both_fail": 113,
"discordant": 0,
"p_two_sided": 1
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": null,
"min_one_sided_discordant": 6,
"note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
},
"direction": "higher"
},
{
"id": "qa-input-tokens",
"metric": "qa_input_tokens",
"gate": "exploratory",
"status": "report_only",
"reasons": [],
"n_pairs": 587,
"n_clusters": 3,
"n_excluded": 0,
"stats": {
"n_pairs": 587,
"n_clusters": 3,
"mean_a": 15,
"mean_b": 15,
"delta": 0,
"se_cluster": 0,
"se_iid": 0,
"design_effect": null,
"effective_n": null,
"ci95": [
0,
0
],
"p_two_sided": 1,
"p_greater": 1,
"p_less": 1,
"p_method": "degenerate"
},
"power": {
"alpha": 0.05,
"power": 0.8,
"mde": null,
"min_one_sided_discordant": null,
"note": "no paired variation was observed, so the detectable effect cannot be estimated from this data"
},
"direction": "lower"
}
]
}
}
]
}