Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
6.6 KB
Raw Blame
{
  "decision_id": "p8-dev-20261004",
  "plan": "P8",
  "stage": "dev",
  "eligible_for_default": false,
  "eligibility_note": "Dev verdicts come from development data and never set a default. A held-out verdict comes from the custodian after a committed preregistration.",
  "verdict_type": "correctness",
  "overall": "pass",
  "decided_at": "2026-10-04T23:07:12.616Z",
  "sources": [
    {
      "source": "n5-forget-residue",
      "verdict": "pass",
      "arms": {
        "baseline": "0.60.48.0",
        "candidate": "0.60.48.0"
      },
      "contracts": [
        {
          "path": "verdict",
          "op": "eq",
          "value": "pass",
          "candidate": "pass",
          "baseline": "pass",
          "ok": true
        }
      ]
    },
    {
      "source": "n1-knowledge-update",
      "verdict": "pass",
      "arms": {
        "baseline": "0.60.48.0",
        "candidate": "0.60.48.0"
      },
      "contracts": [
        {
          "path": "verdict",
          "op": "eq",
          "value": "pass",
          "candidate": "pass",
          "baseline": "pass",
          "ok": true
        }
      ]
    },
    {
      "source": "lme-s-dev",
      "verdict": "pass",
      "arms": {
        "baseline": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "fab1c3142270c58c8357f2298bb3f7bf9b59a1062f5954684e95bd1d1cc11c01",
          "loaded_git_head": "6622a119e40ea09a7719233046aca24741863ed2"
        },
        "candidate": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "e1ee8f75b641083bfc321af009ef2316d709db4a9403c72591fde76525d8aa1a",
          "loaded_git_head": "850e8ef3dfadcaf15e1547d1d754398caf0edd41"
        }
      },
      "family": {
        "family_id": "p8-dev-20261004:lme-s-dev",
        "verdict": "pass",
        "reasons": [],
        "alpha": 0.05,
        "holm_family": [
          "recall-all-5"
        ],
        "comparisons": [
          {
            "id": "recall-all-5",
            "metric": "recall_all_at_5",
            "gate": "noninferiority",
            "status": "pass",
            "reasons": [],
            "n_pairs": 470,
            "n_clusters": 470,
            "n_excluded": 30,
            "stats": {
              "n_pairs": 470,
              "n_clusters": 470,
              "mean_a": 0.9276595744680851,
              "mean_b": 0.9276595744680851,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 0,
              "both_pass": 436,
              "both_fail": 34,
              "discordant": 0,
              "p_two_sided": 1
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": 6,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0,
            "tolerance": 0.01,
            "p_noninferiority": 0
          },
          {
            "id": "ndcg-10",
            "metric": "ndcg_at_10",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 470,
            "n_clusters": 470,
            "n_excluded": 30,
            "stats": {
              "n_pairs": 470,
              "n_clusters": 470,
              "mean_a": 0.9429294330024023,
              "mean_b": 0.9429414225547724,
              "delta": 0.0000119895523700806,
              "se_cluster": 0.000011989552370080597,
              "se_iid": 0.000011989552370080599,
              "design_effect": 0.9999999999999998,
              "effective_n": 470.0000000000001,
              "ci95": [
                0,
                0.0000359686571102418
              ],
              "p_two_sided": 1,
              "p_greater": 0.49905009499050096,
              "p_less": 1,
              "p_method": "monte-carlo-sign-flip"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": 0.000033589752700607585,
              "min_one_sided_discordant": null,
              "note": "with 470 pairs in 470 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 0.0000; the observed delta is below that size, so a null result here is not evidence of no effect"
            },
            "direction": "higher"
          },
          {
            "id": "latency",
            "metric": "latency_ms",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 470,
            "n_clusters": 470,
            "n_excluded": 30,
            "stats": {
              "n_pairs": 470,
              "n_clusters": 470,
              "mean_a": 255.9219148936171,
              "mean_b": 252.40595744680843,
              "delta": -3.5159574468085055,
              "se_cluster": 6.813703022819454,
              "se_iid": 6.813703022819456,
              "design_effect": 0.9999999999999996,
              "effective_n": 470.0000000000002,
              "ci95": [
                -16.65276595744684,
                9.602553191489367
              ],
              "p_two_sided": 0.6028397160283971,
              "p_greater": 0.6982301769823017,
              "p_less": 0.30186981301869814,
              "p_method": "monte-carlo-sign-flip"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": 19.089169674342838,
              "min_one_sided_discordant": null,
              "note": "with 470 pairs in 470 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 19.0892; the observed delta is below that size, so a null result here is not evidence of no effect"
            },
            "direction": "lower"
          }
        ]
      }
    }
  ]
}