Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
6.0 KB
Raw Blame
{
  "decision_id": "p2-attribution-dev",
  "plan": "P2",
  "stage": "dev",
  "eligible_for_default": false,
  "eligibility_note": "Dev verdicts come from development data and never set a default. A held-out verdict comes from the custodian after a committed preregistration.",
  "verdict_type": "quality",
  "overall": "pass",
  "decided_at": "2026-10-05T02:38:54.705Z",
  "sources": [
    {
      "source": "lme-s-dev-assistant-facts",
      "verdict": "pass",
      "arms": {
        "baseline": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7adf6c1f0d9df80f9bd2f5f6c75a32a83d330b9b8377ca86f11590a322dcc2c1",
          "loaded_git_head": "5d5375d438da490ad4d865fa93e3218a57e6bbdd"
        },
        "candidate": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7adf6c1f0d9df80f9bd2f5f6c75a32a83d330b9b8377ca86f11590a322dcc2c1",
          "loaded_git_head": "5d5375d438da490ad4d865fa93e3218a57e6bbdd"
        }
      },
      "family": {
        "family_id": "p2-attribution-dev:lme-s-dev-assistant-facts",
        "verdict": "pass",
        "reasons": [],
        "alpha": 0.05,
        "holm_family": [
          "assistant-qa-from-facts"
        ],
        "comparisons": [
          {
            "id": "assistant-qa-from-facts",
            "metric": "qa_score",
            "gate": "superiority",
            "status": "pass",
            "reasons": [],
            "n_pairs": 30,
            "n_clusters": 30,
            "n_excluded": 0,
            "stats": {
              "n_pairs": 30,
              "n_clusters": 30,
              "mean_a": 0.23333333333333334,
              "mean_b": 0.6,
              "delta": 0.36666666666666664,
              "se_cluster": 0.08948554539839962,
              "se_iid": 0.08948554539839962,
              "design_effect": 1,
              "effective_n": 30,
              "ci95": [
                0.2,
                0.5333333333333333
              ],
              "p_two_sided": 0.000999900009999,
              "p_greater": 0.0005999400059994001,
              "p_less": 1,
              "p_method": "monte-carlo-sign-flip"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 11,
              "both_pass": 7,
              "both_fail": 12,
              "discordant": 11,
              "p_two_sided": 0.0009765625
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": 0.25070138128860187,
              "min_one_sided_discordant": 6,
              "note": "with 30 pairs in 30 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 0.2507; the observed delta is at least that size; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0.00009999000099990002,
            "min_effect": 0,
            "p_superiority": 0.00009999000099990002
          }
        ]
      }
    },
    {
      "source": "lme-s-dev-user-facts",
      "verdict": "pass",
      "arms": {
        "baseline": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7adf6c1f0d9df80f9bd2f5f6c75a32a83d330b9b8377ca86f11590a322dcc2c1",
          "loaded_git_head": "5d5375d438da490ad4d865fa93e3218a57e6bbdd"
        },
        "candidate": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7adf6c1f0d9df80f9bd2f5f6c75a32a83d330b9b8377ca86f11590a322dcc2c1",
          "loaded_git_head": "5d5375d438da490ad4d865fa93e3218a57e6bbdd"
        }
      },
      "family": {
        "family_id": "p2-attribution-dev:lme-s-dev-user-facts",
        "verdict": "pass",
        "reasons": [],
        "alpha": 0.05,
        "holm_family": [
          "user-qa-from-facts"
        ],
        "comparisons": [
          {
            "id": "user-qa-from-facts",
            "metric": "qa_score",
            "gate": "noninferiority",
            "status": "pass",
            "reasons": [],
            "n_pairs": 19,
            "n_clusters": 19,
            "n_excluded": 1,
            "stats": {
              "n_pairs": 19,
              "n_clusters": 19,
              "mean_a": 0.7368421052631579,
              "mean_b": 0.7368421052631579,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 0,
              "both_pass": 14,
              "both_fail": 5,
              "discordant": 0,
              "p_two_sided": 1
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": 6,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0,
            "tolerance": 0.02,
            "p_noninferiority": 0
          }
        ]
      }
    }
  ]
}