Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
28.8 KB
Raw Blame
{
  "decision_id": "p6-think-dates",
  "plan": "P6",
  "stage": "dev",
  "eligible_for_default": false,
  "eligibility_note": "Dev verdicts come from development data and never set a default. A held-out verdict comes from the custodian after a committed preregistration.",
  "verdict_type": "quality",
  "overall": "inconclusive",
  "decided_at": "2026-10-05T00:47:52.754Z",
  "sources": [
    {
      "source": "lme-s-dev",
      "verdict": "pass",
      "arms": {
        "baseline": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "5f9974dcaa7b13d1077ff6fdad3c24fd2b975cb4536efb4693663d0e4bc3c84c",
          "loaded_git_head": "5bd9e84978126689b11cf65107c0a9aa0e91e11b"
        },
        "candidate": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
          "loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
        }
      },
      "family": {
        "family_id": "p6-think-dates:lme-s-dev",
        "verdict": "pass",
        "reasons": [],
        "alpha": 0.05,
        "holm_family": [
          "recall-all-5",
          "recall-any-5"
        ],
        "comparisons": [
          {
            "id": "recall-all-5",
            "metric": "recall_all_at_5",
            "gate": "noninferiority",
            "status": "pass",
            "reasons": [],
            "n_pairs": 142,
            "n_clusters": 142,
            "n_excluded": 8,
            "stats": {
              "n_pairs": 142,
              "n_clusters": 142,
              "mean_a": 0.9436619718309859,
              "mean_b": 0.9436619718309859,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 0,
              "both_pass": 134,
              "both_fail": 8,
              "discordant": 0,
              "p_two_sided": 1
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": 6,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0,
            "tolerance": 0.01,
            "p_noninferiority": 0
          },
          {
            "id": "recall-any-5",
            "metric": "recall_any_at_5",
            "gate": "noninferiority",
            "status": "pass",
            "reasons": [],
            "n_pairs": 142,
            "n_clusters": 142,
            "n_excluded": 8,
            "stats": {
              "n_pairs": 142,
              "n_clusters": 142,
              "mean_a": 0.9788732394366197,
              "mean_b": 0.9788732394366197,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 0,
              "both_pass": 139,
              "both_fail": 3,
              "discordant": 0,
              "p_two_sided": 1
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": 6,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0,
            "tolerance": 0.01,
            "p_noninferiority": 0
          },
          {
            "id": "ndcg-10",
            "metric": "ndcg_at_10",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 142,
            "n_clusters": 142,
            "n_excluded": 8,
            "stats": {
              "n_pairs": 142,
              "n_clusters": 142,
              "mean_a": 0.9482427099525481,
              "mean_b": 0.9482427099525481,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": null,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data"
            },
            "direction": "higher"
          },
          {
            "id": "latency",
            "metric": "latency_ms",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 142,
            "n_clusters": 142,
            "n_excluded": 8,
            "stats": {
              "n_pairs": 142,
              "n_clusters": 142,
              "mean_a": 286.23873239436625,
              "mean_b": 54.86901408450705,
              "delta": -231.36971830985917,
              "se_cluster": 7.3034391039028845,
              "se_iid": 7.303439103902884,
              "design_effect": 1.0000000000000004,
              "effective_n": 141.99999999999994,
              "ci95": [
                -248.1492957746478,
                -219.9802816901409
              ],
              "p_two_sided": 0.00009999000099990002,
              "p_greater": 1,
              "p_less": 0.00009999000099990002,
              "p_method": "monte-carlo-sign-flip"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": 20.46120704024214,
              "min_one_sided_discordant": null,
              "note": "with 142 pairs in 142 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 20.4612; the observed delta is at least that size"
            },
            "direction": "lower"
          }
        ]
      }
    },
    {
      "source": "lme-s-dev-think",
      "verdict": "pass",
      "arms": {
        "baseline": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "5f9974dcaa7b13d1077ff6fdad3c24fd2b975cb4536efb4693663d0e4bc3c84c",
          "loaded_git_head": "5bd9e84978126689b11cf65107c0a9aa0e91e11b"
        },
        "candidate": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
          "loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
        }
      },
      "family": {
        "family_id": "p6-think-dates:lme-s-dev-think",
        "verdict": "pass",
        "reasons": [],
        "alpha": 0.05,
        "holm_family": [
          "qa-score"
        ],
        "comparisons": [
          {
            "id": "qa-score",
            "metric": "qa_score",
            "gate": "superiority",
            "status": "pass",
            "reasons": [],
            "n_pairs": 150,
            "n_clusters": 150,
            "n_excluded": 0,
            "stats": {
              "n_pairs": 150,
              "n_clusters": 150,
              "mean_a": 0.8066666666666666,
              "mean_b": 0.9,
              "delta": 0.09333333333333334,
              "se_cluster": 0.028920421040663936,
              "se_iid": 0.028920421040663932,
              "design_effect": 1.0000000000000004,
              "effective_n": 149.99999999999994,
              "ci95": [
                0.04,
                0.15333333333333332
              ],
              "p_two_sided": 0.0022997700229977,
              "p_greater": 0.000999900009999,
              "p_less": 0.9997000299970003,
              "p_method": "monte-carlo-sign-flip"
            },
            "mcnemar": {
              "losses": 3,
              "wins": 17,
              "both_pass": 118,
              "both_fail": 12,
              "discordant": 20,
              "p_two_sided": 0.0025768280029296875
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": 0.08102302411035046,
              "min_one_sided_discordant": 6,
              "note": "with 150 pairs in 150 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 0.0810; the observed delta is at least that size; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0.00029997000299970003,
            "min_effect": 0,
            "p_superiority": 0.00029997000299970003
          },
          {
            "id": "recall-all-5",
            "metric": "recall_all_at_5",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 142,
            "n_clusters": 142,
            "n_excluded": 8,
            "stats": {
              "n_pairs": 142,
              "n_clusters": 142,
              "mean_a": 0.9436619718309859,
              "mean_b": 0.9436619718309859,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 0,
              "both_pass": 134,
              "both_fail": 8,
              "discordant": 0,
              "p_two_sided": 1
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": 6,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher"
          },
          {
            "id": "qa-input-tokens",
            "metric": "qa_input_tokens",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 150,
            "n_clusters": 150,
            "n_excluded": 0,
            "stats": {
              "n_pairs": 150,
              "n_clusters": 150,
              "mean_a": 23.213333333333335,
              "mean_b": 23.213333333333335,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": null,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data"
            },
            "direction": "lower"
          }
        ]
      }
    },
    {
      "source": "locomo-dev",
      "verdict": "pass",
      "arms": {
        "baseline": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "5f9974dcaa7b13d1077ff6fdad3c24fd2b975cb4536efb4693663d0e4bc3c84c",
          "loaded_git_head": "5bd9e84978126689b11cf65107c0a9aa0e91e11b"
        },
        "candidate": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
          "loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
        }
      },
      "family": {
        "family_id": "p6-think-dates:locomo-dev",
        "verdict": "pass",
        "reasons": [],
        "alpha": 0.05,
        "holm_family": [
          "recall-all-5"
        ],
        "comparisons": [
          {
            "id": "recall-all-5",
            "metric": "recall_all_at_5",
            "gate": "noninferiority",
            "status": "pass",
            "reasons": [
              "McNemar treats items as independent; the clustered sign-flip p-value is the primary test here"
            ],
            "n_pairs": 464,
            "n_clusters": 3,
            "n_excluded": 123,
            "stats": {
              "n_pairs": 464,
              "n_clusters": 3,
              "mean_a": 0.7564655172413793,
              "mean_b": 0.7564655172413793,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 0,
              "both_pass": 351,
              "both_fail": 113,
              "discordant": 0,
              "p_two_sided": 1
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": 6,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0,
            "tolerance": 0.01,
            "p_noninferiority": 0
          },
          {
            "id": "ndcg-10",
            "metric": "ndcg_at_10",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 464,
            "n_clusters": 3,
            "n_excluded": 123,
            "stats": {
              "n_pairs": 464,
              "n_clusters": 3,
              "mean_a": 0.7406125386759213,
              "mean_b": 0.7406125386759213,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": null,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data"
            },
            "direction": "higher"
          },
          {
            "id": "latency",
            "metric": "latency_ms",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 464,
            "n_clusters": 3,
            "n_excluded": 123,
            "stats": {
              "n_pairs": 464,
              "n_clusters": 3,
              "mean_a": 231.07262931034472,
              "mean_b": 21.83879310344827,
              "delta": -209.23383620689648,
              "se_cluster": 5.950696173289319,
              "se_iid": 2.968129301822358,
              "design_effect": 4.019480453814904,
              "effective_n": 115.4378047938051,
              "ci95": [
                -217.51056910569096,
                -199.61518324607314
              ],
              "p_two_sided": 0.25,
              "p_greater": 1,
              "p_less": 0.125,
              "p_method": "exact-sign-flip"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": 16.671382440935652,
              "min_one_sided_discordant": null,
              "note": "with 464 pairs in 3 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 16.6714; the observed delta is at least that size; clustering inflates variance by 4.02x (effective n about 115.4 of 464)"
            },
            "direction": "lower"
          }
        ]
      }
    },
    {
      "source": "locomo-dev-think",
      "verdict": "pass",
      "arms": {
        "baseline": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "5f9974dcaa7b13d1077ff6fdad3c24fd2b975cb4536efb4693663d0e4bc3c84c",
          "loaded_git_head": "5bd9e84978126689b11cf65107c0a9aa0e91e11b"
        },
        "candidate": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
          "loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
        }
      },
      "family": {
        "family_id": "p6-think-dates:locomo-dev-think",
        "verdict": "pass",
        "reasons": [],
        "alpha": 0.05,
        "holm_family": [
          "qa-score"
        ],
        "comparisons": [
          {
            "id": "qa-score",
            "metric": "qa_score",
            "gate": "superiority",
            "status": "pass",
            "reasons": [
              "McNemar treats items as independent; the clustered sign-flip p-value is the primary test here"
            ],
            "n_pairs": 587,
            "n_clusters": 3,
            "n_excluded": 0,
            "stats": {
              "n_pairs": 587,
              "n_clusters": 3,
              "mean_a": 0.7666098807495741,
              "mean_b": 0.8909710391822828,
              "delta": 0.12436115843270869,
              "se_cluster": 0.006212078462093329,
              "se_iid": 0.01652399156017386,
              "design_effect": 0.14133311663356926,
              "effective_n": 4153.308254864993,
              "ci95": [
                0.11392405063291139,
                0.1368421052631579
              ],
              "p_two_sided": 0.25,
              "p_greater": 0.125,
              "p_less": 1,
              "p_method": "exact-sign-flip"
            },
            "mcnemar": {
              "losses": 15,
              "wins": 88,
              "both_pass": 435,
              "both_fail": 49,
              "discordant": 103,
              "p_two_sided": 9.654706383228215e-14
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": 0.01740366719771732,
              "min_one_sided_discordant": 6,
              "note": "with 587 pairs in 3 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 0.0174; the observed delta is at least that size; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0.00009999000099990002,
            "min_effect": 0,
            "p_superiority": 0.00009999000099990002
          },
          {
            "id": "recall-all-5",
            "metric": "recall_all_at_5",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [
              "McNemar treats items as independent; the clustered sign-flip p-value is the primary test here"
            ],
            "n_pairs": 464,
            "n_clusters": 3,
            "n_excluded": 123,
            "stats": {
              "n_pairs": 464,
              "n_clusters": 3,
              "mean_a": 0.7564655172413793,
              "mean_b": 0.7564655172413793,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 0,
              "both_pass": 351,
              "both_fail": 113,
              "discordant": 0,
              "p_two_sided": 1
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": 6,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher"
          },
          {
            "id": "qa-input-tokens",
            "metric": "qa_input_tokens",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 587,
            "n_clusters": 3,
            "n_excluded": 0,
            "stats": {
              "n_pairs": 587,
              "n_clusters": 3,
              "mean_a": 15,
              "mean_b": 15,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": null,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data"
            },
            "direction": "lower"
          }
        ]
      }
    },
    {
      "source": "beam-100k-dev",
      "verdict": "inconclusive",
      "arms": {
        "baseline": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "5f9974dcaa7b13d1077ff6fdad3c24fd2b975cb4536efb4693663d0e4bc3c84c",
          "loaded_git_head": "5bd9e84978126689b11cf65107c0a9aa0e91e11b"
        },
        "candidate": {
          "package": "gbrain",
          "declared_pin": "github:garrytan/gbrain#739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "declared_sha": "739e5cc89ca43b9b9351f0f203c7b12a7c0c571c",
          "version": "0.60.48.0",
          "package_sha256": "7c1796a95adc86aeb9c8b6f5b603b4b3943bd0b508d607a569d75e703590ebe0",
          "loaded_git_head": "0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39"
        }
      },
      "family": {
        "family_id": "p6-think-dates:beam-100k-dev",
        "verdict": "inconclusive",
        "reasons": [
          "recall-all-5: inconclusive"
        ],
        "alpha": 0.05,
        "holm_family": [
          "recall-all-5"
        ],
        "comparisons": [
          {
            "id": "recall-all-5",
            "metric": "recall_all_at_5",
            "gate": "noninferiority",
            "status": "inconclusive",
            "reasons": [
              "McNemar treats items as independent; the clustered sign-flip p-value is the primary test here",
              "6 clusters is below the family minimum of 10"
            ],
            "n_pairs": 108,
            "n_clusters": 6,
            "n_excluded": 12,
            "stats": {
              "n_pairs": 108,
              "n_clusters": 6,
              "mean_a": 0.4537037037037037,
              "mean_b": 0.4537037037037037,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "mcnemar": {
              "losses": 0,
              "wins": 0,
              "both_pass": 49,
              "both_fail": 59,
              "discordant": 0,
              "p_two_sided": 1
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": 6,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data; an exact McNemar test needs at least 6 wins with zero losses to reach 0.05"
            },
            "direction": "higher",
            "p_holm": 0,
            "tolerance": 0.01,
            "p_noninferiority": 0
          },
          {
            "id": "ndcg-10",
            "metric": "ndcg_at_10",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 108,
            "n_clusters": 6,
            "n_excluded": 12,
            "stats": {
              "n_pairs": 108,
              "n_clusters": 6,
              "mean_a": 0.5676755153728674,
              "mean_b": 0.5676755153728674,
              "delta": 0,
              "se_cluster": 0,
              "se_iid": 0,
              "design_effect": null,
              "effective_n": null,
              "ci95": [
                0,
                0
              ],
              "p_two_sided": 1,
              "p_greater": 1,
              "p_less": 1,
              "p_method": "degenerate"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": null,
              "min_one_sided_discordant": null,
              "note": "no paired variation was observed, so the detectable effect cannot be estimated from this data"
            },
            "direction": "higher"
          },
          {
            "id": "latency",
            "metric": "latency_ms",
            "gate": "exploratory",
            "status": "report_only",
            "reasons": [],
            "n_pairs": 108,
            "n_clusters": 6,
            "n_excluded": 12,
            "stats": {
              "n_pairs": 108,
              "n_clusters": 6,
              "mean_a": 279.27592592592583,
              "mean_b": 61.330555555555534,
              "delta": -217.94537037037037,
              "se_cluster": 2.8257944061914877,
              "se_iid": 4.2480772398609075,
              "design_effect": 0.442482717747983,
              "effective_n": 244.07732927890675,
              "ci95": [
                -223.2462962962963,
                -213.1351851851852
              ],
              "p_two_sided": 0.03125,
              "p_greater": 1,
              "p_less": 0.015625,
              "p_method": "exact-sign-flip"
            },
            "power": {
              "alpha": 0.05,
              "power": 0.8,
              "mde": 7.916703839886082,
              "min_one_sided_discordant": null,
              "note": "with 108 pairs in 6 clusters, a two-sided 0.05 test has 80% power for an absolute change of about 7.9167; the observed delta is at least that size"
            },
            "direction": "lower"
          }
        ]
      }
    }
  ]
}