Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
3.2 KB
Raw Blame
{
  "schema_version": 1,
  "decision_id": "p3-feedback-guardrail-dev-v2",
  "plan": "P3",
  "title": "P3 retrieval feedback guardrail (dev): default-on feedback with no ratings must not change retrieval",
  "verdict_type": "quality",
  "stage": "dev",
  "candidate": {
    "gbrain": "/workspace/gbrain@5e5fdcbbe55827998dd22df2e16aa777d522520f",
    "config": {}
  },
  "baseline": {
    "gbrain": "/workspace/gbrain@5bd9e84978126689b11cf65107c0a9aa0e91e11b",
    "config": {}
  },
  "sources": [
    {
      "id": "lme-s-dev",
      "kind": "memory-qa",
      "benchmark": "lme-s",
      "split": "dev",
      "embed": "real",
      "limit": 100,
      "search_pins": {
        "search.mode": "balanced",
        "search.reranker.enabled": "false",
        "search.autocut": "false"
      },
      "top_k": 10,
      "id_field": "id",
      "exclude_when": [
        "abstention=true"
      ],
      "min_clusters": 10,
      "comparisons": [
        {
          "id": "recall-all-5",
          "metric": "recall_all_at_5",
          "gate": "noninferiority",
          "direction": "higher",
          "tolerance": 0.01,
          "cluster_by": "conversation",
          "description": "guardrail: strict recall does not drop by more than one point"
        },
        {
          "id": "ndcg-10",
          "metric": "ndcg_at_10",
          "gate": "exploratory",
          "direction": "higher",
          "cluster_by": "conversation"
        },
        {
          "id": "latency",
          "metric": "latency_ms",
          "gate": "exploratory",
          "direction": "lower",
          "cluster_by": "conversation"
        }
      ],
      "estimate_usd": 6
    },
    {
      "id": "locomo-dev",
      "kind": "memory-qa",
      "benchmark": "locomo",
      "split": "dev",
      "embed": "real",
      "limit": null,
      "search_pins": {
        "search.mode": "balanced",
        "search.reranker.enabled": "false",
        "search.autocut": "false"
      },
      "top_k": 10,
      "id_field": "id",
      "exclude_when": [
        "abstention=true"
      ],
      "min_clusters": 3,
      "comparisons": [
        {
          "id": "recall-all-5",
          "metric": "recall_all_at_5",
          "gate": "noninferiority",
          "direction": "higher",
          "tolerance": 0.01,
          "cluster_by": "conversation",
          "description": "guardrail: strict recall does not drop by more than one point"
        },
        {
          "id": "ndcg-10",
          "metric": "ndcg_at_10",
          "gate": "exploratory",
          "direction": "higher",
          "cluster_by": "conversation"
        },
        {
          "id": "latency",
          "metric": "latency_ms",
          "gate": "exploratory",
          "direction": "lower",
          "cluster_by": "conversation"
        }
      ],
      "estimate_usd": 2
    }
  ],
  "alpha": 0.05,
  "seed": 42,
  "draws": 10000,
  "budget_usd": 8,
  "created_at": "2026-10-04T18:45:19.479Z",
  "notes": "Use-attributed feedback weights: the sequential replay-with-feedback mode lands in milestone M2; until then these are guardrails only. LME-S trimmed to a 100-question category-stratified subset (selectQuestions, seed 42) to stay inside the plan E3 cap; embeddings warm in the content-addressed cache so both arms read identical vectors."
}