Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.1 KB
Raw Blame
{
  "schema_version": 1,
  "decision_id": "p6-think-dates",
  "plan": "P6",
  "title": "P6 R1: think date frame (current date + page content dates), notes off",
  "verdict_type": "quality",
  "stage": "dev",
  "candidate": {
    "gbrain": "/workspace/gbrain@0cba0ec9c35612d8652bcdaf55dbdb4f55d32c39",
    "config": {
      "think.reading_notes": "off"
    }
  },
  "baseline": {
    "gbrain": "/workspace/gbrain@5bd9e84978126689b11cf65107c0a9aa0e91e11b",
    "config": {}
  },
  "sources": [
    {
      "id": "lme-s-dev",
      "kind": "memory-qa",
      "benchmark": "lme-s",
      "split": "dev",
      "embed": "real",
      "limit": 150,
      "search_pins": {
        "search.mode": "balanced",
        "search.reranker.enabled": "false",
        "search.autocut": "false"
      },
      "top_k": 10,
      "id_field": "id",
      "exclude_when": [
        "abstention=true"
      ],
      "min_clusters": 10,
      "comparisons": [
        {
          "id": "recall-all-5",
          "metric": "recall_all_at_5",
          "gate": "noninferiority",
          "direction": "higher",
          "cluster_by": "conversation",
          "description": "guardrail: retrieval unchanged by the date frame",
          "tolerance": 0.01
        },
        {
          "id": "recall-any-5",
          "metric": "recall_any_at_5",
          "gate": "noninferiority",
          "direction": "higher",
          "tolerance": 0.01,
          "cluster_by": "conversation",
          "description": "at least one gold session in the top five"
        },
        {
          "id": "ndcg-10",
          "metric": "ndcg_at_10",
          "gate": "exploratory",
          "direction": "higher",
          "cluster_by": "conversation"
        },
        {
          "id": "latency",
          "metric": "latency_ms",
          "gate": "exploratory",
          "direction": "lower",
          "cluster_by": "conversation"
        }
      ],
      "estimate_usd": 24
    },
    {
      "id": "lme-s-dev-think",
      "kind": "memory-qa",
      "benchmark": "lme-s",
      "split": "dev",
      "embed": "real",
      "limit": 150,
      "search_pins": {
        "search.mode": "balanced",
        "search.reranker.enabled": "false",
        "search.autocut": "false"
      },
      "top_k": 10,
      "id_field": "id",
      "exclude_when": [],
      "min_clusters": 10,
      "comparisons": [
        {
          "id": "qa-score",
          "metric": "qa_score",
          "gate": "superiority",
          "direction": "higher",
          "min_effect": 0,
          "cluster_by": "conversation",
          "description": "judged answer correctness through production think"
        },
        {
          "id": "recall-all-5",
          "metric": "recall_all_at_5",
          "gate": "exploratory",
          "direction": "higher",
          "cluster_by": "conversation"
        },
        {
          "id": "qa-input-tokens",
          "metric": "qa_input_tokens",
          "gate": "exploratory",
          "direction": "lower",
          "cluster_by": "conversation"
        }
      ],
      "estimate_usd": 48,
      "qa": {
        "mode": "think",
        "runs": 1,
        "sessions": 5
      }
    },
    {
      "id": "locomo-dev",
      "kind": "memory-qa",
      "benchmark": "locomo",
      "split": "dev",
      "embed": "real",
      "limit": null,
      "search_pins": {
        "search.mode": "balanced",
        "search.reranker.enabled": "false",
        "search.autocut": "false"
      },
      "top_k": 10,
      "id_field": "id",
      "exclude_when": [
        "abstention=true"
      ],
      "min_clusters": 3,
      "comparisons": [
        {
          "id": "recall-all-5",
          "metric": "recall_all_at_5",
          "gate": "noninferiority",
          "direction": "higher",
          "tolerance": 0.01,
          "cluster_by": "conversation",
          "description": "guardrail: strict recall does not drop by more than one point"
        },
        {
          "id": "ndcg-10",
          "metric": "ndcg_at_10",
          "gate": "exploratory",
          "direction": "higher",
          "cluster_by": "conversation"
        },
        {
          "id": "latency",
          "metric": "latency_ms",
          "gate": "exploratory",
          "direction": "lower",
          "cluster_by": "conversation"
        }
      ],
      "estimate_usd": 2
    },
    {
      "id": "locomo-dev-think",
      "kind": "memory-qa",
      "benchmark": "locomo",
      "split": "dev",
      "embed": "real",
      "limit": null,
      "search_pins": {
        "search.mode": "balanced",
        "search.reranker.enabled": "false",
        "search.autocut": "false"
      },
      "top_k": 10,
      "id_field": "id",
      "exclude_when": [],
      "min_clusters": 3,
      "comparisons": [
        {
          "id": "qa-score",
          "metric": "qa_score",
          "gate": "superiority",
          "direction": "higher",
          "min_effect": 0,
          "cluster_by": "conversation",
          "description": "judged answer correctness through production think"
        },
        {
          "id": "recall-all-5",
          "metric": "recall_all_at_5",
          "gate": "exploratory",
          "direction": "higher",
          "cluster_by": "conversation"
        },
        {
          "id": "qa-input-tokens",
          "metric": "qa_input_tokens",
          "gate": "exploratory",
          "direction": "lower",
          "cluster_by": "conversation"
        }
      ],
      "estimate_usd": 20,
      "qa": {
        "mode": "think",
        "runs": 1,
        "sessions": 5
      }
    },
    {
      "id": "beam-100k-dev",
      "kind": "memory-qa",
      "benchmark": "beam-100k",
      "split": "dev",
      "embed": "real",
      "limit": null,
      "search_pins": {
        "search.mode": "balanced",
        "search.reranker.enabled": "false",
        "search.autocut": "false"
      },
      "top_k": 10,
      "id_field": "id",
      "exclude_when": [
        "abstention=true"
      ],
      "min_clusters": 10,
      "comparisons": [
        {
          "id": "recall-all-5",
          "metric": "recall_all_at_5",
          "gate": "noninferiority",
          "direction": "higher",
          "tolerance": 0.01,
          "cluster_by": "conversation",
          "description": "guardrail: strict recall does not drop by more than one point"
        },
        {
          "id": "ndcg-10",
          "metric": "ndcg_at_10",
          "gate": "exploratory",
          "direction": "higher",
          "cluster_by": "conversation"
        },
        {
          "id": "latency",
          "metric": "latency_ms",
          "gate": "exploratory",
          "direction": "lower",
          "cluster_by": "conversation"
        }
      ],
      "estimate_usd": 4
    }
  ],
  "alpha": 0.05,
  "seed": 42,
  "draws": 10000,
  "budget_usd": 150,
  "created_at": "2026-10-04T23:14:00.025Z",
  "notes": "R1 (date frame) isolates the current-date line and page content dates in think; reading notes stay off. LME-S think on the 150-question stratified sample is the primary; LoCoMo think dev (3 conversations) is supporting; retrieval sources are guardrails (the change also parses chat-export session dates)."
}