Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

5.6 KB
Raw Blame
[
  {
    "id": "engineering-matched",
    "dataset": "engineering",
    "kind": "matched",
    "repeats": 2,
    "planned_calls": 1536,
    "started_calls": 1536,
    "result_calls": 1536,
    "unknown_outcomes": [],
    "not_started_calls": 0,
    "expected_models": {
      "jev114": "typesafe/jev-1.14-20261014",
      "jev113": "typesafe/jev-1.13-20260917",
      "qwen38flash": "qwen3.8-flash"
    },
    "jobs": "execution/engineering-matched/jobs.jsonl.gz",
    "requests": "execution/engineering-matched/requests.jsonl.gz",
    "responses": "responses/engineering-matched.jsonl.gz"
  },
  {
    "id": "engineering-independent",
    "dataset": "engineering",
    "kind": "independent",
    "repeats": 2,
    "planned_calls": 1792,
    "started_calls": 1792,
    "result_calls": 1792,
    "unknown_outcomes": [],
    "not_started_calls": 0,
    "expected_models": {
      "jev113": "typesafe/jev-1.13-20260917",
      "jev114": "typesafe/jev-1.14-20261014",
      "qwen38flash": "qwen3.8-flash"
    },
    "jobs": "execution/engineering-independent/jobs.jsonl.gz",
    "requests": "execution/engineering-independent/requests.jsonl.gz",
    "responses": "responses/engineering-independent.jsonl.gz"
  },
  {
    "id": "mathematics-matched",
    "dataset": "mathematics",
    "kind": "matched",
    "repeats": 2,
    "planned_calls": 768,
    "started_calls": 768,
    "result_calls": 768,
    "unknown_outcomes": [],
    "not_started_calls": 0,
    "expected_models": {
      "jev113": "typesafe/jev-1.13-20260917",
      "qwen38flash": "qwen3.8-flash",
      "jev114": "typesafe/jev-1.14-20261014"
    },
    "jobs": "execution/mathematics-matched/jobs.jsonl.gz",
    "requests": "execution/mathematics-matched/requests.jsonl.gz",
    "responses": "responses/mathematics-matched.jsonl.gz"
  },
  {
    "id": "direct-expression",
    "dataset": "engineering",
    "kind": "expression",
    "repeats": 2,
    "planned_calls": 1024,
    "started_calls": 1024,
    "result_calls": 1024,
    "unknown_outcomes": [],
    "not_started_calls": 0,
    "expected_models": {
      "jev113": "typesafe/jev-1.13-20260917",
      "jev114": "typesafe/jev-1.14-20261014"
    },
    "jobs": "execution/direct-expression/jobs.jsonl.gz",
    "requests": "execution/direct-expression/requests.jsonl.gz",
    "responses": "responses/direct-expression.jsonl.gz"
  },
  {
    "id": "direct-probes",
    "dataset": "direct-probes",
    "kind": "probes",
    "repeats": 1,
    "planned_calls": 32,
    "started_calls": 32,
    "result_calls": 32,
    "unknown_outcomes": [],
    "not_started_calls": 0,
    "expected_models": {
      "jev113": "typesafe/jev-1.13-20260917",
      "jev114": "typesafe/jev-1.14-20261014"
    },
    "jobs": "execution/direct-probes/jobs.jsonl.gz",
    "requests": "execution/direct-probes/requests.jsonl.gz",
    "responses": "responses/direct-probes.jsonl.gz"
  },
  {
    "id": "development-v3",
    "dataset": "development-v3",
    "kind": "development",
    "repeats": 2,
    "planned_calls": 6912,
    "started_calls": 6912,
    "result_calls": 6912,
    "unknown_outcomes": [],
    "not_started_calls": 0,
    "expected_models": {
      "jev113": "typesafe/jev-1.13-20260917",
      "jev114": "typesafe/jev-1.14-20261014",
      "qwen38flash": "qwen3.8-flash"
    },
    "jobs": "execution/development-v3/jobs.jsonl.gz",
    "requests": "execution/development-v3/requests.jsonl.gz",
    "responses": "responses/development-v3.jsonl.gz"
  },
  {
    "id": "aborted-engineering",
    "dataset": "engineering",
    "kind": "aborted",
    "repeats": 2,
    "planned_calls": 1536,
    "started_calls": 598,
    "result_calls": 594,
    "unknown_outcomes": [
      "IG009_a_zh/jev113/decomposed/1",
      "IG010_a_en/jev114/direct/1",
      "IG021_a_reworded_zh/jev114/direct/1",
      "IG026_b_en/jev114/direct/1"
    ],
    "not_started_calls": 938,
    "expected_models": {
      "jev114": "typesafe/jev-1.14-20261014",
      "jev113": "typesafe/jev-1.13-20260917",
      "qwen38flash": "qwen3.8-flash"
    },
    "jobs": "execution/aborted-engineering/jobs.jsonl.gz",
    "requests": "execution/aborted-engineering/requests.jsonl.gz",
    "responses": "responses/aborted-engineering.jsonl.gz"
  },
  {
    "id": "format-development-completed",
    "dataset": "engineering",
    "kind": "format_development",
    "repeats": 1,
    "planned_calls": 20,
    "result_calls": 20,
    "ledger_scope": "Original result-only developer ledger; no start events recorded. Unobserved plan entries are not imputed as observed starts. Not a headline benchmark.",
    "unobserved_plan_entries": 0,
    "expected_models": {
      "qwen38flash": "qwen3.8-flash",
      "jev113": "typesafe/jev-1.13-20260917",
      "jev114": "typesafe/jev-1.14-20261014"
    },
    "jobs": "execution/format-development-completed/jobs.jsonl.gz",
    "requests": "execution/format-development-completed/requests.jsonl.gz",
    "responses": "responses/format-development-completed.jsonl.gz"
  },
  {
    "id": "format-development-stopped",
    "dataset": "engineering",
    "kind": "format_development",
    "repeats": 1,
    "planned_calls": 20,
    "result_calls": 1,
    "ledger_scope": "Original result-only developer ledger; no start events recorded. Unobserved plan entries are not imputed as observed starts. Not a headline benchmark.",
    "unobserved_plan_entries": 19,
    "expected_models": {
      "qwen38flash": "qwen3.8-flash",
      "jev113": "typesafe/jev-1.13-20260917",
      "jev114": "typesafe/jev-1.14-20261014"
    },
    "jobs": "execution/format-development-stopped/jobs.jsonl.gz",
    "requests": "execution/format-development-stopped/requests.jsonl.gz",
    "responses": "responses/format-development-stopped.jsonl.gz"
  }
]