Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

feat(jev): add diagnostics and configurable routing criteria (#22)

澄潭43f9910f3187
diff --git a/apps/website/scripts/prepare-content.mjs b/apps/website/scripts/prepare-content.mjs
index c5cc4a43..cbb19775 100644
--- a/apps/website/scripts/prepare-content.mjs
+++ b/apps/website/scripts/prepare-content.mjs
@@ -21,6 +21,7 @@ await fs.mkdir(installDirectory, { recursive: true });
// Reuse the Desktop logo for raster-only link preview services.
await fs.copyFile(path.join(repository, 'apps/desktop/src-tauri/icons/icon.png'), path.join(publicDirectory, 'brand/app-icon.png'));
await fs.copyFile(path.join(source, 'api/decision.openapi.json'), path.join(apiDirectory, 'decision.openapi.json'));
+await fs.copyFile(path.join(source, 'extensions/jev-decider/jev_decider/policy.default.json'), path.join(apiDirectory, 'jev-policy.default.json'));
for (const entry of await fs.readdir(path.join(source, 'assets'), { withFileTypes: true })) {
if (entry.isFile() && /\.(png|svg)$/.test(entry.name)) {
await fs.copyFile(path.join(source, 'assets', entry.name), path.join(assetsDirectory, entry.name));
diff --git a/apps/website/scripts/validate-build.mjs b/apps/website/scripts/validate-build.mjs
index 033c8850..bfa32f38 100644
--- a/apps/website/scripts/validate-build.mjs
+++ b/apps/website/scripts/validate-build.mjs
@@ -15,7 +15,8 @@ const required = [
'docs/decision-extensions/index.html', 'en/docs/decision-extensions/index.html',
'docs/decision-api/index.html', 'en/docs/decision-api/index.html',
'docs/jev-decider/index.html', 'en/docs/jev-decider/index.html',
- 'api/decision.openapi.json', 'install.sh', 'install/standalone.py', '404.html', 'sitemap-index.xml',
+ 'api/decision.openapi.json', 'api/jev-policy.default.json',
+ 'install.sh', 'install/standalone.py', '404.html', 'sitemap-index.xml',
];
for (const relative of required) {
if (!fs.existsSync(path.join(dist, relative))) throw new Error(`missing website output: ${relative}`);
diff --git a/apps/website/src/lib/decision-docs.mjs b/apps/website/src/lib/decision-docs.mjs
index d4ecc823..76e4050d 100644
--- a/apps/website/src/lib/decision-docs.mjs
+++ b/apps/website/src/lib/decision-docs.mjs
@@ -28,6 +28,7 @@ const maps = {
'README.md': '/en/docs/jev-decider/', 'README.zh-CN.md': '/docs/jev-decider/',
'../../README.md': '/en/docs/decision-extensions/', '../../README.zh-CN.md': '/docs/decision-extensions/',
'../../api/README.md': '/en/docs/decision-api/', '../../api/README.zh-CN.md': '/docs/decision-api/',
+ 'jev_decider/policy.default.json': '/api/jev-policy.default.json',
},
};
diff --git a/apps/website/tests/decision-docs.test.mjs b/apps/website/tests/decision-docs.test.mjs
index 5de32c84..e09eecf3 100644
--- a/apps/website/tests/decision-docs.test.mjs
+++ b/apps/website/tests/decision-docs.test.mjs
@@ -6,12 +6,16 @@ import { renderDecisionDoc } from '../src/lib/decision-docs.mjs';
test('canonical decision documents render into bilingual website routes', async () => {
const zh = await renderDecisionDoc('mechanism', 'zh');
const en = await renderDecisionDoc('api', 'en');
+ const jevZh = await renderDecisionDoc('jev', 'zh');
+ const jevEn = await renderDecisionDoc('jev', 'en');
const rendered = await Promise.all(['mechanism', 'api', 'jev'].flatMap(page =>
['zh', 'en'].map(language => renderDecisionDoc(page, language))));
assert.match(zh, /href="\/docs\/jev-decider\/"/);
assert.match(zh, /src="\/decision-assets\/quality-zh-CN\.png"/);
assert.match(en, /href="\/api\/decision\.openapi\.json"/);
assert.match(en, /href="\/en\/docs\/model-routing\/"/);
+ assert.match(jevZh, /href="\/api\/jev-policy\.default\.json"/);
+ assert.match(jevEn, /href="\/api\/jev-policy\.default\.json"/);
assert.match(zh, /href="#illustration-evidence-boundaries"/);
assert.match(zh, /id="illustration-evidence-boundaries"/);
assert.match(rendered.find(html => html.includes('Illustration evidence boundaries')), /id="illustration-evidence-boundaries"/);
@@ -26,6 +30,12 @@ test('prepared OpenAPI is byte-identical to the one repository contract', () =>
assert(parsed.paths['/v1/decisions']);
});
+test('prepared Jev policy is byte-identical to the bundled default', () => {
+ const canonical = fs.readFileSync(new URL('../../../decision-extensions/extensions/jev-decider/jev_decider/policy.default.json', import.meta.url));
+ const prepared = fs.readFileSync(new URL('../public/api/jev-policy.default.json', import.meta.url));
+ assert.deepEqual(prepared, canonical);
+});
+
test('native homepage screenshots are distinct, correctly sized and copied unchanged', () => {
const captures = ['en', 'zh-CN'].map(language => {
const name = `quality-native-${language}.png`;
diff --git a/decision-extensions/extensions/jev-decider/README.md b/decision-extensions/extensions/jev-decider/README.md
index 08bdfe2a..57074573 100644
--- a/decision-extensions/extensions/jev-decider/README.md
+++ b/decision-extensions/extensions/jev-decider/README.md
@@ -72,6 +72,7 @@ The container build command runs from the repository root. Existing deployments
| `OPENROUTER_API_KEY_FILE` | required | Absolute path to a UTF-8 file; its value is sent only as the OpenRouter Bearer credential |
| `JEV_MODE` | `auto` | Exactly `auto` or `rules` |
| `JEV_MODEL` | `typesafe/jev-1.13` | OpenRouter Decisions model |
+| `JEV_POLICY_FILE` | Bundled criteria | Optional absolute path to custom criteria JSON; loaded at startup, restart after edits |
| `OPENROUTER_DECISIONS_URL` | `https://openrouter.ai/api/alpha/decisions` | Override only for a controlled gateway/test upstream |
| `JEV_REQUEST_TIMEOUT_SECONDS` | `2.8` | Whole request budget, including validation, context preparation, queueing and the upstream call; must be in `(0, 3600]`; set it slightly below the matching HiRoute plan's `timeout_ms` and keep the default for a 3000 ms plan |
| `JEV_MAX_STATE_TOKENS` | `24000` | Conservative state budget described below |
@@ -85,6 +86,32 @@ Outbound OpenRouter calls honor the deployment's standard `HTTP_PROXY`, `HTTPS_P
If inbound authentication is enabled, store the full header value in a HiRoute Secret and configure the matching `auth_header`. For example, the Secret may contain `Bearer ...`; HiRoute does not prepend a scheme.
+### Define simple and complex work
+
+Smart-saving decisions use the bundled [policy.default.json](jev_decider/policy.default.json)
+without requiring an environment variable or a copied file. To customize it, copy and
+edit the file, then set `JEV_POLICY_FILE=/absolute/path/policy.json`. The UTF-8 JSON file must be at most
+4096 bytes and contain exactly two nonempty strings, `simple` and `complex`.
+Describe work appropriate for your economy and primary models, optionally with
+a few general examples, not a desired answer for a specific benchmark.
+
+The policy is shared by all plans using this service instance; it is not a Desktop
+plan field. Both auto and rules use the bundled or custom definitions for the two
+smart-saving branches, replacing descriptions in upstream state and Choice criteria.
+An explicit custom policy accepts only those two branches. Without an override, auto
+still uses request descriptions for other branch sets. The allowed branch set is
+never expanded. Invalid files prevent startup instead of reverting to defaults. Changes
+require a restart; there is no hot reload, policy DSL, or new session store.
+Effective descriptions count toward the context budget. Rules thresholds and
+competence scoring are unchanged: broader simple criteria cannot bypass the
+low-competence guard.
+
+Startup logs include the effective smart-saving criteria hash.
+Decision logs include the effective `criteria_source` (default/policy/request) and
+`criteria_sha256`, never the descriptions themselves. Freeze criteria and use
+independent tasks to calibrate thresholds; neither the definitions nor Jev's
+probabilities establish the economy model's actual success rate.
+
## Context and privacy boundary
HiRoute sends the complete, non-empty `latest_user` plus its in-memory `visible_conversation`. A decision can begin because there is no inheritable branch, ContextHold no longer has a preferred candidate, or a user message was appended while the plan's follow-up re-selection setting was enabled; new plans default to disabled. The service does not detect compaction. `latest_user` can therefore repeat a prior round or be a client-generated summary/continuation. That shape is not itself feedback. Each visible item is a sealed routing execution round, which may have inherited its decision; an item sealed at a decision boundary may have `unknown` status while still containing assessable progress.
@@ -124,6 +151,24 @@ To implement a different strategy, keep the HTTP validation and response contrac
## Offline tests
+### Decision logs
+
+The CLI emits JSON events to stderr at INFO level. Redirect stderr to a file or use
+container logs. `service_started` records effective mode, model, thresholds and limits.
+Each decision has a generated `decision_id`, returned in `X-Jev-Decision-Id`, linking
+`decision_started` to `decision_completed`, `decision_cancelled` or `decision_failed`.
+This ID is local to the decider; HiRoute does not currently persist it.
+
+Results include upstream choice, rules probabilities, normalized competence and its
+validity, threshold pass/fail flags, final branch and rule reason (`auto_choice`,
+`complexity_threshold`, `competence_guard`, or `economy_eligible`). History size,
+assessment/trimming boundaries, queue and total duration, HTTP status and failure phase
+are also recorded. Missing competence retains the existing non-blocking behavior.
+These explain the rule, not model-generated reasoning. Request text, credentials and
+raw upstream bodies are not logged. Embedded hosts must enable `jev_decider` at INFO.
+
+### Running tests
+
The suite starts the real HTTP handler and a controlled upstream server. It does not read a real key or access the network:
```sh
diff --git a/decision-extensions/extensions/jev-decider/README.zh-CN.md b/decision-extensions/extensions/jev-decider/README.zh-CN.md
index b92f96f1..b7aac789 100644
--- a/decision-extensions/extensions/jev-decider/README.zh-CN.md
+++ b/decision-extensions/extensions/jev-decider/README.zh-CN.md
@@ -1,5 +1,19 @@
# HiRoute Jev 决策器
+## 决策日志
+
+CLI 默认以 INFO 级别向 stderr 输出 JSON 事件,可重定向到文件或由容器收集。
+`service_started` 记录实际模式、模型、阈值及资源限制。每次请求生成 `decision_id`,
+通过响应头 `X-Jev-Decision-Id` 返回,关联开始、完成、取消或异常事件。
+这个 ID 属于决策服务,HiRoute 当前尚未持久化该响应头。
+
+结果包含上游原始选择、rules 概率、归一化能力分及其有效性、阈值判断、最终分支、
+规则原因、历史裁剪与评估边界、排队及总耗时、HTTP 状态和失败阶段。
+规则原因为 `auto_choice`、`complexity_threshold`、`competence_guard` 或 `economy_eligible`。
+缺失能力分仍沿用当前“不阻止省钱分支”的规则,日志会明确记录缺失。
+这里记录的是规则依据,不是模型生成的推理过程;不记录正文、凭证或上游原始响应。
+以库方式运行时,需要将 `jev_decider` logger 启用为 INFO。
+
[English](README.md) · [决策机制](../../README.zh-CN.md) · [API 与 OpenAPI](../../api/README.zh-CN.md)
这是可以自行部署的官方参考扩展:接收 HiRoute 的五字段决策请求,通过一次 OpenRouter `typesafe/jev-1.13` Decisions 调用,返回分支选择与可选的上一执行阶段胜任度评分。它是独立受信服务,HiRoute 负责执行、路由执行轮次边界、评分归属和持久化,服务负责策略、提示词与模型上下文裁剪。部署后,可见对话投影会发往 OpenRouter/TypeSafe。
@@ -62,6 +76,7 @@ HiRoute 的智能省钱计划配置:
| `OPENROUTER_API_KEY_FILE` | 必填 | UTF-8 key 文件的绝对路径,仅用于出站 Bearer 认证 |
| `JEV_MODE` | `auto` | `auto` 或 `rules` 二选一 |
| `JEV_MODEL` | `typesafe/jev-1.13` | OpenRouter Decisions 模型 |
+| `JEV_POLICY_FILE` | 内置判断标准 | 可选自定义定义 JSON 文件的绝对路径;启动读取,修改后重启 |
| `OPENROUTER_DECISIONS_URL` | `https://openrouter.ai/api/alpha/decisions` | 仅在受控网关或测试时覆盖 |
| `JEV_REQUEST_TIMEOUT_SECONDS` | `2.8` | 包含校验、上下文、排队及上游调用的总预算,范围 `(0,3600]`;应略小于 HiRoute `timeout_ms/1000` |
| `JEV_MAX_STATE_TOKENS` | `24000` | 保守 state 预算,见下节 |
@@ -73,6 +88,32 @@ HiRoute 的智能省钱计划配置:
出站使用一个长生命周期连接池,遵循标准 `HTTP_PROXY/HTTPS_PROXY/NO_PROXY` 及小写变量。没有代理配置时直连;容器须显式接收需要的代理环境变量。入站认证与 OpenRouter key 分开;在 HiRoute Secret 中保存完整 header 值,例如 `Bearer ...`,HiRoute 不补认证前缀。
+### 自定义简单与复杂的含义
+
+智能省钱默认使用随 Python 包发布的 [policy.default.json](jev_decider/policy.default.json),无需设置环境变量或复制文件。
+如需自定义,复制并编辑该文件,设置 `JEV_POLICY_FILE=/absolute/path/policy.json` 后启动服务。
+文件必须是 UTF-8 JSON,最多 4096 字节,恰好包含两个非空字符串:
+
+```json
+{
+ "simple": "需求明确、可沿用已有实现模式的常规功能、修复、测试和文档工作。不能只因涉及多个文件就判复杂。",
+ "complex": "需要定位未知根因、权衡不同设计方案、改变核心架构或处理复杂并发、状态一致性和算法交互的工作。"
+}
+```
+
+定义应描述适合当前省钱/主力模型组合的工作,可附少量通用例子;不是指定某个实验题必须走哪个模型。
+这只是判断口径,不代表经济模型真实成功率。先固定定义,再用独立任务校准阈值。
+
+配置属于服务实例,所有接入该实例的计划共用;目前不是 Desktop 计划编辑字段。
+对于智能省钱两个分支,`auto` 和 `rules` 都用内置或自定义定义同时替换上游 state 与 Choice criteria 的说明,
+避免判断口径冲突。显式自定义文件只适用于这两个分支,不扩大允许集合;未配置文件时,auto 对其他分支集合仍使用请求提供的说明。
+非法文件拒绝启动,不静默忽略或退回默认值。
+修改后需重启,无热加载、策略 DSL 或额外会话存储。有效定义计入上下文预算。
+`rules` 仍按原概率/胜任度公式执行;把更多工作定义为简单,不会取消低胜任度阻止降本的机制。
+
+启动日志记录智能省钱的有效定义哈希;每次决策日志记录实际 `criteria_source`(default/policy/request)和
+`criteria_sha256`,不记录定义正文。比对不同实验时应同时固定需求、可见历史、定义哈希、模型和阈值。
+
## 上下文边界
服务接收完整、非空的 `latest_user` 和保留的 `visible_conversation`。一次决策可能因为没有可继承分支、追加了用户消息,或 ContextHold 不再有候选保持而开始;服务不识别压缩。因此 `latest_user` 可以与上一执行轮次相同,也可以是客户端生成的摘要/继续消息,这种形状本身不是用户反馈。每个 visible 项是已封存的路由执行轮次;仅因进入下一决策边界而封存的项可以是 `unknown`,但仍可能包含可评分的实际推进。
diff --git a/decision-extensions/extensions/jev-decider/jev_decider/policy.default.json b/decision-extensions/extensions/jev-decider/jev_decider/policy.default.json
new file mode 100644
index 00000000..b9b30909
--- /dev/null
+++ b/decision-extensions/extensions/jev-decider/jev_decider/policy.default.json
@@ -0,0 +1,4 @@
+{
+ "simple": "Use the economy model group for work with explicit requirements and a bounded change that can follow existing repository patterns: routine feature increments, ordinary bug fixes, configuration updates, tests and documentation. Multiple files, a need for regression tests, or the word fix alone do not make work complex. Judge the work still required in this routing round, not the reputation or total size of the project.",
+ "complex": "Use the primary model group when the work requires discovering an unknown root cause, choosing between materially different designs, changing core architecture, or resolving intricate concurrency, state-consistency or algorithmic interactions. Judge the reasoning and uncertainty actually required, not keywords alone. A clear request can still require complex work; do not assume an undocumented solution or capability."
+}
diff --git a/decision-extensions/extensions/jev-decider/jev_decider/server.py b/decision-extensions/extensions/jev-decider/jev_decider/server.py
index f4589769..6589ae30 100644
--- a/decision-extensions/extensions/jev-decider/jev_decider/server.py
+++ b/decision-extensions/extensions/jev-decider/jev_decider/server.py
@@ -1,12 +1,16 @@
from __future__ import annotations
import asyncio
+import hashlib
import hmac
import json
+import logging
import math
import os
+import time
+import uuid
from pathlib import Path
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from typing import Any
from urllib.parse import urlsplit
@@ -19,6 +23,14 @@ DEFAULT_MODEL = "typesafe/jev-1.13"
DEFAULT_UPSTREAM = "https://openrouter.ai/api/alpha/decisions"
MAX_UPSTREAM_RESPONSE_BYTES = 64 * 1024
MAX_REQUEST_TIMEOUT_SECONDS = 3600.0
+LOGGER = logging.getLogger("jev_decider")
+
+
+def log_event(event: str, **fields: Any) -> None:
+ LOGGER.info(json.dumps({"timestamp_ms": time.time_ns() // 1_000_000,
+ "event": event, **fields}, ensure_ascii=False))
+
+
FORBIDDEN_INBOUND_HEADERS = {
"host",
"content-length",
@@ -165,6 +177,8 @@ class Settings:
competence_floor: float
inbound_header_name: str | None
inbound_header_value: str | None
+ branch_criteria: dict[str, str] = field(default_factory=lambda: load_policy(None))
+ policy_configured: bool = False
@classmethod
def from_env(cls) -> "Settings":
@@ -227,9 +241,40 @@ class Settings:
competence_floor=competence_floor,
inbound_header_name=header_name,
inbound_header_value=header_value,
+ branch_criteria=load_policy(os.getenv("JEV_POLICY_FILE")),
+ policy_configured="JEV_POLICY_FILE" in os.environ,
)
+def load_policy(filename: str | None) -> dict[str, str]:
+ path = Path(filename) if filename is not None else Path(__file__).with_name("policy.default.json")
+ if not path.is_absolute():
+ raise RuntimeError("JEV_POLICY_FILE must be an absolute path")
+ try:
+ with path.open("rb") as stream:
+ data = stream.read(4097)
+ except OSError as error:
+ raise RuntimeError("JEV_POLICY_FILE cannot be read") from error
+ if len(data) > 4096:
+ raise RuntimeError("JEV_POLICY_FILE must be at most 4096 bytes")
+ try:
+ data.decode("utf-8")
+ value = strict_json(data)
+ if not isinstance(value, dict) or set(value) != {"simple", "complex"}:
+ raise ProtocolError("invalid policy fields")
+ if any(not isinstance(text, str) or not text.strip() for text in value.values()):
+ raise ProtocolError("invalid policy descriptions")
+ # Validate escaped Unicode too, before a request attempts UTF-8 encoding.
+ _encoded(value)
+ except (ProtocolError, UnicodeError) as error:
+ raise RuntimeError("JEV_POLICY_FILE requires UTF-8 JSON with nonempty simple and complex strings") from error
+ return {SIMPLE_BRANCH: value["simple"], COMPLEX_BRANCH: value["complex"]}
+
+
+def criteria_hash(branches: dict[str, str]) -> str:
+ return hashlib.sha256(json.dumps(branches, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
+
+
def _finite_env(name: str, default: float) -> float:
try:
value = float(os.getenv(name, str(default)))
@@ -383,14 +428,17 @@ def decision_response(
request: dict[str, Any],
upstream: Any,
target_trimmed: bool,
+ trace: dict[str, Any] | None = None,
) -> dict[str, Any]:
+ trace = trace if trace is not None else {}
if not isinstance(upstream, dict) or not isinstance(upstream.get("answers"), dict):
raise ProtocolError("upstream response has no answers")
answers = upstream["answers"]
if settings.mode == "auto":
branch, _ = _choice(answers.get("branch"), request["branches"], probabilities=False)
+ trace.update(upstream_choice=branch, reason="auto_choice")
else:
- _, distribution = _choice(
+ selected, distribution = _choice(
answers.get("complexity"), request["branches"], probabilities=True
)
competence = _competence(answers.get("competence"))
@@ -400,6 +448,22 @@ def decision_response(
and (competence is None or competence >= settings.competence_floor)
else COMPLEX_BRANCH
)
+ opportunity = distribution[SIMPLE_BRANCH] >= settings.simple_threshold
+ guard = competence is None or competence >= settings.competence_floor
+ trace.update(
+ upstream_choice=selected, probabilities=distribution,
+ simple_threshold=settings.simple_threshold,
+ competence_floor=settings.competence_floor,
+ simplicity_passed=opportunity, competence_passed=guard,
+ reason=("complexity_threshold" if not opportunity else
+ "competence_guard" if not guard else "economy_eligible"),
+ )
+ trace.update(
+ competence=_competence(answers.get("competence")),
+ competence_status=("missing" if "competence" not in answers else
+ "invalid" if _competence(answers["competence"]) is None else "valid"),
+ branch_id=branch,
+ )
response: dict[str, Any] = {"branch_id": branch}
if request["assessment_from"] is not None:
score = _competence(answers.get("competence"))
@@ -425,16 +489,41 @@ async def _bounded_response(response: Any) -> bytes:
async def decide(request: web.Request) -> web.Response:
+ decision_id = uuid.uuid4().hex
+ started = time.monotonic()
+ trace: dict[str, Any] = {"decision_id": decision_id, "phase": "authentication"}
settings: Settings = request.app[SETTINGS]
+ log_event("decision_started", decision_id=decision_id, mode=settings.mode, model=settings.model)
+
+ def finish(body: dict[str, Any], status: int = 200) -> web.Response:
+ log_event("decision_completed", **trace, status=status,
+ error=body.get("error"), duration_ms=round((time.monotonic() - started) * 1000, 2))
+ return web.json_response(body, status=status, headers={"X-Jev-Decision-Id": decision_id})
+
if settings.inbound_header_name is not None:
supplied = request.headers.get(settings.inbound_header_name, "")
if not hmac.compare_digest(supplied, settings.inbound_header_value or ""):
- return web.json_response({"error": "unauthorized"}, status=401)
+ return finish({"error": "unauthorized"}, status=401)
upstream_started = False
try:
async with asyncio.timeout(settings.request_timeout_seconds):
+ trace["phase"] = "validation"
incoming = validate_hiroute_request(strict_json(await request.read()))
+ criteria_source = "request"
+ if set(incoming["branches"]) == {SIMPLE_BRANCH, COMPLEX_BRANCH}:
+ incoming = {**incoming, "branches": settings.branch_criteria}
+ criteria_source = "policy" if settings.policy_configured else "default"
+ elif settings.policy_configured:
+ raise ProtocolError("configured policy requires the two smart-saving branches")
+ trace.update(criteria_source=criteria_source,
+ criteria_sha256=criteria_hash(incoming["branches"]))
state, target_trimmed = prepare_state(incoming, settings.max_state_tokens)
+ trace.update(
+ input_turns=len(incoming["visible_conversation"]),
+ retained_turns=len(state["visible_conversation"]),
+ state_bytes=len(_encoded(state)), history_partial=state["history_partial"],
+ assessment_from=state["assessment_from"], target_trimmed=target_trimmed,
+ )
body = {
"model": settings.model,
"state": state,
@@ -446,7 +535,10 @@ async def decide(request: web.Request) -> web.Response:
}
session: ClientSession = request.app[CLIENT]
upstream_started = True
+ trace["phase"] = "queue"
+ queued = time.monotonic()
async with request.app[CONCURRENCY]:
+ trace.update(queue_ms=round((time.monotonic() - queued) * 1000, 2), phase="upstream")
async with session.post(
settings.upstream_url,
json=body,
@@ -457,28 +549,47 @@ async def decide(request: web.Request) -> web.Response:
},
allow_redirects=False,
) as upstream_response:
+ trace["upstream_status"] = upstream_response.status
if upstream_response.status != 200:
- return web.json_response(
+ return finish(
{"error": "upstream_rejected", "upstream_status": upstream_response.status},
status=502,
)
upstream = strict_json(await _bounded_response(upstream_response))
- return web.json_response(decision_response(settings, state, upstream, target_trimmed))
+ trace["phase"] = "decision"
+ result = decision_response(settings, state, upstream, target_trimmed, trace)
+ return finish(result)
except ProtocolError as error:
+ trace["validation_error"] = str(error)
if upstream_started:
- return web.json_response({"error": "upstream_invalid"}, status=502)
- return web.json_response(
+ return finish({"error": "upstream_invalid"}, status=502)
+ return finish(
{"error": str(error)},
status=413 if "context budget" in str(error) else 400,
)
except asyncio.TimeoutError:
- return web.json_response({"error": "upstream_timeout"}, status=504)
+ return finish({"error": "upstream_timeout"}, status=504)
except ClientError:
- return web.json_response({"error": "upstream_invalid"}, status=502)
+ return finish({"error": "upstream_invalid"}, status=502)
+ except asyncio.CancelledError:
+ log_event("decision_cancelled", **trace,
+ duration_ms=round((time.monotonic() - started) * 1000, 2))
+ raise
+ except Exception as error:
+ log_event("decision_failed", **trace, error_type=type(error).__name__,
+ duration_ms=round((time.monotonic() - started) * 1000, 2))
+ raise
async def _client(app: web.Application) -> None:
settings: Settings = app[SETTINGS]
+ log_event("service_started", mode=settings.mode, model=settings.model,
+ request_timeout_seconds=settings.request_timeout_seconds,
+ max_state_tokens=settings.max_state_tokens, max_concurrency=settings.max_concurrency,
+ simple_threshold=settings.simple_threshold if settings.mode == "rules" else None,
+ competence_floor=settings.competence_floor if settings.mode == "rules" else None,
+ criteria_source="policy" if settings.policy_configured else "default",
+ criteria_sha256=criteria_hash(settings.branch_criteria))
app[CLIENT] = ClientSession(
timeout=ClientTimeout(total=settings.request_timeout_seconds),
trust_env=True,
@@ -506,6 +617,7 @@ def create_app(settings: Settings | None = None) -> web.Application:
def main() -> None:
+ logging.basicConfig(level=logging.INFO, format="%(message)s")
web.run_app(
create_app(),
host=os.getenv("HOST", "127.0.0.1"),
diff --git a/decision-extensions/extensions/jev-decider/pyproject.toml b/decision-extensions/extensions/jev-decider/pyproject.toml
index ffb97fb2..5f912094 100644
--- a/decision-extensions/extensions/jev-decider/pyproject.toml
+++ b/decision-extensions/extensions/jev-decider/pyproject.toml
@@ -14,3 +14,6 @@ build-backend = "setuptools.build_meta"
[tool.setuptools.packages.find]
include = ["jev_decider*"]
+
+[tool.setuptools.package-data]
+jev_decider = ["policy.default.json"]
diff --git a/decision-extensions/extensions/jev-decider/tests/test_server.py b/decision-extensions/extensions/jev-decider/tests/test_server.py
index d8724692..cd6db44a 100644
--- a/decision-extensions/extensions/jev-decider/tests/test_server.py
+++ b/decision-extensions/extensions/jev-decider/tests/test_server.py
@@ -11,26 +11,34 @@ from unittest.mock import patch
from aiohttp import web
from aiohttp.test_utils import TestClient, TestServer
-from jev_decider.server import Settings, create_app
+from jev_decider.server import Settings, create_app, criteria_hash, load_policy
BRANCHES = {
"smart_saving_simple": "Use the economy model group for a clear, well-scoped task.",
"smart_saving_complex": "Use the primary model group for a complex task.",
}
+POLICY = {"smart_saving_simple": "Bounded changes following existing patterns.",
+ "smart_saving_complex": "Uncertain root causes and architectural tradeoffs."}
class SettingsTests(unittest.TestCase):
- def load(self, timeout: str) -> Settings:
+ def load(self, timeout: str, policy: bytes | None = None) -> Settings:
with tempfile.TemporaryDirectory() as directory:
key_file = os.path.join(directory, "key")
with open(key_file, "w", encoding="utf-8") as file:
file.write("test-key")
+ extra = {}
+ if policy is not None:
+ policy_file = Path(directory) / "policy.json"
+ policy_file.write_bytes(policy)
+ extra["JEV_POLICY_FILE"] = str(policy_file)
with patch.dict(
os.environ,
{
"OPENROUTER_API_KEY_FILE": key_file,
"JEV_REQUEST_TIMEOUT_SECONDS": timeout,
+ **extra,
},
clear=True,
):
@@ -43,6 +51,36 @@ class SettingsTests(unittest.TestCase):
with self.assertRaisesRegex(RuntimeError, r"\(0, 3600\]"):
self.load("3600.1")
+ def test_builtin_policy_is_default_and_can_be_overridden_from_env(self) -> None:
+ default = self.load("1")
+ self.assertFalse(default.policy_configured)
+ example = Path(__file__).resolve().parents[1] / "jev_decider/policy.default.json"
+ result = self.load("1", example.read_bytes())
+ self.assertTrue(result.policy_configured)
+ self.assertEqual(default.branch_criteria, result.branch_criteria)
+ self.assertEqual(set(result.branch_criteria), set(BRANCHES))
+ self.assertIn("existing repository patterns", result.branch_criteria["smart_saving_simple"])
+
+ def test_policy_rejects_invalid_files_without_fallback(self) -> None:
+ for value in (b"{}", b"[]", b"null", b"not json", b"\xff",
+ b'{"simple":"a","complex":"b","extra":"c"}',
+ b'{"simple":"a","simple":"b","complex":"c"}',
+ b'{"simple":" ","complex":"b"}',
+ b'{"simple":3,"complex":"b"}',
+ b'{"simple":"a","complex":null}',
+ b'{"simple":"\\ud800","complex":"b"}',
+ '{"simple":"a","complex":"b"}'.encode("utf-16"),
+ b" " * 4097):
+ with self.subTest(value=value[:50]), self.assertRaisesRegex(RuntimeError, "JEV_POLICY_FILE"):
+ self.load("1", value)
+ for name in ("", "relative.json", "/nonexistent/hiroute-test-policy.json"):
+ with self.subTest(name=name), self.assertRaisesRegex(RuntimeError, "JEV_POLICY_FILE"):
+ load_policy(name)
+
+ def test_policy_hash_is_key_order_independent(self) -> None:
+ self.assertEqual(criteria_hash(POLICY), criteria_hash(dict(reversed(list(POLICY.items())))))
+ self.assertNotEqual(criteria_hash(POLICY), criteria_hash(BRANCHES))
+
def input_body(history: list[dict] | None = None, assessment_from: int | None = None) -> dict:
return {
@@ -119,6 +157,8 @@ class DeciderHttpTests(unittest.IsolatedAsyncioTestCase):
inbound_header_value=None,
)
values.update(changes)
+ if "branch_criteria" in changes:
+ values["policy_configured"] = True
return Settings(**values)
async def client(self, settings: Settings) -> TestClient:
@@ -149,6 +189,140 @@ class DeciderHttpTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(await response.json(), {"branch_id": "smart_saving_simple"})
self.assertEqual(set(self.calls[0]["questions"]), {"branch"})
+ async def test_custom_policy_replaces_state_and_choice_for_both_modes(self) -> None:
+ for mode, question in (("auto", "branch"), ("rules", "complexity")):
+ with self.subTest(mode=mode):
+ self.answer = {"answers": {
+ question: {"type": "choice", "choice": "smart_saving_simple",
+ "probabilities": {"smart_saving_simple": 0.9, "smart_saving_complex": 0.1}},
+ }}
+ client = await self.client(self.settings(mode, branch_criteria=POLICY))
+ body = input_body()
+ with self.assertLogs("jev_decider", level="INFO") as captured:
+ response = await client.post("/v1/decisions", json=body)
+ self.assertEqual(response.status, 200)
+ self.assertEqual(await response.json(), {"branch_id": "smart_saving_simple"})
+ self.assertEqual(self.calls[-1]["state"]["branches"], POLICY)
+ self.assertEqual(self.calls[-1]["questions"][question]["criteria"], POLICY)
+ self.assertEqual(body["branches"], BRANCHES)
+ event = json.loads(captured.records[-1].getMessage())
+ self.assertEqual(event["criteria_source"], "policy")
+ self.assertEqual(event["criteria_sha256"], criteria_hash(POLICY))
+ self.assertNotIn(POLICY["smart_saving_simple"], " ".join(captured.output))
+
+ async def test_custom_policy_does_not_override_low_competence(self) -> None:
+ self.answer = {"answers": {
+ "complexity": {"type": "choice", "choice": "smart_saving_simple",
+ "probabilities": {"smart_saving_simple": 0.99, "smart_saving_complex": 0.01}},
+ "competence": {"type": "score", "score": 0.2},
+ }}
+ client = await self.client(self.settings("rules", branch_criteria=POLICY))
+ response = await client.post("/v1/decisions", json=input_body([turn("previous")], 0))
+ self.assertEqual(response.status, 200)
+ self.assertEqual((await response.json())["branch_id"], "smart_saving_complex")
+ from jev_decider.server import COMPETENCE_CRITERIA
+ self.assertEqual(self.calls[0]["questions"]["competence"]["criteria"], COMPETENCE_CRITERIA)
+
+ async def test_custom_policy_rejects_other_branches_before_upstream(self) -> None:
+ client = await self.client(self.settings(branch_criteria=POLICY))
+ body = input_body()
+ body["branches"] = {**BRANCHES, "third": "another option"}
+ response = await client.post("/v1/decisions", json=body)
+ self.assertEqual(response.status, 400)
+ self.assertEqual(len(self.calls), 0)
+
+ async def test_custom_policy_counts_toward_context_budget(self) -> None:
+ # The same input fits without an override. Added definitions, not the
+ # original request, must be what causes the bounded-context rejection.
+ baseline = await self.client(self.settings(max_state_tokens=2000))
+ response = await baseline.post("/v1/decisions", json=input_body())
+ self.assertEqual(response.status, 200)
+ self.calls.clear()
+ client = await self.client(self.settings("rules", max_state_tokens=2000,
+ branch_criteria={**POLICY, "smart_saving_simple": "x" * 2100}))
+ response = await client.post("/v1/decisions", json=input_body())
+ self.assertEqual(response.status, 413)
+ self.assertEqual(len(self.calls), 0)
+
+ async def test_without_policy_uses_builtin_definitions_and_logs_hash(self) -> None:
+ client = await self.client(self.settings())
+ with self.assertLogs("jev_decider", level="INFO") as captured:
+ response = await client.post("/v1/decisions", json=input_body())
+ self.assertEqual(response.status, 200)
+ expected = load_policy(None)
+ self.assertNotEqual(expected, BRANCHES)
+ self.assertEqual(self.calls[0]["state"]["branches"], expected)
+ self.assertEqual(self.calls[0]["questions"]["branch"]["criteria"], expected)
+ event = json.loads(captured.records[-1].getMessage())
+ self.assertEqual(event["criteria_source"], "default")
+ self.assertEqual(event["criteria_sha256"], criteria_hash(expected))
+
+ async def test_generic_auto_branches_keep_request_definitions_without_override(self) -> None:
+ client = await self.client(self.settings())
+ body = input_body()
+ body["branches"] = {**BRANCHES, "third": "Another allowed branch"}
+ response = await client.post("/v1/decisions", json=body)
+ self.assertEqual(response.status, 200)
+ self.assertEqual(self.calls[0]["state"]["branches"], body["branches"])
+ self.assertEqual(self.calls[0]["questions"]["branch"]["criteria"], body["branches"])
+
+ async def test_logs_explain_rule_override_and_correlate_response(self) -> None:
+ self.answer = {"answers": {
+ "complexity": {"type": "choice", "choice": "smart_saving_complex",
+ "probabilities": {"smart_saving_simple": 0.1, "smart_saving_complex": 0.9}},
+ }}
+ client = await self.client(self.settings("rules", simple_threshold=0))
+ with self.assertLogs("jev_decider", level="INFO") as captured:
+ response = await client.post("/v1/decisions", json=input_body())
+ self.assertEqual((await response.json())["branch_id"], "smart_saving_simple")
+ events = [json.loads(record.getMessage()) for record in captured.records]
+ self.assertEqual([event["event"] for event in events], ["decision_started", "decision_completed"])
+ result = events[-1]
+ self.assertEqual(result["decision_id"], response.headers["X-Jev-Decision-Id"])
+ self.assertEqual(events[0]["decision_id"], result["decision_id"])
+ self.assertEqual(result["upstream_choice"], "smart_saving_complex")
+ self.assertEqual(result["branch_id"], "smart_saving_simple")
+ self.assertEqual(result["simple_threshold"], 0)
+ self.assertEqual(result["reason"], "economy_eligible")
+ self.assertEqual(result["competence_status"], "missing")
+ self.assertGreaterEqual(result["duration_ms"], result["queue_ms"])
+ self.assertNotIn("Fix the typo", " ".join(captured.output))
+ self.assertNotIn("secret", " ".join(captured.output))
+
+ async def test_logs_guard_trim_and_failures(self) -> None:
+ self.answer = {"answers": {
+ "complexity": {"type": "choice", "choice": "smart_saving_simple",
+ "probabilities": {"smart_saving_simple": 0.95, "smart_saving_complex": 0.05}},
+ "competence": {"type": "score", "score": 0.2},
+ }}
+ client = await self.client(self.settings("rules", max_state_tokens=900, branch_criteria=BRANCHES))
+ with self.assertLogs("jev_decider", level="INFO") as captured:
+ response = await client.post("/v1/decisions", json=input_body([turn("x" * 400), turn("recent")], 0))
+ self.assertEqual(response.status, 200)
+ result = json.loads(captured.records[-1].getMessage())
+ self.assertEqual(result["reason"], "competence_guard")
+ self.assertEqual(result["competence"], 0.1)
+ self.assertTrue(result["target_trimmed"])
+ self.assertLess(result["retained_turns"], result["input_turns"])
+ self.upstream_status = 429
+ with self.assertLogs("jev_decider", level="INFO") as captured:
+ response = await client.post("/v1/decisions", json=input_body())
+ result = json.loads(captured.records[-1].getMessage())
+ self.assertEqual(response.status, 502)
+ self.assertEqual(result["upstream_status"], 429)
+ self.assertEqual(result["error"], "upstream_rejected")
+ self.assertEqual(result["phase"], "upstream")
+
+ async def test_timeout_logs_terminal_event(self) -> None:
+ self.upstream_delay = 0.1
+ client = await self.client(self.settings(request_timeout_seconds=0.02))
+ with self.assertLogs("jev_decider", level="INFO") as captured:
+ response = await client.post("/v1/decisions", json=input_body())
+ result = json.loads(captured.records[-1].getMessage())
+ self.assertEqual(response.status, 504)
+ self.assertEqual(result["error"], "upstream_timeout")
+ self.assertEqual(result["event"], "decision_completed")
+
async def test_openapi_route_is_served_without_legacy_aliases(self) -> None:
document = json.loads(
(Path(__file__).resolve().parents[3] / "api/decision.openapi.json").read_text()
@@ -252,7 +426,7 @@ class DeciderHttpTests(unittest.IsolatedAsyncioTestCase):
async def test_trimming_removes_whole_old_turns_and_marks_target_partial(self) -> None:
history = [turn("old-" + "x" * 400), turn("recent")]
- client = await self.client(self.settings(max_state_tokens=900))
+ client = await self.client(self.settings(max_state_tokens=900, branch_criteria=BRANCHES))
response = await client.post("/v1/decisions", json=input_body(history, 0))
body = await response.json()
self.assertEqual(response.status, 200, body)
@@ -339,7 +513,7 @@ class DeciderHttpTests(unittest.IsolatedAsyncioTestCase):
async def test_trimmed_non_target_history_marks_state_partial_not_assessment(self) -> None:
history = [turn("old-" + "x" * 500), turn("target")]
- client = await self.client(self.settings(max_state_tokens=1000))
+ client = await self.client(self.settings(max_state_tokens=1000, branch_criteria=BRANCHES))
response = await client.post("/v1/decisions", json=input_body(history, 1))
body = await response.json()
self.assertEqual(response.status, 200, body)