Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Tool-result context signals extracted from the conversation history.
//!
//! The extractor walks normalized messages, finds tool calls and results,
//! reads explicit failure flags, matches text against an error table, and aggregates
//! conversation-history metrics used by [`crate::StageRouter`] and the
//! advisor gate's request-side guards.
//!
//! All logic is pure and deterministic — no I/O, no shared state.
#![allow(dead_code)]
use std::collections::HashSet;
use std::path::Path;
use std::sync::LazyLock;
use async_trait::async_trait;
use regex::Regex;
use serde::Deserialize;
use serde_json::Value;
use switchyard_protocol::codex_namespaces::{split_qualified_name, tool_namespaces};
use switchyard_protocol::{ContentBlock, Request, Role, WireFormat};
use super::affinity::is_user_turn;
use crate::{LibsyError, Result};
use crate::core::processor::{Event, Processor};
use crate::core::state::State;
// ─── severity constants ───────────────────────────────────────────────────────
const SOFT: f32 = 0.3;
const HARD: f32 = 0.7;
const CRITICAL: f32 = 1.0;
// ─── pattern table ────────────────────────────────────────────────────────────
/// (name, severity, lower-cased substrings — any hit fires the pattern)
static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[
(
"oom",
CRITICAL,
&["out of memory", "memoryerror", "cannot allocate memory"],
),
(
"connection_refused",
HARD,
&[
"connection refused",
"connectionrefusederror",
"econnrefused",
],
),
("traceback", HARD, &["traceback (most recent call last)"]),
(
"import_error",
HARD,
&["modulenotfounderror:", "importerror:", "no module named "],
),
(
"cmd_not_found",
HARD,
&["command not found", "/usr/bin/env: "],
),
("assertion", HARD, &["assertionerror"]),
("value_error", HARD, &["valueerror:"]),
("syntax_error", HARD, &["syntaxerror:"]),
(
"timeout",
HARD,
&[
"timed out",
"timeouterror",
"timeout expired",
"deadline exceeded",
],
),
(
"no_such_file",
HARD,
&[
"filenotfounderror:",
"no such file or directory",
// Claude Code's missing-file message. Reads are exempt (`is_missing_file`).
// Anchored as "file does not exist" (not a bare "does not exist", which
// fires on `ls` output and prose) — trace-mined across 1006 local
// trajectories at 22 true / 2 false positives.
"file does not exist",
],
),
// SOFT: plain non-zero exit without a recognisable exception traceback.
("exit_nonzero", SOFT, &["returned non-zero"]),
];
static NONZERO_EXIT_PHRASES: &[&str] = &[
"exit code",
"exit status",
"exited with code",
"exited with status",
];
static EDIT_TOOL_NAMES: &[&str] = &[
"edit",
"multiedit",
"notebookedit",
"str_replace",
"str_replace_based_edit_tool",
"apply_patch", // codex's edit tool
"text_editor",
"patch", // hermes's str_replace-style edit tool
];
/// Editor tools whose `command` argument picks the action. `view` only reads.
static EDITOR_TOOL_NAMES: &[&str] = &["str_replace_based_edit_tool", "text_editor"];
static WRITE_TOOL_NAMES: &[&str] = &["write", "create_file", "new_file", "write_file"];
// Bash subcommand patterns. Lowercased; callers must lowercase the command
// before matching. Bucketed into write_count / edit_count alongside the
// dedicated `Write` / `Edit` tools.
static BASH_WRITE_PATTERNS: &[&str] = &[
"cat >",
"cat >>",
"echo >",
"echo >>",
"tee ",
"printf >",
"printf >>",
"> /",
">> /",
"<< 'eof'",
"<<eof",
"<<'eof'",
"<< eof",
];
/// Python file-write expressions, which only indicate a write when an
/// interpreter is running them rather than a search looking for them.
static PYTHON_WRITE_PATTERNS: &[&str] = &["write_text(", "writelines(", ".write("];
static JAVASCRIPT_WRITE_PATTERNS: &[&str] = &[
"writefilesync(",
"writefile(",
"appendfilesync(",
"appendfile(",
];
static BASH_EDIT_PATTERNS: &[&str] = &[
"sed -i",
"sed --in-place",
"awk -i inplace",
"awk 'inplace=1'",
"patch ",
"patch -p",
"perl -i",
"perl -p -i",
"perl -pi",
];
// Read-like Bash inspections. Match only when none of the write/edit patterns
// fire (redirection / in-place edit trumps the read intent of the command).
static BASH_READ_PATTERNS: &[&str] = &[
"cat /", "cat ./", "cat ../", "grep ", "ls ", "ls -", "find ", "head ", "tail ", "wc ",
"diff ", "which ", "ps ", "df ", "du ", "stat ", "file ", "less ", "more ",
];
/// Read-only shell programs seen in Codex trajectories. Matching is limited to
/// command-segment starts so prose and arguments do not masquerade as actions.
static BASH_READ_COMMANDS: &[&str] = &[
"cat", "rg", "nl", "jq", "pwd", "tree", "sed", "grep", "ls", "find", "head", "tail", "wc",
"diff", "which", "ps", "df", "du", "stat", "file", "less", "more", "readlink", "realpath",
"basename", "dirname", "printenv",
];
static GIT_READ_SUBCOMMANDS: &[&str] = &[
"status",
"diff",
"log",
"show",
"show-ref",
"rev-parse",
"ls-files",
"ls-remote",
"ls-tree",
"grep",
"blame",
"merge-base",
"check-ignore",
"tag",
];
static READ_TOOL_NAMES: &[&str] = &[
"read",
"view",
"read_file",
"search_files",
"glob",
"grep",
"find",
"ls",
];
// Planning / scratchpad tool calls — investigative (non-producing) activity.
// `update_plan` is codex's equivalent of `todowrite`.
static PLAN_TOOL_NAMES: &[&str] = &[
"todowrite",
"todo_write",
"todo",
"update_plan",
"todo_list",
];
// Tool names that route through Bash-command pattern matching. `bash` is
// claude-code's name; `shell_command` is codex's; `shell` / `local_shell_call`
// are seen on some OpenAI-derived harnesses; `terminal` is hermes's (it carries
// a `command` arg like the others, so its intent comes from the pattern match).
static BASH_TOOL_NAMES: &[&str] = &[
"bash",
"shell_command",
"shell",
"local_shell_call",
"terminal",
"exec_command", // codex
"exec", // openclaw
"powershell", // pi on Windows
];
// Prefer false negatives: tests_passed clears a capable hold, so a false positive
// could hand an unfinished task back too early.
static TEST_PASS_PHRASES: &[&str] = &[
" passed",
"passed in",
"tests passed",
"all tests passed",
"test ok",
"test result: ok",
"passed.\n",
"tests pass",
"\nok ", // go test; newline-anchored to avoid "...lookup..." mid-text
"✓ ",
];
// Literal failure phrases that cannot appear inside a clean run. Substring
// matched as-is. Patterns that pair with a count (e.g. "failed", "errors")
// are handled separately by `has_nonzero_failure_count` so "0 failed" /
// "0 errors" do not trigger a false negative.
static TEST_FAILURE_LITERAL: &[&str] = &["✗ ", "fatal:", "assertionerror", "error:"];
// Count-prefixed failure keywords. Trip only when a nonzero integer precedes
// the keyword (modulo whitespace), so cargo's "0 failed" and go's
// "0 errors" summaries on a clean run are not misread as failures.
static NUMERIC_FAILURE_KEYWORDS: &[&str] = &["failed", "failure", "failures", "errors", "error"];
/// Default sliding-window size for `recent_*` counts and windowed severity.
///
/// A short horizon captures "what is the agent doing right now" while keeping
/// signals sticky — an error or stall persists a few recovery turns instead of
/// flickering off the moment one clean result lands. Override per request by
/// passing a window to [`ToolSignals::from_request`].
pub const DEFAULT_RECENT_WINDOW: usize = 3;
/// Exact tool-name semantics added to the built-in vocabulary.
///
/// Matching is ASCII case-insensitive. An MCP or Codex namespaced tool also
/// matches by its bare tool name. These lists are additive: built-in tool names
/// cannot be reclassified.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
#[serde(default, deny_unknown_fields)]
pub struct ToolSemantics {
/// Read-only lookup or inspection tools.
pub observe: Vec<String>,
/// Tools that change task or external state.
pub mutate: Vec<String>,
/// Explicit planning or task-decomposition tools.
pub plan: Vec<String>,
/// Tools that demonstrate new forward activity without favoring either tier.
pub new: Vec<String>,
}
impl ToolSemantics {
/// Rejects ambiguous mappings and attempts to reclassify built-in tools.
pub fn validate(&self) -> Result<()> {
let mut seen: Vec<(String, &'static str)> = Vec::new();
for (category, names) in [
("observe", &self.observe),
("mutate", &self.mutate),
("plan", &self.plan),
("new", &self.new),
] {
for name in names {
if name.trim().is_empty() {
return Err(tool_semantics_error(format!(
"tool_semantics.{category} contains an empty tool name"
)));
}
let normalized = name.to_ascii_lowercase();
if is_builtin_tool_name(&name.to_lowercase()) {
return Err(tool_semantics_error(format!(
"tool {name:?} already has built-in semantics and cannot be reclassified"
)));
}
if let Some((_, previous)) = seen.iter().find(|(seen, _)| seen == &normalized) {
return Err(tool_semantics_error(format!(
"tool {name:?} appears in both tool_semantics.{previous} and tool_semantics.{category}"
)));
}
seen.push((normalized, category));
}
}
Ok(())
}
fn classify(&self, name: &str) -> Option<ToolSemantic> {
if contains_name(&self.observe, name) {
Some(ToolSemantic::Observe)
} else if contains_name(&self.mutate, name) {
// The stage scorer treats writes and edits identically. Custom
// mutations use the write counter to preserve the public signal shape.
Some(ToolSemantic::Mutate(MutationKind::Write))
} else if contains_name(&self.plan, name) {
Some(ToolSemantic::Plan)
} else if contains_name(&self.new, name) {
Some(ToolSemantic::New)
} else {
None
}
}
}
fn contains_name(names: &[String], candidate: &str) -> bool {
names
.iter()
.any(|name| name.eq_ignore_ascii_case(candidate))
}
fn tool_semantics_error(message: String) -> LibsyError {
LibsyError::AlgorithmError { message }
}
// ─── output type ─────────────────────────────────────────────────────────────
/// Tool-execution signals extracted from a normalized [`Request`].
///
/// A request-side processor stores these signals in [`State`](crate::State) for
/// [`crate::StageRouter`] and its classifier to consume. The advisor gate's
/// request-side guards read the conversation-shape counts directly via
/// [`ToolSignals::from_request`].
#[derive(Clone, Debug, Default)]
pub struct ToolSignals {
/// Max severity across the recent window (last `recent_window` tool results):
/// `0.0` clean · `0.3` soft (exit_nonzero) · `0.7` hard · `1.0` critical.
/// Windowed so an error persists through the recovery turns instead of clearing
/// the instant the next result is clean.
pub severity: f32,
/// The same hard-or-critical failure appeared at least twice in the recent
/// tool-result window.
pub repeated_failure: bool,
/// Consecutive clean tool results back from the most recent. `0` if the last failed.
pub no_error_streak: u32,
/// Total edit-style tool calls in the request.
pub edit_count: u32,
/// Total write-style tool calls in the request.
pub write_count: u32,
/// Read-type calls (Read tool + read-like Bash). Used by the build-pit gate.
pub read_count: u32,
/// TodoWrite / planning tool calls. Investigative (non-producing) activity —
/// recent todowrites distinguish `exploring` from `spinning` in the scorer.
pub todowrite_count: u32,
/// Edit-type calls within the configured recent window (default: [`DEFAULT_RECENT_WINDOW`]).
pub recent_edit_count: u32,
/// Write-type calls within the configured recent window (default: [`DEFAULT_RECENT_WINDOW`]).
pub recent_write_count: u32,
/// Read-type calls within the configured recent window (default: [`DEFAULT_RECENT_WINDOW`]).
pub recent_read_count: u32,
/// TodoWrite calls within the configured recent window (default: [`DEFAULT_RECENT_WINDOW`]).
pub recent_todowrite_count: u32,
/// Configured `new` tool calls across the full request history.
pub new_count: u32,
/// Configured `new` tool calls within the recent window.
pub recent_new_count: u32,
/// Consecutive trailing tool calls in the `Unknown` category (no Write/Edit/Read/
/// Plan match). Surfaced in the classifier state summary; not scored directly.
pub pure_bash_streak: u32,
/// A tool result after the latest recent failure matched a test-pass pattern.
pub tests_passed: bool,
/// Total `ToolResult` blocks, counted per block (a message batching N
/// results contributes N) and including empty-content results.
pub tool_result_count: u32,
/// Messages with `Role::Assistant`, unlike [`ToolSignals::turn_depth`],
/// which counts every message regardless of role.
pub assistant_turn_count: u32,
/// Message-count proxy for turn depth. Wire-format dependent (Anthropic batches
/// tool results into fewer messages than OpenAI-chat), so gates keyed on it are
/// approximate across request origins.
pub turn_depth: u32,
/// The request carries a context-compaction summary (the agent's context was
/// summarised after overflowing). Compaction resets the router's accumulated
/// signals, so a task that was on the strong tier de-escalates back to weak — the
/// picker uses this to force + hold the strong tier. Self-latching: the summary
/// stays in the context prefix on every subsequent turn.
pub compacted: bool,
}
impl ToolSignals {
/// Extracts tool and activity signals from `request`.
///
/// `window_size` limits recent counters to the newest tool results. `None`
/// uses [`DEFAULT_RECENT_WINDOW`].
pub fn from_request(request: &Request, window_size: Option<usize>) -> Self {
Self::from_request_with_semantics(request, window_size, &ToolSemantics::default())
}
/// Extracts signals using the built-in vocabulary plus additive semantics.
pub fn from_request_with_semantics(
request: &Request,
window_size: Option<usize>,
semantics: &ToolSemantics,
) -> Self {
extract_tool_signals_with_window_and_semantics(
request,
window_size.unwrap_or(DEFAULT_RECENT_WINDOW),
semantics,
false,
)
}
}
// `command` is the lowercased Bash command line; None for non-Bash tools.
// `bare_name` is the tool's own name when `name` joins it to a namespace or MCP server.
// `is_retrieval` marks a call that only reads; it counts as an observation.
#[derive(Debug, Clone)]
struct ObservedToolCall<'a> {
name: String,
bare_name: Option<&'a str>,
command: Option<String>,
is_retrieval: bool,
code_semantic: Option<ToolSemantic>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum MutationKind {
Write,
Edit,
}
/// Domain-neutral meaning assigned to an observed tool call.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ToolSemantic {
Mutate(MutationKind),
Observe,
Plan,
New,
Unknown,
}
/// Request-side processor that extracts tool-result signals from each request
/// and stores them on the request `State` for downstream routing.
#[derive(Debug, Clone)]
pub struct ToolSignalProcessor {
/// Number of trailing tool results the `recent_*` counts and windowed
/// severity are computed over.
pub recent_window: usize,
/// Route-scoped additions to the built-in tool vocabulary.
pub tool_semantics: ToolSemantics,
}
impl Default for ToolSignalProcessor {
fn default() -> Self {
Self {
recent_window: DEFAULT_RECENT_WINDOW,
tool_semantics: ToolSemantics::default(),
}
}
}
impl ToolSignalProcessor {
pub(crate) fn for_user_turn(&self, request: &Request) -> ToolSignals {
extract_tool_signals_with_window_and_semantics(
request,
self.recent_window,
&self.tool_semantics,
true,
)
}
}
#[async_trait]
impl Processor for ToolSignalProcessor {
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Request { request: req, .. } = event {
let tool_signal = ToolSignals::from_request_with_semantics(
req,
Some(self.recent_window),
&self.tool_semantics,
);
state.tool_signals = Some(tool_signal);
}
Ok(())
}
}
fn classify_tool_call(name: &str, command: Option<&str>) -> ToolSemantic {
classify_tool_call_with_semantics(name, command, &ToolSemantics::default())
}
/// Recognize straight-line Codex patch calls, with optional literal bindings and
/// `text(...)` wrappers. Other JavaScript stays unknown, including control flow.
fn code_mode_has_patch(mut input: &str) -> bool {
static STATEMENT: LazyLock<Regex> = LazyLock::new(|| {
let identifier = r"[a-zA-Z_$][a-zA-Z0-9_$]*";
let literal = r#"(?:"(?:[^"\\\r\n]|\\[^\r\n])*"|'(?:[^'\\\r\n]|\\[^\r\n])*'|`(?:[^`\\$]|\\[^\r\n])*`)"#;
let argument = format!(r"(?:{literal}|{identifier})");
let patch = format!(r"await\s+tools\s*\.\s*apply_patch\s*\(\s*{argument}\s*\)");
// The pattern uses only constant fragments.
Regex::new(&format!(
r"(?x)\A(?://[^\r\n]*|(?s:/\*.*?\*/)|(?:
(?:const|let)\s+(?P<binding>{identifier})\s*=\s*{literal}|
(?P<patch>{patch}|text\s*\(\s*{patch}\s*\))|
text\s*\(\s*{argument}\s*\)
)[\t\x20]*(?:;|\r?\n|\z))"
))
.expect("constant code-mode statement pattern")
});
let mut has_patch = false;
while !input.trim_start().is_empty() {
input = input.trim_start();
let Some(statement) = STATEMENT.captures(input) else {
return false;
};
if statement
.name("binding")
.is_some_and(|binding| matches!(binding.as_str(), "tools" | "text"))
{
return false;
}
has_patch |= statement.name("patch").is_some();
input = &input[statement[0].len()..];
}
has_patch
}
fn classify_tool_call_with_semantics(
name: &str,
command: Option<&str>,
semantics: &ToolSemantics,
) -> ToolSemantic {
// Built-in names and Bash command inference take precedence over route-scoped mappings.
let lower = name.to_lowercase();
if WRITE_TOOL_NAMES.contains(&lower.as_str()) {
return ToolSemantic::Mutate(MutationKind::Write);
}
if EDITOR_TOOL_NAMES.contains(&lower.as_str()) && command == Some("view") {
return ToolSemantic::Observe;
}
if EDIT_TOOL_NAMES.contains(&lower.as_str()) {
return ToolSemantic::Mutate(MutationKind::Edit);
}
if READ_TOOL_NAMES.contains(&lower.as_str()) {
return ToolSemantic::Observe;
}
if PLAN_TOOL_NAMES.contains(&lower.as_str()) {
return ToolSemantic::Plan;
}
if BASH_TOOL_NAMES.contains(&lower.as_str())
&& let Some(cmd) = command
{
// Write/edit redirection trumps read-like operands.
if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_write(cmd) {
return ToolSemantic::Mutate(MutationKind::Write);
}
if cmd.contains("python") && PYTHON_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) {
return ToolSemantic::Mutate(MutationKind::Write);
}
if shell_invokes_program(cmd, "node")
&& JAVASCRIPT_WRITE_PATTERNS
.iter()
.any(|pattern| cmd.contains(pattern))
{
return ToolSemantic::Mutate(MutationKind::Write);
}
if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_edit(cmd) {
return ToolSemantic::Mutate(MutationKind::Edit);
}
if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_read(cmd) {
return ToolSemantic::Observe;
}
}
semantics.classify(name).unwrap_or(ToolSemantic::Unknown)
}
/// Tools whose successful output is retrieved content or inspection data.
fn is_retrieval_tool(name: &str, command: Option<&Value>) -> bool {
let lower = name.to_lowercase();
READ_TOOL_NAMES.contains(&lower.as_str())
|| (EDITOR_TOOL_NAMES.contains(&lower.as_str())
&& command
.and_then(Value::as_str)
.is_some_and(|command| command.eq_ignore_ascii_case("view")))
// PowerShell quoting and escapes do not follow the POSIX rules below.
|| (lower != "powershell"
&& BASH_TOOL_NAMES.contains(&lower.as_str())
&& command.is_some_and(command_is_retrieval))
}
/// A shell command field given as one line or as argv words.
fn command_is_retrieval(command: &Value) -> bool {
match command {
Value::String(line) => shell_is_retrieval(line, 0),
Value::Array(argv) => {
// Charge each word a separator byte, as in the one-line form, so the
// budget also bounds the argument count.
argv.iter()
.try_fold(0, |size, word| {
let size = size + word.as_str()?.len() + 1;
(size <= MAX_RETRIEVAL_COMMAND_BYTES).then_some(size)
})
.is_some()
&& retrieval_command(
&argv.iter().filter_map(Value::as_str).collect::<Vec<_>>(),
0,
)
}
_ => false,
}
}
// Keep shell parsing bounded on the request thread. Larger commands retain
// ordinary error scanning, just like unrecognized shell syntax.
const MAX_RETRIEVAL_COMMAND_BYTES: usize = 16 * 1024;
/// Only suppress output when every shell segment is a recognized inspection.
/// Observation counters can count mixed commands; suppressing errors needs all
/// commands to qualify. Substitutions and redirects keep their error signals.
fn shell_is_retrieval(command: &str, depth: usize) -> bool {
if command.len() > MAX_RETRIEVAL_COMMAND_BYTES {
return false;
}
if quoted_chars(command).any(|(_, c, quote)| {
(quote != Some('\'') && matches!(c, '$' | '`'))
|| (quote.is_none() && matches!(c, '<' | '>' | '(' | ')' | '{' | '}' | '#'))
}) {
return false;
}
let mut segments = shell_segments(command).peekable();
segments.peek().is_some()
&& segments.all(|segment| {
// `split` also rejects an unclosed quote or a trailing backslash.
shlex::split(segment).is_some_and(|words| {
retrieval_command(&words.iter().map(String::as_str).collect::<Vec<_>>(), depth)
})
})
}
fn skip_shell_options<'a>(mut args: &'a [&'a str], takes_value: &[&str]) -> &'a [&'a str] {
while let Some((option, rest)) = args.split_first() {
if !option.starts_with('-') || *option == "-" {
break;
}
args = rest;
if *option == "--" {
break;
}
if takes_value.contains(option) {
args = args.get(1..).unwrap_or_default();
}
}
args
}
/// A short-option word such as `-Hx` that sets any of `flags`.
fn has_short_flag(word: &str, flags: &[char]) -> bool {
word.starts_with('-') && !word.starts_with("--") && word.contains(flags)
}
// Bounds recursion through nested wrappers and `sh -c` scripts.
const MAX_COMMAND_DEPTH: usize = 20;
fn retrieval_command(mut words: &[&str], depth: usize) -> bool {
if depth > MAX_COMMAND_DEPTH {
return false;
}
while words.first().is_some_and(|word| {
word.split_once('=').is_some_and(|(name, _)| {
!name.is_empty()
&& name.bytes().enumerate().all(|(i, c)| {
c == b'_' || c.is_ascii_alphabetic() || (i > 0 && c.is_ascii_digit())
})
})
}) {
words = &words[1..];
}
let Some((&program, args)) = words.split_first() else {
return false;
};
// The standard bin dirs hold the real programs. Any other path, such as
// `./test`, names a local script.
let program = [
"/bin/",
"/sbin/",
"/usr/bin/",
"/usr/sbin/",
"/usr/local/bin/",
"/opt/homebrew/bin/",
]
.iter()
.find_map(|dir| program.strip_prefix(dir))
.unwrap_or(program);
// Unwrap launchers before inspecting the actual command and its flags.
let wrapper_options: Option<&[&str]> = match program {
"sudo" => Some(&[
"-u", "-g", "-h", "-p", "-C", "-T", "--user", "--group", "--host",
]),
"env" => Some(&["-u", "--unset", "-C", "--chdir"]),
"command" | "nohup" => Some(&[]),
"exec" => Some(&["-a"]),
"time" => Some(&["-f", "--format", "-o", "--output"]),
"timeout" => Some(&["-s", "--signal", "-k", "--kill-after"]),
"xargs" => Some(&[
"-n",
"--max-args",
"-P",
"--max-procs",
"-I",
"--replace",
"-d",
"--delimiter",
"-L",
"--max-lines",
"-s",
"--max-chars",
]),
_ => None,
};
if let Some(options) = wrapper_options {
// `env -S` splits its argument into a new command line.
if program == "env"
&& args
.iter()
.any(|arg| arg.starts_with("--split-string") || has_short_flag(arg, &['S']))
{
return false;
}
let mut rest = skip_shell_options(args, options);
if program == "timeout" {
rest = rest.get(1..).unwrap_or_default();
}
return (program == "env" && rest.is_empty()) || retrieval_command(rest, depth + 1);
}
if matches!(program, "bash" | "sh" | "dash" | "zsh" | "ksh") {
return args
.first()
.is_some_and(|option| has_short_flag(option, &['c']))
&& args
.get(1)
.is_some_and(|command| shell_is_retrieval(command, depth + 1));
}
if program == "git" {
return git_is_retrieval(args);
}
if matches!(args, ["--help" | "--version"]) {
return true;
}
match program {
"find" => !args.iter().any(|arg| {
matches!(
*arg,
"-delete"
| "-exec"
| "-execdir"
| "-ok"
| "-okdir"
| "-fprint"
| "-fprint0"
| "-fprintf"
)
}),
// fd joins short flags, so `-Hx` runs a command too.
"fd" => !args
.iter()
.any(|arg| arg.starts_with("--exec") || has_short_flag(arg, &['x', 'X'])),
// Only the common line-range printing form; sed scripts can execute commands.
"sed" => {
args.len() >= 2
&& args[0] == "-n"
&& args[2..].iter().all(|arg| !arg.starts_with('-'))
&& args[1].strip_suffix('p').is_some_and(|range| {
!range.is_empty()
&& range
.bytes()
.all(|c| c.is_ascii_digit() || matches!(c, b',' | b'$'))
})
}
"sort" => !args.iter().any(|arg| {
arg.starts_with("--compress-program")
|| arg.starts_with("--output")
|| arg.starts_with("-o")
}),
"rg" => !args.iter().any(|arg| arg.starts_with("--pre")),
"xxd" => !args.contains(&"-r") && !args.contains(&"-revert"),
"go" => {
matches!(args.first(), Some(&"list" | &"doc" | &"env" | &"version"))
&& !args.iter().any(|arg| matches!(*arg, "-w" | "-u"))
}
"docker" | "podman" => matches!(args.first(), Some(&"ps" | &"version")),
"cat" | "grep" | "ls" | "nl" | "head" | "tail" | "wc" | "pwd" | "stat" | "file" | "du"
| "df" | "which" | "type" | "diff" | "cmp" | "jq" | "uniq" | "cut" | "readlink"
| "realpath" | "tree" | "basename" | "dirname" | "printenv" | "echo" | "printf"
| "less" | "more" | "test" | "[" | "ps" | "pgrep" | "pkg-config" | "strings" | "uname"
| "od" | "true" | ":" | "cd" | "pstree" | "sha256sum" | "sha1sum" | "md5sum" | "lsof"
| "tr" | "free" | "id" | "namei" | "whoami" | "paste" | "ss" | "getent" => true,
_ => false,
}
}
fn git_is_retrieval(args: &[&str]) -> bool {
let args = skip_shell_options(
args,
&["-C", "-c", "--git-dir", "--work-tree", "--namespace"],
);
let Some((subcommand, rest)) = args.split_first() else {
return false;
};
match *subcommand {
"diff" => !rest.contains(&"--check"),
"branch" => {
rest.is_empty()
|| rest.iter().all(|arg| {
matches!(
*arg,
"-a" | "-r"
| "--all"
| "--list"
| "--show-current"
| "-v"
| "-vv"
| "--verbose"
)
})
}
"remote" => {
rest.is_empty()
|| matches!(rest, ["-v" | "--verbose"])
|| rest.first() == Some(&"get-url")
}
"tag" => rest.is_empty() || matches!(rest[0], "-l" | "--list"),
"config" => matches!(
rest.first(),
Some(&"--get" | &"--get-all" | &"--list" | &"-l")
),
"worktree" => rest.first() == Some(&"list"),
"submodule" => rest.first() == Some(&"status"),
"status" | "log" | "show" | "blame" | "ls-files" | "ls-remote" | "rev-parse"
| "merge-base" | "grep" | "describe" | "show-ref" | "check-ignore" | "rev-list"
| "ls-tree" | "diff-tree" | "version" => true,
_ => false,
}
}
fn is_builtin_tool_name(lower: &str) -> bool {
WRITE_TOOL_NAMES.contains(&lower)
|| EDIT_TOOL_NAMES.contains(&lower)
|| READ_TOOL_NAMES.contains(&lower)
|| PLAN_TOOL_NAMES.contains(&lower)
|| BASH_TOOL_NAMES.contains(&lower)
}
/// Split a shell line at unquoted command separators. This intentionally avoids
/// pretending to be a full shell parser; only the leading program and flags of
/// each segment are inspected below.
fn shell_segments(command: &str) -> impl Iterator<Item = &str> {
let mut start = 0;
quoted_chars(command)
.filter(|&(_, c, quote)| quote.is_none() && matches!(c, '\n' | ';' | '|' | '&'))
.map(|(index, _, _)| index)
.chain([command.len()])
.filter_map(move |end| {
let segment = command[start..end].trim();
// Separators are one byte.
start = end + 1;
(!segment.is_empty()).then_some(segment)
})
}
/// Unescaped characters with their byte index and the quote around them.
fn quoted_chars(command: &str) -> impl Iterator<Item = (usize, char, Option<char>)> {
let mut quote = None;
let mut escaped = false;
command.char_indices().filter_map(move |(index, c)| {
if escaped {
escaped = false;
return None;
}
if c == '\\' && quote != Some('\'') {
escaped = true;
return None;
}
let around = quote;
if quote == Some(c) {
quote = None;
} else if quote.is_none() && matches!(c, '\'' | '"') {
quote = Some(c);
}
Some((index, c, around))
})
}
fn shell_words(segment: &str) -> std::iter::Peekable<std::str::SplitAsciiWhitespace<'_>> {
let mut words = segment.split_ascii_whitespace().peekable();
if words.peek().copied() == Some("env") {
words.next();
while words.peek().is_some_and(|word| word.starts_with('-')) {
words.next();
}
}
while words
.peek()
.is_some_and(|word| word.contains('=') && !word.starts_with('='))
{
words.next();
}
words
}
fn program_name(word: &str) -> &str {
Path::new(word)
.file_name()
.and_then(|name| name.to_str())
.unwrap_or(word)
}
fn shell_invokes_program(command: &str, expected: &str) -> bool {
shell_segments(command).any(|segment| {
shell_words(segment)
.next()
.is_some_and(|word| program_name(word) == expected)
})
}
fn shell_command_is_write(command: &str) -> bool {
shell_segments(command).any(|segment| {
let mut words = shell_words(segment);
let Some(program) = words.next().map(program_name) else {
return false;
};
if matches!(program, "cp" | "mkdir" | "touch" | "install") {
return true;
}
let redirects_output = words.any(|word| matches!(word, ">" | ">>"));
redirects_output
&& (matches!(program, "echo" | "printf" | "git")
|| BASH_READ_COMMANDS.contains(&program))
})
}
fn shell_command_is_edit(command: &str) -> bool {
shell_segments(command).any(|segment| {
let mut words = shell_words(segment);
let Some(program) = words.next().map(program_name) else {
return false;
};
let has_arg = |arg: &str| words.clone().any(|word| word == arg);
match program {
"mv" | "rm" => true,
"perl" => words
.take_while(|word| word.starts_with('-'))
.any(|option| {
option
.trim_start_matches('-')
.chars()
.any(|flag| flag == 'i')
}),
"git" => words
.next()
.is_some_and(|subcommand| matches!(subcommand, "apply" | "am" | "restore")),
"gofmt" => has_arg("-w"),
"cargo" => words.clone().next() == Some("fmt") && !has_arg("--check"),
"ruff" => {
let subcommand = words.clone().next();
(subcommand == Some("format") && !has_arg("--check"))
|| (subcommand == Some("check") && has_arg("--fix"))
}
"prettier" => has_arg("--write"),
"black" => !has_arg("--check"),
_ => {
(words.clone().any(|word| program_name(word) == "prettier") && has_arg("--write"))
|| (words.clone().any(|word| program_name(word) == "ruff")
&& ((has_arg("format") && !has_arg("--check"))
|| (has_arg("check") && has_arg("--fix"))))
}
}
})
}
fn shell_command_is_read(command: &str) -> bool {
shell_segments(command).any(|segment| {
if segment == "env" {
return true;
}
let mut words = shell_words(segment);
let Some(program) = words.next().map(program_name) else {
return false;
};
if BASH_READ_COMMANDS.contains(&program) {
return true;
}
if program == "command" && words.next() == Some("-v") {
return true;
}
if program == "type" {
return true;
}
if program != "git" {
return false;
}
match words.next() {
Some("branch") => words.next().is_none_or(|arg| arg.starts_with('-')),
Some("remote") => words
.next()
.is_none_or(|arg| arg.starts_with('-') || arg == "get-url"),
Some("config") => words
.next()
.is_some_and(|arg| matches!(arg, "--get" | "--get-all" | "--list" | "-l")),
Some(subcommand) => GIT_READ_SUBCOMMANDS.contains(&subcommand),
None => false,
}
})
}
// ─── extraction entry point ───────────────────────────────────────────────────
/// Extract all tool-execution signals from a normalized [`Request`].
///
/// Returns [`ToolSignals::default()`] when the message history contains no tool
/// activity, so callers can always inspect the signal fields.
fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> ToolSignals {
extract_tool_signals_with_window_and_semantics(
request,
recent_window,
&ToolSemantics::default(),
false,
)
}
fn extract_tool_signals_with_window_and_semantics(
request: &Request,
recent_window: usize,
semantics: &ToolSemantics,
current_user_turn: bool,
) -> ToolSignals {
// Read the decoded conversation, including preserved built-in tool outputs.
let messages = &request.llm_request.messages;
let boundary = if current_user_turn {
messages.iter().rposition(is_user_turn)
} else {
None
};
let start = boundary.map_or(0, |index| index + 1);
let namespaces = tool_namespaces(&request.llm_request.extensions);
let mut tool_texts: Vec<(String, bool)> = Vec::new();
let mut tool_calls: Vec<ObservedToolCall> = Vec::new();
// IDs whose latest call is a retrieval tool.
let mut retrieval_calls: HashSet<&str> = HashSet::new();
let mut tool_result_count = 0usize;
let mut assistant_turn_count = 0usize;
for (index, message) in messages.iter().enumerate() {
if index >= start && message.role == Role::Assistant {
assistant_turn_count += 1;
}
for block in &message.content {
match block {
_ if index < start => continue,
ContentBlock::ToolCall(call) => {
// Responses namespaced tools arrive as `<namespace>__<tool>`.
let bare_name = namespaces
.and_then(|namespaces| split_qualified_name(namespaces, &call.name))
.map(|(tool, _)| tool)
.or_else(|| mcp_tool_name(&call.name));
// The Responses wire format sends arguments as a JSON string.
let decoded = call
.arguments
.as_str()
.and_then(|raw| serde_json::from_str::<Value>(raw).ok());
let arguments = decoded.as_ref().unwrap_or(&call.arguments);
let command_field = command_of(arguments);
// Codex uses exec.input for JavaScript. OpenClaw's exec.command
// remains a shell call, even when the names are identical.
let code_semantic = (bare_name.unwrap_or(&call.name) == "exec"
&& arguments.get("command").is_none()
&& arguments.get("cmd").is_none())
.then(|| arguments.get("input").and_then(Value::as_str))
.flatten()
.map(|input| {
if code_mode_has_patch(input) {
ToolSemantic::Mutate(MutationKind::Edit)
} else {
ToolSemantic::Unknown
}
});
let command = command_field.and_then(command_text);
// The joined name wins, as in `build_signal`. A joined name
// configured as observe still counts when its bare name is a
// retrieval tool, such as `mcp__files__read`.
let full = classify_tool_call_with_semantics(
&call.name,
command.as_deref(),
semantics,
);
let name = match (full, bare_name) {
(ToolSemantic::Unknown | ToolSemantic::Observe, Some(bare_name)) => {
bare_name
}
_ => call.name.as_str(),
};
let is_retrieval =
code_semantic.is_none() && is_retrieval_tool(name, command_field);
if !call.id.is_empty() {
// A reused ID links to its latest call.
if is_retrieval {
retrieval_calls.insert(call.id.as_str());
} else {
retrieval_calls.remove(call.id.as_str());
}
}
tool_calls.push(ObservedToolCall {
name: call.name.clone(),
bare_name,
command,
is_retrieval,
code_semantic,
});
}
ContentBlock::ToolResult(result) => {
// Before the empty-text filter: empty results still count.
tool_result_count += 1;
let is_error = result.is_error == Some(true);
let text = result
.content
.iter()
.filter_map(text_of)
.collect::<Vec<_>>()
.join("\n");
let is_retrieval_result =
retrieval_calls.contains(result.tool_call_id.as_str());
if is_retrieval_result {
// Shell reads can return bare JSON from a file. Hermes wraps
// terminal output with output and exit_code fields.
let shell_result = serde_json::Deserializer::from_str(&text)
.into_iter::<Value>()
.next()
.and_then(|result| result.ok())
.filter(|value| {
value["output"].is_string() && value.get("exit_code").is_some()
});
// Codex reports the exit status in a header before `Output:`.
// An exit line after it is part of what was read.
let status = text
.split_once("\nOutput:\n")
.map_or(text.as_str(), |(header, _)| header);
// Hermes reports it in fields, so its output is not scanned.
let shell_failed = match &shell_result {
Some(value) => {
value["exit_code"].as_i64().is_some_and(|code| code != 0)
|| value["success"].as_bool() == Some(false)
|| value["error"]
.as_str()
.is_some_and(|error| !error.trim().is_empty())
}
None => has_nonzero_exit_status(&status.to_lowercase()),
};
if is_missing_file(&text) || (!is_error && !shell_failed) {
// Keep the window slot after dropping retrieved content.
if !text.is_empty() {
tool_texts.push((String::new(), false));
}
continue;
}
}
// An explicit failure remains a signal even without text.
if !text.is_empty() || is_error {
tool_texts.push((text, is_error));
}
}
// Built-in tool history stays opaque so it can be replayed unchanged.
ContentBlock::Unknown { provider, raw }
if provider.as_str() == WireFormat::OpenAiResponses.as_str()
&& raw.get("type").and_then(Value::as_str)
== Some("apply_patch_call_output") =>
{
tool_result_count += 1;
let text = raw
.get("output")
.and_then(Value::as_str)
.unwrap_or_default();
let is_error = raw.get("status").and_then(Value::as_str) == Some("failed");
tool_texts.push((text.to_owned(), is_error));
}
ContentBlock::Unknown { provider, raw }
if provider.as_str() == WireFormat::OpenAiResponses.as_str()
&& raw.get("type").and_then(Value::as_str) == Some("shell_call_output") =>
{
tool_result_count += 1;
let mut texts = Vec::new();
let mut is_error = false;
if let Some(outputs) = raw.get("output").and_then(Value::as_array) {
for output in outputs {
for field in ["stdout", "stderr"] {
if let Some(text) = output.get(field).and_then(Value::as_str)
&& !text.is_empty()
{
texts.push(text);
}
}
if let Some(outcome) = output.get("outcome") {
is_error |= match outcome.get("type").and_then(Value::as_str) {
Some("timeout") => true,
Some("exit")
if outcome
.get("exit_code")
.and_then(Value::as_i64)
.is_some_and(|code| code != 0) =>
{
// Keep plain nonzero exits SOFT, including empty output.
texts.push("returned non-zero");
output
.get("stderr")
.and_then(Value::as_str)
.is_some_and(|text| !text.trim().is_empty())
}
_ => false,
};
}
}
}
let text = texts.join("\n");
tool_texts.push((text, is_error));
}
_ => {}
}
}
}
let mut signal = build_signal(
tool_texts,
tool_calls,
(messages.len() - boundary.unwrap_or(0)) as u32,
recent_window,
semantics,
);
signal.compacted = request.llm_request.has_compaction;
signal.tool_result_count = u32::try_from(tool_result_count).unwrap_or(u32::MAX);
signal.assistant_turn_count = u32::try_from(assistant_turn_count).unwrap_or(u32::MAX);
signal
}
/// The tool part of an `mcp__<server>__<tool>` name, the form Claude Code uses
/// for MCP tools. The server name is assumed not to contain `__`; the tool name
/// may.
fn mcp_tool_name(name: &str) -> Option<&str> {
let (_server, tool) = name.strip_prefix("mcp__")?.split_once("__")?;
(!tool.is_empty()).then_some(tool)
}
/// The command field a tool call carries, when it has one. Harnesses name it
/// `command`, `cmd` or `input`; anything else is a tool whose category comes
/// from its name.
fn command_of(arguments: &Value) -> Option<&Value> {
["command", "cmd", "input"]
.iter()
.find_map(|key| arguments.get(*key))
}
/// A command field as lowercase text, from a string or an argv array.
fn command_text(value: &Value) -> Option<String> {
match value {
Value::String(text) => Some(text.to_lowercase()),
Value::Array(parts) => {
let joined = parts
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>()
.join(" ");
(!joined.is_empty()).then(|| joined.to_lowercase())
}
_ => None,
}
}
/// Text carried by a content block, ignoring the non-textual kinds.
fn text_of(block: &ContentBlock) -> Option<&str> {
match block {
ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text.as_str()),
_ => None,
}
}
/// Reads of missing paths happen as often on tasks that succeed as on tasks
/// that fail, so they are not a failure signal.
fn is_missing_file(text: &str) -> bool {
let lower = text.to_lowercase();
// The OS error from shell tools, and Claude Code's Read tool message.
lower.contains("no such file or directory") || lower.contains("file does not exist")
}
fn build_signal(
tool_texts: Vec<(String, bool)>,
tool_calls: Vec<ObservedToolCall>,
turn_depth: u32,
recent_window: usize,
semantics: &ToolSemantics,
) -> ToolSignals {
// Windowed severity: take the MAX severity across the last `recent_window` tool
// results rather than only the last one. An error's severity then persists for
// the recent window and decays out of it — parallel to the windowed `recent_*`
// counts — so a fix written a couple of turns after an error still routes on the
// error signal instead of the router flapping straight back to the weak tier.
let sev_start = tool_texts.len().saturating_sub(recent_window.max(1));
let mut severity = 0.0f32;
let mut failure_fingerprints = Vec::new();
let mut repeated_failure = false;
for (text, is_error) in &tool_texts[sev_start..] {
let (sev, _patterns) = classify_text(text);
// Explicit failure is at least hard; retain stronger text diagnostics.
let sev = if *is_error { sev.max(HARD) } else { sev };
if sev > severity {
severity = sev;
}
if let Some(fingerprint) = failure_fingerprint(text, *is_error) {
repeated_failure |= failure_fingerprints.contains(&fingerprint);
failure_fingerprints.push(fingerprint);
}
}
let no_error_streak = compute_no_error_streak(&tool_texts);
// Single pass: cumulative + sliding-window counters together. Also tracks
// the trailing pure-bash streak (consecutive `Unknown` calls back
// from the end) — the build-pit proxy.
let recent_start = tool_calls.len().saturating_sub(recent_window);
let mut write_count = 0u32;
let mut edit_count = 0u32;
let mut read_count = 0u32;
let mut todowrite_count = 0u32;
let mut recent_write_count = 0u32;
let mut recent_edit_count = 0u32;
let mut recent_read_count = 0u32;
let mut recent_todowrite_count = 0u32;
let mut new_count = 0u32;
let mut recent_new_count = 0u32;
let mut pure_bash_streak = 0u32;
let mut streak_open = true;
for (i, tc) in tool_calls.iter().enumerate().rev() {
// Every read, found or missed, is an observation. The read check parses
// the command, so it beats the substring patterns. Otherwise the joined
// name wins, so configs that list it keep working.
let mut cat = if let Some(semantic) = tc.code_semantic {
semantic
} else if tc.is_retrieval {
ToolSemantic::Observe
} else {
classify_tool_call_with_semantics(&tc.name, tc.command.as_deref(), semantics)
};
if matches!(cat, ToolSemantic::Unknown)
&& tc.code_semantic.is_none()
&& let Some(bare_name) = tc.bare_name
{
cat = classify_tool_call_with_semantics(bare_name, tc.command.as_deref(), semantics);
}
if streak_open {
if matches!(cat, ToolSemantic::Unknown) {
pure_bash_streak += 1;
} else {
streak_open = false;
}
}
match cat {
ToolSemantic::Mutate(MutationKind::Write) => {
write_count += 1;
if i >= recent_start {
recent_write_count += 1;
}
}
ToolSemantic::Mutate(MutationKind::Edit) => {
edit_count += 1;
if i >= recent_start {
recent_edit_count += 1;
}
}
ToolSemantic::Observe => {
read_count += 1;
if i >= recent_start {
recent_read_count += 1;
}
}
ToolSemantic::Plan => {
todowrite_count += 1;
if i >= recent_start {
recent_todowrite_count += 1;
}
}
ToolSemantic::New => {
new_count += 1;
if i >= recent_start {
recent_new_count += 1;
}
}
ToolSemantic::Unknown => {}
}
}
let tests_passed = detect_tests_passed(&tool_texts, recent_window);
ToolSignals {
severity,
repeated_failure,
no_error_streak,
edit_count,
write_count,
read_count,
todowrite_count,
recent_edit_count,
recent_write_count,
recent_read_count,
recent_todowrite_count,
new_count,
recent_new_count,
pure_bash_streak,
tests_passed,
turn_depth,
// Set by extract_tool_signals_with_window after the format-specific extract,
// which scans all message contents for the compaction marker and tallies
// the raw conversation-shape counts.
tool_result_count: 0,
assistant_turn_count: 0,
compacted: false,
}
}
// ─── pure helpers ─────────────────────────────────────────────────────────────
/// Normalise a JSON tool-result content value to a plain string.
fn content_to_text(content: Option<&Value>) -> Option<String> {
match content? {
Value::String(s) => Some(s.clone()),
Value::Array(blocks) => {
let parts: Vec<&str> = blocks
.iter()
.filter_map(|b| {
b.as_object()
.filter(|o| o.get("type").and_then(Value::as_str) == Some("text"))
.and_then(|o| o.get("text"))
.and_then(Value::as_str)
})
.collect();
if parts.is_empty() {
None
} else {
Some(parts.join("\n"))
}
}
_ => None,
}
}
/// Match tool text and structured result fields against error patterns.
///
/// Returns `(max_severity, matched_pattern_names)`.
pub(crate) fn classify_text(text: &str) -> (f32, Vec<String>) {
let lower = text.to_lowercase();
let mut patterns = Vec::new();
let mut severity: f32 = 0.0;
for (name, sev, substrings) in ERROR_PATTERNS {
if substrings.iter().any(|sub| lower.contains(sub)) {
patterns.push(name.to_string());
severity = severity.max(*sev);
}
}
// Hermes can append a loop warning after the JSON result.
let result = serde_json::Deserializer::from_str(text)
.into_iter::<Value>()
.next()
.and_then(|result| result.ok())
.unwrap_or_default();
let nonzero_exit = result["exit_code"].as_i64().is_some_and(|code| code != 0);
let tool_error = result["success"].as_bool() == Some(false)
|| result["error"]
.as_str()
.is_some_and(|error| !error.trim().is_empty());
if (nonzero_exit || has_nonzero_exit_status(&lower))
&& !patterns.iter().any(|p| p == "exit_nonzero")
{
patterns.push("exit_nonzero".to_string());
severity = severity.max(SOFT);
}
for (name, matched) in [
("tool_error", tool_error),
("compile_error", has_compiler_diagnostic(&lower)),
("runtime_exception", has_runtime_exception(&lower)),
("runtime_panic", has_runtime_panic(&lower)),
("patch_error", has_patch_failure(&lower)),
("cmd_not_found", ends_with_not_found(&lower)),
] {
if matched && !patterns.iter().any(|pattern| pattern == name) {
patterns.push(name.to_string());
severity = severity.max(HARD);
}
}
(severity, patterns)
}
/// Stable identity for a material failure. Soft non-zero exits need an explicit
/// failure flag to count as a repeated mistake.
fn failure_fingerprint(text: &str, is_error: bool) -> Option<String> {
let (severity, patterns) = classify_text(text);
if severity < HARD && !is_error {
return None;
}
let lower = text.to_lowercase();
let diagnostic = lower
.lines()
.find(|line| is_failure_diagnostic(line))
.or_else(|| lower.lines().find(|line| !line.trim().is_empty()))
.unwrap_or_default();
let normalized = normalize_failure_text(diagnostic);
Some(format!("{}|{normalized}", patterns.join(",")))
}
fn is_failure_diagnostic(line: &str) -> bool {
let line = line.trim();
[
"error",
"exception",
"panic",
"failed",
"timed out",
"timeout",
"connection refused",
"cannot allocate memory",
"out of memory",
"not found",
]
.iter()
.any(|marker| line.contains(marker))
}
/// Removes values that normally change between retries while retaining the
/// diagnostic wording that distinguishes one failure from another.
fn normalize_failure_text(text: &str) -> String {
let mut normalized = String::new();
for word in text.split_whitespace() {
if !normalized.is_empty() {
normalized.push(' ');
}
let mut in_digits = false;
if word.starts_with('/') || word.contains("/src/") || word.contains("/tmp/") {
normalized.push_str("<path>");
continue;
}
for character in word.chars() {
if character.is_ascii_digit() {
if !in_digits {
normalized.push('#');
in_digits = true;
}
} else {
normalized.push(character);
in_digits = false;
}
}
}
normalized.chars().take(240).collect()
}
fn has_compiler_diagnostic(lower: &str) -> bool {
lower.lines().any(|line| {
let line = line.trim_start();
if matches!(
line,
"compilation failed" | "error: compilation failed" | "error: could not compile"
) || line.starts_with("error: could not compile ")
{
return true;
}
let Some(rest) = line.strip_prefix("error[e") else {
return false;
};
let Some((code, _)) = rest.split_once("]:") else {
return false;
};
!code.is_empty() && code.chars().all(|character| character.is_ascii_digit())
})
}
fn has_runtime_exception(lower: &str) -> bool {
let has_exception_line = lower.lines().any(|line| {
let line = line.trim_start();
[
"typeerror:",
"referenceerror:",
"rangeerror:",
"runtimeerror:",
"keyerror:",
"attributeerror:",
]
.iter()
.any(|prefix| line.starts_with(prefix))
});
has_exception_line && (lower.contains("\n at ") || lower.contains("\n at "))
}
fn has_runtime_panic(lower: &str) -> bool {
lower
.lines()
.any(|line| line.trim_start().starts_with("panic: runtime error:"))
&& (lower.contains("\ngoroutine ") || lower.contains("[signal sig"))
}
fn has_patch_failure(lower: &str) -> bool {
lower.lines().any(|line| {
let line = line.trim_start();
line.starts_with("error: patch failed:")
|| line.starts_with("patch failed:")
|| line.contains(": patch does not apply")
|| line.starts_with("invalid context")
})
}
/// Shell "not found" line (e.g. `git: not found`, `sh: 1: git: not found`)
/// anchored to end-of-line so it still fires when the caller has trimmed the
/// tool result's trailing newline, without matching prose that merely mentions
/// "not found" mid-sentence.
///
/// The `: ` separator is required because `classify_text` lowercases its input,
/// so a bare end-of-line match would also claim successful output such as
/// `HTTP/1.1 404 Not Found`.
fn ends_with_not_found(lower: &str) -> bool {
lower
.lines()
.any(|line| line.trim_end().ends_with(": not found"))
}
/// Detects `exit_nonzero` only when a supported exit phrase is followed by a
/// nonzero decimal status.
///
/// Codex includes "Process exited with code 0" on clean tool results, so exit
/// phrases must parse their numeric status instead of matching the phrase alone.
fn has_nonzero_exit_status(lower: &str) -> bool {
NONZERO_EXIT_PHRASES
.iter()
.any(|phrase| phrase_followed_by_nonzero_integer(lower, phrase))
}
/// Matches common "exit code/status N" spellings after optional separators.
fn phrase_followed_by_nonzero_integer(lower: &str, phrase: &str) -> bool {
let mut cursor = 0usize;
while let Some(rel) = lower[cursor..].find(phrase) {
let value_start = cursor + rel + phrase.len();
let rest = lower[value_start..].trim_start_matches(|c: char| {
c.is_ascii_whitespace() || matches!(c, ':' | '=' | '\'' | '"' | '`')
});
let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
if !digits.is_empty() && digits.chars().any(|d| d != '0') {
return true;
}
cursor = value_start;
}
false
}
fn compute_no_error_streak(tool_texts: &[(String, bool)]) -> u32 {
let mut streak = 0u32;
for (text, is_error) in tool_texts.iter().rev() {
let (sev, _) = classify_text(text);
if *is_error || sev > 0.0 {
break;
}
streak += 1;
}
streak
}
fn detect_tests_passed(tool_texts: &[(String, bool)], recent_window: usize) -> bool {
let start = tool_texts.len().saturating_sub(recent_window.max(1));
let recent = &tool_texts[start..];
let after_latest_failure = recent
.iter()
.rposition(|(text, is_error)| *is_error || classify_text(text).0 > 0.0)
.map_or(recent, |index| &recent[index + 1..]);
after_latest_failure.iter().any(|(text, _)| {
let lower = text.to_lowercase();
TEST_PASS_PHRASES.iter().any(|p| lower.contains(p))
&& !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p))
&& !has_nonzero_failure_count(&lower)
})
}
// True iff `lower` contains a `NUMERIC_FAILURE_KEYWORDS` token preceded
// (modulo whitespace) by a nonzero integer. The "modulo whitespace" lets
// "1 failed", "1\nfailed", and "1 failed" all trip; the nonzero guard
// keeps cargo's "0 failed" / go's "0 errors" / pytest's "0 errors in"
// summaries from being misread as failures on a clean run.
fn has_nonzero_failure_count(lower: &str) -> bool {
for kw in NUMERIC_FAILURE_KEYWORDS {
let mut cursor = 0usize;
while let Some(rel) = lower[cursor..].find(kw) {
let kw_start = cursor + rel;
let kw_end = kw_start + kw.len();
// Word boundary AFTER the keyword — "errors" mid-word (e.g.
// "errored") shouldn't count as a failure-count site.
let boundary_after = lower[kw_end..]
.chars()
.next()
.is_none_or(|c| !c.is_ascii_alphanumeric());
if boundary_after {
let prefix = &lower[..kw_start];
let trimmed = prefix.trim_end_matches(|c: char| c.is_whitespace());
let digits_rev: String = trimmed
.chars()
.rev()
.take_while(|c| c.is_ascii_digit())
.collect();
if !digits_rev.is_empty() && digits_rev.chars().any(|d| d != '0') {
return true;
}
}
cursor = kw_start + kw.len();
}
}
false
}
// ─── tests ───────────────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::*;
use crate::algorithms::util::stage::score_signal;
use serde_json::json;
use switchyard_protocol::codex_namespaces::TOOL_NAMESPACES_KEY;
use switchyard_protocol::{
ContentBlock, LlmRequest, Message, Metadata, Role, ToolCall, ToolResult,
};
fn with_messages(messages: Vec<Message>) -> Request {
let mut request = Request {
llm_request: LlmRequest {
messages,
..LlmRequest::default()
},
raw_request: None,
metadata: None,
};
switchyard_translation::normalize_compaction(&mut request.llm_request, None);
request
}
// assistant message with a single named tool call
fn tc(name: &str) -> Message {
Message::new(
Role::Assistant,
vec![ContentBlock::ToolCall(ToolCall {
id: String::new(),
name: name.to_string(),
arguments: json!({}),
})],
)
}
// assistant Bash message carrying `command`
fn bash(command: &str) -> Message {
Message::new(
Role::Assistant,
vec![ContentBlock::ToolCall(ToolCall {
id: String::new(),
name: "Bash".to_string(),
arguments: json!({"command": command}),
})],
)
}
// a tool result message (goes in a user-role message, as in Anthropic's normalised form)
fn tr(text: &str) -> Message {
Message::new(
Role::User,
vec![ContentBlock::ToolResult(ToolResult {
tool_call_id: String::new(),
content: vec![ContentBlock::Text {
text: text.to_string(),
}],
is_error: None,
})],
)
}
#[test]
fn user_turn_signals_exclude_old_errors_and_keep_user_role_tool_results() {
let mut user = Message::text(Role::User, "next");
user.content.extend(tr("out of memory").content);
let request = with_messages(vec![tr("out of memory"), user, tr("ok")]);
let signals = ToolSignalProcessor::default().for_user_turn(&request);
assert_eq!(signals.tool_result_count, 1);
assert_eq!(signals.severity, 0.0);
assert_eq!(signals.turn_depth, 2);
}
#[test]
fn clean_text_has_zero_severity() {
let (sev, patterns) = classify_text("everything went fine");
assert_eq!(sev, 0.0);
assert!(patterns.is_empty());
}
#[test]
fn structured_tool_failures_affect_recovery_signals() {
for (text, severity) in [
(r#"{"output":"","exit_code":7,"error":null}"#, SOFT),
(r#"{"output":"","exit_code":-1,"error":null}"#, SOFT),
(r#"{"success":false,"error":"No matching text"}"#, HARD),
(r#"{"success":false,"error":null}"#, HARD),
(
r#"{"error":"Overwrite refused","stale_write_blocked":true}"#,
HARD,
),
(r#"{"output":"out of memory","exit_code":1}"#, CRITICAL),
(r#"{"output":"done","exit_code":0,"error":null}"#, 0.0),
(r#"{"success":true,"error":" "}"#, 0.0),
(r#"{"output":"running","exit_code":null,"error":null}"#, 0.0),
(r#"{"output":{"success":false},"exit_code":0}"#, 0.0),
] {
let warned = format!("{text}\n\n[Tool loop warning: repeated identical call]");
let request = with_messages(vec![tr("5 passed in 0.12s"), tr(text), tr(&warned)]);
let signal = ToolSignals::from_request(&request, None);
let clean = severity == 0.0;
assert_eq!(signal.severity, severity, "{text}");
assert_eq!(signal.no_error_streak, if clean { 3 } else { 0 }, "{text}");
assert_eq!(signal.repeated_failure, severity >= HARD, "{text}");
assert_eq!(signal.tests_passed, clean, "{text}");
}
}
#[test]
fn traceback_is_hard() {
let (sev, patterns) = classify_text("Traceback (most recent call last):\n ValueError");
assert_eq!(sev, HARD);
assert!(patterns.contains(&"traceback".to_string()));
}
#[test]
fn oom_is_critical() {
let (sev, _) = classify_text("Out of memory: kill process 1234");
assert_eq!(sev, CRITICAL);
}
#[test]
fn connection_refused_is_hard() {
let (severity, _) = classify_text("Connection refused on port 8000");
assert_eq!(severity, HARD);
}
#[test]
fn repeated_failure_ignores_volatile_paths_and_numbers() {
let request = with_messages(vec![
tr("error[E0308]: mismatched types at /tmp/a/src/lib.rs:12"),
tr("error[E0308]: mismatched types at /tmp/b/src/lib.rs:47"),
]);
assert!(ToolSignals::from_request(&request, None).repeated_failure);
}
#[test]
fn different_failures_are_not_repeated() {
let request = with_messages(vec![
tr("error[E0308]: mismatched types"),
tr("error[E0509]: cannot move out"),
]);
assert!(!ToolSignals::from_request(&request, None).repeated_failure);
}
#[test]
fn one_material_failure_is_not_repeated() {
let request = with_messages(vec![tr("Connection refused on port 8000")]);
assert!(!ToolSignals::from_request(&request, None).repeated_failure);
}
/// Explicit failures count even without diagnostic text and cannot signal recovery.
#[test]
fn structured_tool_failures_feed_error_and_recovery_signals() {
for text in ["Dependency unavailable", "", "5 passed in 0.12s"] {
let mut failed = tr(text);
let ContentBlock::ToolResult(result) = &mut failed.content[0] else {
panic!("expected tool result");
};
result.is_error = Some(true);
let mut request = with_messages(vec![tr("5 passed in 0.12s"), failed.clone()]);
let signals = ToolSignals::from_request(&request, Some(3));
assert_eq!(signals.severity, HARD);
assert!(!signals.repeated_failure);
assert_eq!(signals.no_error_streak, 0);
assert!(!signals.tests_passed);
request.llm_request.messages.push(failed);
assert!(ToolSignals::from_request(&request, Some(3)).repeated_failure);
request.llm_request.messages.push(tr("5 passed in 0.12s"));
let recovered = ToolSignals::from_request(&request, Some(1));
assert_eq!(recovered.severity, 0.0);
assert!(!recovered.repeated_failure);
assert_eq!(recovered.no_error_streak, 1);
assert!(recovered.tests_passed);
}
}
#[test]
fn severity_is_max_across_patterns() {
// exit_nonzero (SOFT) + traceback (HARD) → HARD.
let (sev, _) = classify_text("exit code 1\nTraceback (most recent call last):");
assert_eq!(sev, HARD);
}
#[test]
fn codex_process_exit_zero_stays_clean() {
let (sev, patterns) =
classify_text("Chunk ID: abc\nProcess exited with code 0\nOutput:\nok");
assert_eq!(sev, 0.0);
assert!(!patterns.contains(&"exit_nonzero".to_string()));
}
#[test]
fn nonzero_exit_codes_are_soft_errors() {
let cases = [
"Process exited with code 1",
"Process exited with code 127",
"exit code: 2",
"exit status 3",
"exited with status 9",
];
for case in cases {
let (sev, patterns) = classify_text(case);
assert_eq!(sev, SOFT, "expected soft severity for {case}");
assert!(patterns.contains(&"exit_nonzero".to_string()));
}
}
#[test]
fn partial_process_failures_are_hard_errors() {
let cases = [
(
"Process running with session ID 12\nOutput:\nerror[E0509]: cannot move out",
"compile_error",
),
(
"Process exited with code 0\nOutput:\nTypeError: value is undefined\n at main.js:1:2",
"runtime_exception",
),
(
"Process running with session ID 13\nOutput:\npanic: runtime error: index out of range\n\ngoroutine 6 [running]:",
"runtime_panic",
),
(
"Process exited with code 0\nOutput:\nerror: patch failed: src/lib.rs:4\nerror: src/lib.rs: patch does not apply",
"patch_error",
),
];
for (text, expected_pattern) in cases {
let (severity, patterns) = classify_text(text);
assert_eq!(severity, HARD, "expected hard severity for {text}");
assert!(patterns.iter().any(|pattern| pattern == expected_pattern));
}
}
/// A harness that strips the trailing newline from captured output must not
/// silently drop a shell "not found" error to severity 0.0.
#[test]
fn trimmed_command_not_found_is_still_hard() {
let cases = [
"git: not found",
"sh: 1: git: not found",
"/bin/sh: 1: git: not found",
];
for text in cases {
let (sev, patterns) = classify_text(text);
assert_eq!(sev, HARD, "expected hard severity for {text}");
assert!(patterns.contains(&"cmd_not_found".to_string()));
}
}
/// Precision guard: "not found" must end the line to count as a shell error,
/// so prose that merely mentions it does not trip a false error.
#[test]
fn not_found_mid_sentence_stays_clean() {
let (sev, _) =
classify_text("Looking for config key, not found in environment, using default.");
assert_eq!(sev, 0.0);
}
/// Precision guard: a line may end in "not found" and still report success.
/// Only the shell's `<name>: not found` shape counts.
#[test]
fn not_found_without_shell_separator_stays_clean() {
for text in [
"HTTP/1.1 404 Not Found",
"lookup finished, key not found",
"GET /healthz -> 404 Not Found",
] {
let (sev, patterns) = classify_text(text);
assert_eq!(sev, 0.0, "expected clean classification for {text}");
assert!(!patterns.contains(&"cmd_not_found".to_string()));
}
}
#[test]
fn source_text_that_names_exceptions_stays_clean() {
let text =
"pub enum TypeError: this is documentation\nlet sample = 'panic: runtime error:';";
assert_eq!(classify_text(text).0, 0.0);
}
#[test]
fn file_does_not_exist_is_hard() {
// Claude Code Read-tool miss. Trace-mined addition (22 true / 2 false positives).
let (sev, patterns) =
classify_text("Error: File does not exist. Note: current working directory is /app.");
assert_eq!(sev, HARD);
assert!(patterns.contains(&"no_such_file".to_string()));
}
#[test]
fn bare_does_not_exist_stays_clean() {
// Precision guard: only the anchored "file does not exist" fires, so a bare
// "does not exist" in prose or directory output must not trip a false error.
let (sev, _) = classify_text("The directory does not exist yet, creating it now.");
assert_eq!(sev, 0.0);
}
#[test]
fn no_error_streak_all_clean() {
let texts = vec![("ok".to_string(), false), ("all good".to_string(), false)];
assert_eq!(compute_no_error_streak(&texts), 2);
}
#[test]
fn no_error_streak_stops_at_error() {
let texts = vec![
("Traceback (most recent call last):".to_string(), false),
("ok".to_string(), false),
("ok".to_string(), false),
];
assert_eq!(compute_no_error_streak(&texts), 2);
}
#[test]
fn tests_passed_detects_pytest_output() {
assert!(detect_tests_passed(
&[("====== 5 passed in 0.12s ======".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_ignores_partial_failures() {
assert!(!detect_tests_passed(
&[("2 failed, 5 passed in 0.56s".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_must_follow_the_latest_failure() {
assert!(!detect_tests_passed(
&[
("5 passed in 0.12s".to_string(), false),
(
"Traceback (most recent call last):\nValueError".to_string(),
false
),
("edit applied".to_string(), false),
],
DEFAULT_RECENT_WINDOW
));
assert!(detect_tests_passed(
&[
(
"Traceback (most recent call last):\nValueError".to_string(),
false
),
("5 passed in 0.12s".to_string(), false),
],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn retrieved_file_contents_are_ignored() {
let call = |id: &str, name: &str, arguments: Value| {
Message::new(
Role::Assistant,
vec![ContentBlock::ToolCall(ToolCall {
id: id.to_string(),
name: name.to_string(),
arguments,
})],
)
};
let result = |id: &str, text: &str| {
Message::new(
Role::User,
vec![ContentBlock::ToolResult(ToolResult {
tool_call_id: id.to_string(),
content: vec![ContentBlock::Text {
text: text.to_string(),
}],
is_error: None,
})],
)
};
let signal = extract_tool_signals_with_window(
&with_messages(vec![
call("a", "Bash", json!({"command": "pytest"})),
result("a", "Traceback (most recent call last):\nValueError"),
call("b", "Read", json!({"file_path": "notes.md"})),
result(
"b",
r#"{"success":false,"error":"out of memory","exit_code":7}"#,
),
call("c", "Grep", json!({"pattern": "passed"})),
result("c", "CHANGELOG.md: all tests passed"),
]),
DEFAULT_RECENT_WINDOW,
);
// Only the real pytest run counts.
assert_eq!(signal.severity, HARD);
assert!(!signal.tests_passed);
assert_eq!(signal.tool_result_count, 3);
for (command, text, severity) in [
("cat config.json", r#"{"error":"fixture data"}"#, 0.0),
(
"cat missing.json",
r#"{"output":"","exit_code":7,"error":null}"#,
SOFT,
),
(
"cat config.json && python check.py",
r#"{"error":"check rejected"}"#,
HARD,
),
] {
let request = with_messages(vec![
call("shell", "Bash", json!({"command": command})),
result("shell", text),
]);
assert_eq!(
ToolSignals::from_request(&request, None).severity,
severity,
"{text}"
);
}
}
// A request whose tool result answers the tool call.
fn call_with_result(mut call: Message, mut result: Message) -> Request {
if let ContentBlock::ToolCall(call) = &mut call.content[0] {
call.id = "call-1".into();
}
if let ContentBlock::ToolResult(result) = &mut result.content[0] {
result.tool_call_id = "call-1".into();
}
with_messages(vec![call, result])
}
#[test]
fn shell_reads_are_ignored_but_keep_their_window_slot() {
for command in [
"/usr/bin/cat logfile.txt",
"cd /repo && tail -n 100 logfile.txt | grep MemoryError",
"env MODE=debug timeout 5s rg 'MemoryError' .",
] {
let mut request = call_with_result(bash(command), tr("MemoryError\nall tests passed"));
request.llm_request.messages.splice(
0..0,
[bash("pytest"), tr("Traceback (most recent call last):")],
);
// The read adds no error and no test pass, so the earlier failure stands.
let signal = ToolSignals::from_request(&request, Some(2));
assert_eq!(signal.severity, HARD, "{command}");
assert!(!signal.tests_passed, "{command}");
// The read still fills a window slot, so the failure can age out.
let signal = ToolSignals::from_request(&request, Some(1));
assert_eq!(signal.severity, 0.0, "{command}");
}
}
#[test]
fn shell_commands_that_run_code_keep_signals() {
let oversized = format!("cat {}", "a".repeat(MAX_RETRIEVAL_COMMAND_BYTES));
for command in [
"cat logfile.txt; pytest",
"echo $(pytest)",
"find . -name '*.py' -exec pytest ';'",
oversized.as_str(),
] {
let request = call_with_result(bash(command), tr("MemoryError"));
let signal = ToolSignals::from_request(&request, None);
assert_eq!(signal.severity, CRITICAL, "{command}");
}
}
#[test]
fn failed_reads_signal_unless_the_file_is_missing() {
let failed = |text: &str| {
let mut result = tr(text);
if let ContentBlock::ToolResult(result) = &mut result.content[0] {
result.is_error = Some(true);
}
result
};
let missing = "missing.txt: No such file or directory";
let severities = [
call_with_result(bash("cat missing.txt"), failed(missing)),
call_with_result(tc("Read"), failed("File does not exist.")),
// A read that fails for another reason still counts.
call_with_result(bash("cat logfile.txt"), failed("MemoryError")),
// Only reads are exempt.
call_with_result(bash("python missing.txt"), failed(missing)),
]
.map(|request| ToolSignals::from_request(&request, None).severity);
assert_eq!(severities, [0.0, 0.0, CRITICAL, HARD]);
let request = call_with_result(
bash("cat missing.json"),
tr(r#"{"output":"No such file or directory","exit_code":1}"#),
);
assert_eq!(ToolSignals::from_request(&request, None).severity, 0.0);
}
#[test]
fn codex_read_status_comes_from_the_header() {
let read = |header: &str| {
let output = format!(
"{header}\nOutput:\nTraceback (most recent call last):\nMemoryError\nProcess exited with code 1\n"
);
let call = exec_command(json!({"cmd": "cat run.log"}));
ToolSignals::from_request(&call_with_result(call, tr(&output)), None).severity
};
// The exit line belongs to the log that was read.
assert_eq!(read("Chunk ID: abc\nProcess exited with code 0"), 0.0);
// The read itself failed, so its output counts.
assert_eq!(read("Chunk ID: abc\nProcess exited with code 1"), CRITICAL);
}
#[test]
fn hermes_read_status_comes_from_exit_code() {
let read = |exit_code: i64| {
let output = json!({
"output": "Traceback (most recent call last):\nMemoryError\nProcess exited with code 1\n",
"exit_code": exit_code,
"error": null,
});
let call = Message::new(
Role::Assistant,
vec![ContentBlock::ToolCall(ToolCall {
id: String::new(),
name: "terminal".to_string(),
arguments: json!({"command": "cat run.log"}),
})],
);
ToolSignals::from_request(&call_with_result(call, tr(&output.to_string())), None)
.severity
};
// The exit line belongs to the log that was read.
assert_eq!(read(0), 0.0);
// The read itself failed, so its output counts.
assert_eq!(read(1), CRITICAL);
}
#[test]
fn argv_commands_keep_word_boundaries() {
let signal = |argv: Value| {
let call = exec_command(json!({"cmd": argv}));
ToolSignals::from_request(&call_with_result(call, tr("MemoryError")), None)
};
let mut many_args = vec![""; MAX_RETRIEVAL_COMMAND_BYTES];
many_args[0] = "cat";
let severities = [
json!(["bash", "-lc", "cd repo && rg foo"]),
json!(["bash", "-lc", "cat log; pytest"]),
// One argv word is one file name, even with a separator in it.
json!(["cat", "log; pytest"]),
// Over the byte budget, by size and by argument count.
json!(["cat", "a".repeat(MAX_RETRIEVAL_COMMAND_BYTES)]),
json!(many_args),
]
.map(|argv| signal(argv).severity);
assert_eq!(severities, [0.0, CRITICAL, 0.0, CRITICAL, CRITICAL]);
// A read is an observation, even as a whole `-lc` script.
let read = signal(json!(["bash", "-lc", "sed -n 1,200p x"]));
assert_eq!(read.read_count, 1);
// Other commands are counted inside the script too.
let write = signal(json!(["bash", "-lc", "cd repo && mkdir x"]));
assert_eq!(write.write_count, 1);
}
#[test]
fn severity_is_windowed_over_recent_results() {
// An error two results back, then two clean results.
let request = with_messages(vec![
tr("Traceback (most recent call last):\n ValueError"),
tr("ok"),
tr("ok"),
]);
// window covers the error → severity persists (max over the window)
assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD);
// window of 1 sees only the last (clean) result → severity has decayed out
assert_eq!(extract_tool_signals_with_window(&request, 1).severity, 0.0);
}
#[test]
fn extract_openai_chat_tool_results() {
let request = with_messages(vec![
Message::text(Role::User, "do something"),
tc("Edit"),
tr("Traceback (most recent call last):\n ValueError"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.severity, HARD);
assert_eq!(sig.edit_count, 1);
assert_eq!(sig.turn_depth, 3);
}
#[test]
fn extract_anthropic_tool_results() {
let request = with_messages(vec![tr("Traceback (most recent call last):\n ValueError")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.severity, HARD);
}
#[test]
fn extract_responses_api_tool_results() {
let request = with_messages(vec![tc("Write"), tr("file written successfully")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.severity, 0.0);
assert_eq!(sig.write_count, 1);
}
#[test]
fn responses_builtin_tool_failures_escalate() {
use crate::algorithms::util::stage::{PickOutcome, PickerMode, Tier, pick_tier};
let mut cases = Vec::new();
for (status, output) in [
(
"failed",
"Synthetic dependency unavailable; retry with the recovery path.",
),
(
"completed",
"Synthetic dependency unavailable; retry with the recovery path.",
),
("failed", ""),
] {
cases.push((
json!({
"type": "apply_patch_call_output",
"status": status,
"output": output,
}),
if status == "failed" { HARD } else { 0.0 },
));
}
for (outcome, stdout, stderr, severity) in [
(json!({"type": "exit", "exit_code": 1}), "", "", SOFT),
(json!({"type": "exit", "exit_code": 1}), "", " \n", SOFT),
(
json!({"type": "exit", "exit_code": 1}),
"",
"command failed",
HARD,
),
(json!({"type": "timeout"}), "", "", HARD),
(json!({"type": "exit", "exit_code": 0}), "done", "", 0.0),
(
json!({"type": "exit", "exit_code": 0}),
"Traceback (most recent call last):",
"",
HARD,
),
(
json!({"type": "exit", "exit_code": 0}),
"",
"Traceback (most recent call last):",
HARD,
),
] {
cases.push((
json!({
"type": "shell_call_output",
"output": [
{"stdout": stdout, "stderr": stderr, "outcome": outcome},
{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}
],
}),
severity,
));
}
for (raw, severity) in cases {
let is_error = severity >= HARD;
let mut request = with_messages(
["call_1", "call_2"]
.into_iter()
.map(|call_id| {
let mut raw = raw.clone();
raw["call_id"] = json!(call_id);
Message::new(
Role::User,
vec![ContentBlock::Unknown {
provider: WireFormat::OpenAiResponses.into(),
raw,
}],
)
})
.collect(),
);
let signal = ToolSignals::from_request(&request, Some(3));
assert_eq!(signal.severity, severity, "{raw}");
assert_eq!(signal.repeated_failure, is_error, "{raw}");
assert_eq!(signal.tool_result_count, 2);
assert_eq!(
matches!(
pick_tier(&signal, PickerMode::EfficientFirst, 0.5),
PickOutcome::Resolved {
tier: Tier::Capable,
..
}
),
is_error,
"{raw}"
);
let mut success = match raw["type"].as_str() {
Some("apply_patch_call_output") => json!({
"type": "apply_patch_call_output", "status": "completed", "output": ""
}),
Some("shell_call_output") => json!({
"type": "shell_call_output",
"output": [{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}]
}),
_ => unreachable!(),
};
for index in 0..3 {
success["call_id"] = json!(format!("success_{index}"));
request.llm_request.messages.push(Message::new(
Role::User,
vec![ContentBlock::Unknown {
provider: WireFormat::OpenAiResponses.into(),
raw: success.clone(),
}],
));
}
let recovered = ToolSignals::from_request(&request, Some(3));
assert_eq!(recovered.severity, 0.0, "{raw}");
assert!(!recovered.repeated_failure, "{raw}");
assert!(recovered.no_error_streak >= 3, "{raw}");
assert_eq!(recovered.tool_result_count, 5);
}
}
#[test]
fn conversation_counts_are_per_block_and_role_aware() {
// A batched user message (Anthropic shape) contributes one count per
// ToolResult block, empty-content results included. assistant_turn_count
// tracks Role::Assistant only, while turn_depth counts every message.
let result = |content: Vec<ContentBlock>| {
ContentBlock::ToolResult(ToolResult {
tool_call_id: String::new(),
content,
is_error: None,
})
};
let request = with_messages(vec![
Message::text(Role::User, "do something"),
Message::text(Role::Assistant, "working"),
Message::new(
Role::User,
vec![
result(vec![ContentBlock::Text {
text: "ok".to_string(),
}]),
result(Vec::new()),
],
),
tc("Bash"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.tool_result_count, 2);
assert_eq!(sig.assistant_turn_count, 2);
assert_eq!(sig.turn_depth, 4);
}
#[test]
fn recent_window_counts_only_last_default_window_tool_calls() {
// 5 writes + 1 edit at the end → the default window (3) should see
// the last 3 calls: 1 edit + 2 writes (not all 6 calls).
let request = with_messages(vec![
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Edit"),
tr("ok"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.write_count, 5);
assert_eq!(sig.edit_count, 1);
assert_eq!(sig.recent_write_count, 2);
assert_eq!(sig.recent_edit_count, 1);
}
#[test]
fn codex_apply_patch_counts_as_an_edit() {
let request = with_messages(vec![tc("apply_patch"), tr("Success. Updated the file")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.edit_count, 1);
assert_eq!(sig.recent_edit_count, 1);
}
fn exec_command(cmd: Value) -> Message {
Message::new(
Role::Assistant,
vec![ContentBlock::ToolCall(ToolCall {
id: String::new(),
name: "exec_command".to_string(),
arguments: cmd,
})],
)
}
#[test]
fn codex_exec_command_is_classified() {
// arguments arrive as a JSON string, with the command under `cmd`
let args = json!(r#"{"cmd":"sed -i s/a/b/ src/lib.rs","workdir":"/x"}"#);
let request = with_messages(vec![exec_command(args), tr("ok")]);
assert_eq!(
ToolSignals::from_request(&request, None).recent_edit_count,
1
);
}
#[test]
fn python_write_expressions_need_a_python_command() {
let write = with_messages(vec![
exec_command(json!({"cmd": "python3 - <<'PY'\np.write_text(s)\nPY"})),
tr("ok"),
]);
assert_eq!(
ToolSignals::from_request(&write, None).recent_write_count,
1
);
let search = with_messages(vec![
exec_command(json!({"cmd": "grep -R '.write(' src"})),
tr("ok"),
]);
assert_eq!(
ToolSignals::from_request(&search, None).recent_write_count,
0
);
}
#[test]
fn recent_window_size_is_caller_overridable() {
// Same six tool calls (1 edit at the end, 5 writes before).
// With recent_window=3 → recent_writes=2, recent_edits=1.
// With recent_window=6 → recent_writes=5, recent_edits=1 (all calls).
let request = with_messages(vec![
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Edit"),
tr("ok"),
]);
let narrow = extract_tool_signals_with_window(&request, 3);
assert_eq!(narrow.recent_write_count, 2);
assert_eq!(narrow.recent_edit_count, 1);
let wide = extract_tool_signals_with_window(&request, 6);
assert_eq!(wide.recent_write_count, 5);
assert_eq!(wide.recent_edit_count, 1);
}
#[test]
fn compaction_marker_sets_compacted() {
for summary in [
"This session is being continued from a previous conversation that ran out of context.",
"Another language model started to solve this problem and produced a summary of its thinking process.",
] {
let request = with_messages(vec![
Message::text(Role::User, summary),
Message::text(Role::User, "Continue with the next task"),
bash("ls"),
]);
assert!(ToolSignals::from_request(&request, None).compacted);
assert!(
ToolSignalProcessor::default()
.for_user_turn(&request)
.compacted
);
}
}
#[test]
fn hermes_compaction_headers_set_compacted() {
for header in [
"[CONTEXT COMPACTION \u{2014} REFERENCE ONLY] Earlier turns were compacted into the summary below.",
"[STILL IN PROGRESS \u{2014} this is the active request, restated after the compaction boundary because it was not finished yet. Continue it\u{3b} do not start over.]",
] {
for role in [Role::User, Role::Assistant] {
let request = with_messages(vec![
Message::text(role, format!("{header}\nContinue the task.")),
bash("ls"),
]);
assert!(ToolSignals::from_request(&request, None).compacted);
}
let request = with_messages(vec![Message::text(
Role::User,
format!("Prior context.\n\n {header}\nContinue the task."),
)]);
assert!(ToolSignals::from_request(&request, None).compacted);
let request = with_messages(vec![tr(header)]);
assert!(!ToolSignals::from_request(&request, None).compacted);
}
}
#[test]
fn ordinary_compaction_text_stays_uncompacted() {
for text in [
"The task is still in progress after the compaction boundary.",
"Explain [CONTEXT COMPACTION \u{2014} REFERENCE ONLY] in the docs.",
"[STILL IN PROGRESS] Continue the task.",
] {
let request = with_messages(vec![Message::text(Role::User, text)]);
assert!(!ToolSignals::from_request(&request, None).compacted);
}
}
#[test]
fn codex_compaction_metadata_stays_on_parent_route() {
let mut request = with_messages(vec![bash("ls")]);
request.metadata = Some(Metadata {
is_subagent: true,
agent_kind: Some("compact".to_string()),
..Default::default()
});
assert!(!ToolSignals::from_request(&request, None).compacted);
}
#[test]
fn no_compaction_marker_stays_uncompacted() {
let request = with_messages(vec![
Message::text(Role::User, "Write a script that parses the log file."),
bash("ls"),
]);
assert!(!ToolSignals::from_request(&request, None).compacted);
}
#[test]
fn bash_heredoc_counts_as_write() {
// Claude Code's pattern on TB 2.0 — write a scratch file via heredoc.
let request = with_messages(vec![bash("cat > /tmp/test.py <<'EOF'\nprint(1)\nEOF")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(
sig.write_count, 1,
"Bash heredoc should bucket into write_count"
);
assert_eq!(sig.edit_count, 0);
}
#[test]
fn bash_sed_inplace_counts_as_edit() {
let request = with_messages(vec![bash("sed -i 's/foo/bar/g' /app/file.py")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(
sig.edit_count, 1,
"Bash sed -i should bucket into edit_count"
);
assert_eq!(sig.write_count, 0);
}
#[test]
fn bash_non_mutating_does_not_count() {
// ls, cat, grep — should not increment either counter.
let request = with_messages(vec![bash("ls -la /app"), bash("cat /app/main.py")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.write_count, 0);
assert_eq!(sig.edit_count, 0);
}
#[test]
fn tests_passed_detects_pytest_with_failure_block() {
// Mixed pytest run: 2 failed + 5 passed → NOT considered tests_passed.
assert!(!detect_tests_passed(
&[("2 failed, 5 passed in 0.56s".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_accepts_cargo_clean_summary() {
// Cargo's clean-run summary contains "0 failed" — must not trip the
// failure list (regression: previously substring-matched "failed").
assert!(detect_tests_passed(
&[(
"running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string(),
false
)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_rejects_cargo_real_failure() {
// Cargo's actual-failure summary: nonzero count before "failed".
assert!(!detect_tests_passed(
&[(
"running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string(),
false
)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_accepts_go_clean_summary() {
// Go test's clean-run "0 errors" must not trip (regression).
assert!(detect_tests_passed(
&[(
"ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string(),
false
)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_accepts_pytest_zero_errors() {
// Pytest long-form: "0 errors in 0.3s" on a clean run.
assert!(detect_tests_passed(
&[("5 passed, 0 errors in 0.30s".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_detects_diy_checkmark() {
assert!(detect_tests_passed(
&[("✓ all checks passed".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn anthropic_bash_heredoc_extracts_command() {
// Anthropic format: tool_use.input is an object, not a JSON string.
let request = with_messages(vec![bash("cat > /tmp/foo.txt << 'EOF'\nhi\nEOF")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(
sig.write_count, 1,
"Anthropic Bash heredoc must also be detected"
);
}
#[test]
fn recent_window_falls_back_to_full_history_when_short() {
let request = with_messages(vec![tc("Write")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.recent_write_count, 1);
assert_eq!(sig.recent_edit_count, 0);
}
#[test]
fn clean_tool_result_has_zero_severity_and_non_empty_streak() {
let request = with_messages(vec![tr("output ok"), tr("another ok")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.severity, 0.0);
assert_eq!(sig.no_error_streak, 2);
}
// ─── asymmetric-signal extensions ────────────────────────────────────
#[test]
fn todowrite_classifies_as_plan() {
assert_eq!(classify_tool_call("TodoWrite", None), ToolSemantic::Plan);
assert_eq!(classify_tool_call("todo_write", None), ToolSemantic::Plan);
}
#[test]
fn codex_update_plan_classifies_as_plan() {
assert_eq!(classify_tool_call("update_plan", None), ToolSemantic::Plan);
}
#[test]
fn codex_shell_command_runs_bash_pattern_match() {
// shell_command + heredoc -> Write.
assert_eq!(
classify_tool_call("shell_command", Some("cat > /app/foo.py <<'eof'\nx=1\neof")),
ToolSemantic::Mutate(MutationKind::Write),
);
// shell_command + read-like inspection -> Read.
assert_eq!(
classify_tool_call("shell_command", Some("ls /app")),
ToolSemantic::Observe,
);
// shell_command without matching patterns -> Unknown.
assert_eq!(
classify_tool_call("shell_command", Some("./run_tests.sh")),
ToolSemantic::Unknown,
);
}
#[test]
fn text_editor_view_is_a_read() {
for name in ["str_replace_based_edit_tool", "text_editor"] {
assert_eq!(
classify_tool_call(name, Some("view")),
ToolSemantic::Observe
);
for command in [
Some("create"),
Some("insert"),
Some("str_replace"),
Some("undo_edit"),
None,
] {
assert_eq!(
classify_tool_call(name, command),
ToolSemantic::Mutate(MutationKind::Edit),
);
}
}
let arguments = [
json!({"command": "view", "path": "/app/main.py"}),
// the Responses wire format sends arguments as a JSON string
json!(r#"{"command":"view","path":"/app/main.py"}"#),
];
for arguments in arguments {
let call = Message::new(
Role::Assistant,
vec![ContentBlock::ToolCall(ToolCall {
id: String::new(),
name: "str_replace_based_edit_tool".to_string(),
arguments,
})],
);
let request = with_messages(vec![call, tr("print('hi')")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.read_count, 1);
assert_eq!(sig.recent_read_count, 1);
assert_eq!(sig.edit_count, 0);
}
}
#[test]
fn read_tool_classifies_as_read() {
assert_eq!(classify_tool_call("Read", None), ToolSemantic::Observe);
assert_eq!(classify_tool_call("View", None), ToolSemantic::Observe);
}
#[test]
fn hermes_tool_names_classify() {
// Hermes (NousResearch) file tools route by name.
assert_eq!(
classify_tool_call("write_file", None),
ToolSemantic::Mutate(MutationKind::Write)
);
assert_eq!(
classify_tool_call("patch", None),
ToolSemantic::Mutate(MutationKind::Edit)
);
assert_eq!(classify_tool_call("read_file", None), ToolSemantic::Observe);
assert_eq!(
classify_tool_call("search_files", None),
ToolSemantic::Observe
);
// Hermes runs shell through `terminal`, which carries a `command` arg,
// so its intent comes from the Bash-pattern match like codex's shell_command.
assert_eq!(
classify_tool_call("terminal", Some("sed -i 's/a/b/' /app/x.py")),
ToolSemantic::Mutate(MutationKind::Edit),
);
assert_eq!(
classify_tool_call("terminal", Some("grep foo /app")),
ToolSemantic::Observe,
);
assert_eq!(
classify_tool_call("terminal", Some("./run_tests.sh")),
ToolSemantic::Unknown,
);
}
#[test]
fn bash_read_patterns_classify_as_read() {
let cases = [
"cat /etc/passwd",
"grep foo bar.txt",
"ls /app",
"find . -name '*.py'",
];
for cmd in cases {
assert_eq!(
classify_tool_call("Bash", Some(cmd)),
ToolSemantic::Observe,
"expected Read for {cmd}"
);
}
}
#[test]
fn codex_inspection_commands_classify_as_read() {
let cases = [
"sed -n '1,80p' src/lib.rs",
"rg -n 'needle' src",
"nl -ba src/lib.rs",
"cat package.json",
"jq '.scripts' package.json",
"git status --short",
"git log --oneline -5",
"git show HEAD:src/lib.rs",
"git branch --show-current",
"git remote -v",
"git config --get remote.origin.url",
];
for command in cases {
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Observe,
"expected Read for {command}"
);
}
}
#[test]
fn quoted_shell_separators_do_not_create_commands() {
for command in ["rg 'foo|rm obsolete.rs'", "rg \"foo; rm obsolete.rs\""] {
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Observe,
"quoted text must not be parsed as a command: {command}"
);
}
}
#[test]
fn codex_shell_mutations_classify_as_production() {
let writes = [
"cp source.rs destination.rs",
"mkdir -p src/generated",
"touch src/generated/mod.rs",
"git show HEAD:file.rs > file.rs",
"node <<'node'\nfs.writefilesync('file.js', text)\nnode",
];
for command in writes {
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Mutate(MutationKind::Write),
"expected Write for {command}"
);
}
let edits = [
"mv old.rs new.rs",
"rm obsolete.rs",
"gofmt -w main.go",
"cargo fmt",
"ruff check --fix src",
"perl -0pi -e 's/old/new/' src/lib.rs",
"npx prettier --write src/lib.ts",
"uv run ruff format src",
"git apply fix.patch",
];
for command in edits {
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Mutate(MutationKind::Edit),
"expected Edit for {command}"
);
}
}
#[test]
fn formatter_checks_are_not_edits() {
for command in [
"cargo fmt --check",
"ruff format --check src",
"black --check src",
] {
assert_ne!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Mutate(MutationKind::Edit),
"read-only formatter check must not be Edit: {command}"
);
}
}
#[test]
fn embedded_comparison_is_not_a_shell_write() {
let command = "node <<'node'\nif (index > 0) console.log(index)\nnode";
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Unknown
);
}
#[test]
fn bash_write_precedence_over_read() {
// `cat /file > out` contains both `cat /` (read) and ` > ` (write);
// write redirection must win.
assert_eq!(
classify_tool_call("Bash", Some("cat /etc/hosts > /tmp/out")),
ToolSemantic::Mutate(MutationKind::Write),
);
}
#[test]
fn pure_bash_streak_counts_trailing_other() {
// 5 trailing non-classified Bash calls → streak == 5.
let request = with_messages(vec![
bash("make"),
tr("ok"),
bash("./configure"),
tr("ok"),
bash("make install"),
tr("ok"),
bash("./run.sh"),
tr("ok"),
bash("./test"),
tr("ok"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.pure_bash_streak, 5);
assert_eq!(sig.write_count, 0);
assert_eq!(sig.read_count, 0);
}
#[test]
fn pure_bash_streak_resets_on_write() {
let request = with_messages(vec![bash("make"), tr("ok"), tc("Write"), tr("ok")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.pure_bash_streak, 0);
assert_eq!(sig.write_count, 1);
}
#[test]
fn recent_window_tracks_todowrite_and_read() {
// Final 3 tool calls: TodoWrite, Read, TodoWrite.
let request = with_messages(vec![
bash("make"),
tr("ok"),
tc("TodoWrite"),
tr("ok"),
tc("Read"),
tr("ok"),
tc("TodoWrite"),
tr("ok"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.todowrite_count, 2);
assert_eq!(sig.recent_todowrite_count, 2);
assert_eq!(sig.read_count, 1);
assert_eq!(sig.recent_read_count, 1);
}
#[test]
fn configured_tool_semantics_extend_the_builtin_vocabulary() {
let semantics = ToolSemantics {
observe: vec!["KB_search".to_string()],
mutate: vec!["send_payment_request".to_string()],
plan: vec!["create_research_plan".to_string()],
new: vec!["send_message_to_user".to_string()],
};
semantics.validate().expect("valid additive semantics");
let request = with_messages(vec![
tc("Read"),
tc("Write"),
tc("TodoWrite"),
tc("kb_SEARCH"),
tc("send_payment_request"),
tc("create_research_plan"),
tc("send_message_to_user"),
tc("unlisted_tool"),
]);
let signal = ToolSignals::from_request_with_semantics(&request, None, &semantics);
assert_eq!(signal.read_count, 2);
assert_eq!(signal.write_count, 2);
assert_eq!(signal.todowrite_count, 2);
assert_eq!(signal.new_count, 1);
assert_eq!(signal.recent_new_count, 1);
assert_eq!(signal.pure_bash_streak, 1);
}
#[test]
fn configured_tool_semantics_match_namespaced_and_mcp_tools() {
// The Responses decoder flattens namespaced tools and records the mapping.
let mut request = with_messages(vec![tc("mcp__billing__send_payment_request")]);
request.llm_request.extensions.fields.insert(
TOOL_NAMESPACES_KEY.to_string(),
json!({"mcp__billing__send_payment_request": "mcp__billing"}),
);
// Claude Code sends MCP tools flat, with no namespace mapping.
let claude_request = with_messages(vec![tc("mcp__billing__send_payment_request")]);
for request in [&request, &claude_request] {
for name in ["send_payment_request", "mcp__billing__send_payment_request"] {
let semantics = ToolSemantics {
mutate: vec![name.to_string()],
..Default::default()
};
let signal = ToolSignals::from_request_with_semantics(request, None, &semantics);
assert_eq!(signal.write_count, 1, "{name}");
}
}
}
#[test]
fn configured_tool_semantics_only_fold_ascii_case() {
let semantics = ToolSemantics {
observe: vec!["kb_search".to_string()],
..Default::default()
};
assert_eq!(
classify_tool_call_with_semantics("KB_SEARCH", None, &semantics),
ToolSemantic::Observe
);
// U+212A lowercases to ASCII `k` under Unicode rules, but custom names
// intentionally ignore only ASCII case.
assert_eq!(
classify_tool_call_with_semantics("KB_SEARCH", None, &semantics),
ToolSemantic::Unknown
);
}
#[test]
fn custom_semantics_preserve_builtin_unicode_lowercasing() {
let semantics = ToolSemantics {
observe: vec!["lookup_customer".to_string()],
..Default::default()
};
// This matched the built-in `notebookedit` before custom semantics existed.
assert_eq!(
classify_tool_call_with_semantics("notebooKedit", None, &semantics),
ToolSemantic::Mutate(MutationKind::Edit)
);
}
#[test]
fn configured_semantics_never_replace_builtin_classifications() {
let semantics = ToolSemantics {
observe: vec!["lookup_customer".to_string()],
mutate: vec!["send_payment".to_string()],
plan: vec!["create_workflow".to_string()],
new: vec!["send_message".to_string()],
};
for name in WRITE_TOOL_NAMES {
assert_eq!(
classify_tool_call_with_semantics(name, None, &semantics),
ToolSemantic::Mutate(MutationKind::Write),
"write tool {name:?} changed classification"
);
}
for name in EDIT_TOOL_NAMES {
assert_eq!(
classify_tool_call_with_semantics(name, None, &semantics),
ToolSemantic::Mutate(MutationKind::Edit),
"edit tool {name:?} changed classification"
);
}
for name in READ_TOOL_NAMES {
assert_eq!(
classify_tool_call_with_semantics(name, None, &semantics),
ToolSemantic::Observe,
"read tool {name:?} changed classification"
);
}
for name in PLAN_TOOL_NAMES {
assert_eq!(
classify_tool_call_with_semantics(name, None, &semantics),
ToolSemantic::Plan,
"plan tool {name:?} changed classification"
);
}
for (command, expected) in [
("cat /tmp/input", ToolSemantic::Observe),
(
"cat /tmp/input > /tmp/output",
ToolSemantic::Mutate(MutationKind::Write),
),
(
"sed -i 's/a/b/' /tmp/file",
ToolSemantic::Mutate(MutationKind::Edit),
),
("./run_tests.sh", ToolSemantic::Unknown),
] {
assert_eq!(
classify_tool_call_with_semantics("BASH", Some(command), &semantics),
expected,
"bash command {command:?} changed classification"
);
}
}
#[test]
fn configured_semantics_score_like_their_builtin_equivalents() {
let semantics = ToolSemantics {
observe: vec!["lookup_customer".to_string()],
mutate: vec!["send_payment".to_string()],
plan: vec!["create_workflow".to_string()],
..Default::default()
};
for (builtin, configured) in [
("Read", "lookup_customer"),
("Write", "send_payment"),
("TodoWrite", "create_workflow"),
] {
let messages_before_tool = || {
vec![
Message::text(Role::User, "start"),
Message::text(Role::Assistant, "working"),
Message::text(Role::User, "continue"),
Message::text(Role::Assistant, "working"),
Message::text(Role::User, "continue"),
Message::text(Role::Assistant, "working"),
Message::text(Role::User, "continue"),
]
};
let mut builtin_messages = messages_before_tool();
builtin_messages.push(tc(builtin));
let mut configured_messages = messages_before_tool();
configured_messages.push(tc(configured));
let builtin_score = score_signal(&ToolSignals::from_request(
&with_messages(builtin_messages),
None,
));
let configured_score = score_signal(&ToolSignals::from_request_with_semantics(
&with_messages(configured_messages),
None,
&semantics,
));
assert_ne!(
builtin_score.score, 0.0,
"the {builtin:?} control must exercise a scoring dimension"
);
assert_eq!(
configured_score, builtin_score,
"configured tool {configured:?} must score exactly like {builtin:?}"
);
}
}
#[test]
fn tool_semantics_reject_duplicates_and_builtin_reclassification() {
let duplicate = ToolSemantics {
observe: vec!["lookup".to_string()],
mutate: vec!["LOOKUP".to_string()],
..Default::default()
};
assert!(
duplicate
.validate()
.expect_err("duplicate should fail")
.to_string()
.contains("appears in both")
);
let builtin = ToolSemantics {
new: vec!["write_file".to_string()],
..Default::default()
};
assert!(
builtin
.validate()
.expect_err("built-in should fail")
.to_string()
.contains("built-in semantics")
);
let empty = ToolSemantics {
observe: vec![" \t".to_string()],
..Default::default()
};
assert!(
empty
.validate()
.expect_err("empty name should fail")
.to_string()
.contains("empty tool name")
);
}
}