Craft Anything: understand how any software works, and craft your own.
Rust
Retry builder turns from typed failures, including stateless backends (#552)
Afloatf307d39480ec
diff --git a/crates/ai/src/agent.rs b/crates/ai/src/agent.rsindex f9a45d9b..d732a488 100644--- a/crates/ai/src/agent.rs+++ b/crates/ai/src/agent.rs@@ -8,7 +8,7 @@ use crate::context;use crate::model::{Block, ChatModel, Delta, Effort, Msg, Speaker, Stop, StopReason, ToolDef, TurnRequest, Usage};use crate::price::{self, Meter};use crate::prompt;-use crate::{AgentEvent, Conversation};+use crate::{AgentEvent, Conversation, TurnFailure};use craft_anything_engine::{CallOptions, Caller, CancelToken, Consent, Engine, Registry, ToolOutput, Toolset};use craft_anything_evidence::{ErrorCode, ToolError};use craft_anything_platform::sync::Mutex;@@ -276,6 +276,7 @@ pub struct AgentLoop {last_input: u64,/// Estimated history size sent with that turn, before its reply and tool results were added.last_history_estimate: u64,+ failure: Arc<Mutex<Option<TurnFailure>>>,}impl AgentLoop {@@ -300,16 +301,26 @@ impl AgentLoop {/// Handle one user message to completion.pub fn handle(&mut self, text: &str) {+ self.handle_turn(Some(text));+ }++ /// A retry has no new user message: keep completed tool calls and their results, then+ /// continue the failed model request using the same history.+ fn handle_turn(&mut self, text: Option<&str>) {self.stop.reset();let cancel = CancelToken::default();*self.cancel.lock().unwrap_or_else(PoisonError::into_inner) = cancel.clone();- self.host.journal("user_message", json!({"text": crate::journal_text(text)}));+ if let Some(text) = text {+ self.host.journal("user_message", json!({"text": crate::journal_text(text)}));+ }if self.system.is_none() {self.emit(AgentEvent::Status { text: "Reading the investigation so far…".into() });let inv = self.host.investigation();self.system = Some(prompt::system_prompt(self.cfg.beginner, self.host.clean_room(), &inv));}- context::push_user_text(&mut self.history, text);+ if let Some(text) = text {+ context::push_user_text(&mut self.history, text);+ }let configured = self.chain.get(self.current).and_then(|s| s.effort);let effort = prompt::choose_effort(configured);let max_turns = self.chain.get(self.current).map(|s| s.max_turns).unwrap_or(DEFAULT_MAX_TURNS).max(1);@@ -564,6 +575,7 @@ impl AgentLoop {self.emit(AgentEvent::Status { text: format!("{name} failed ({}). Trying {next}…", e.message) });continue;}+ *self.failure.lock().unwrap_or_else(PoisonError::into_inner) = Some(TurnFailure::from_api(&e));self.emit(AgentEvent::Error { message: format!("{name}: {}", e.message), remediation: e.remediation, log: None });return None;}@@ -685,7 +697,7 @@ fn wait_answer(rx: &Receiver<bool>, stop: &Stop) -> Option<bool> {/// A conversation with an API backend: a worker thread runs [`AgentLoop`].pub struct ApiConversation {- to_worker: Sender<String>,+ to_worker: Sender<Option<String>>,events: Receiver<AgentEvent>,busy: Arc<AtomicBool>,stop: Stop,@@ -693,6 +705,7 @@ pub struct ApiConversation {waiters: Waiters,queued: Vec<AgentEvent>,resumed: bool,+ failure: Arc<Mutex<Option<TurnFailure>>>,}impl ApiConversation {@@ -706,11 +719,12 @@ impl ApiConversation {return Err(ToolError::unavailable("No AI backend could be started.").with_remediation("Add one in Settings → AI."));}let (etx, erx) = std::sync::mpsc::channel();- let (wtx, wrx) = std::sync::mpsc::channel::<String>();+ let (wtx, wrx) = std::sync::mpsc::channel::<Option<String>>();let busy = Arc::new(AtomicBool::new(false));let stop = Stop::default();let cancel = Arc::new(Mutex::new(CancelToken::default()));let waiters: Waiters = Arc::default();+ let failure: Arc<Mutex<Option<TurnFailure>>> = Arc::default();// A kept conversation continues where it stopped.let history = cfg.transcript.as_deref().and_then(context::load).unwrap_or_default();let resumed = !history.is_empty();@@ -729,13 +743,14 @@ impl ApiConversation {meter: Meter::default(),last_input: 0,last_history_estimate: 0,+ failure: failure.clone(),};let busy2 = busy.clone();std::thread::Builder::new().name("ai-agent".into()).spawn(move || {while let Ok(text) = wrx.recv() {- let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| agent.handle(&text)));+ let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| agent.handle_turn(text.as_deref())));if r.is_err() {let _ = etx.send(AgentEvent::Error {message: "The assistant hit a bug and stopped safely.".into(),@@ -747,19 +762,18 @@ impl ApiConversation {}}).map_err(|e| ToolError::internal(format!("couldn't start the AI worker: {e}")))?;- Ok(ApiConversation { to_worker: wtx, events: erx, busy, stop, cancel, waiters, queued: Vec::new(), resumed })+ Ok(ApiConversation { to_worker: wtx, events: erx, busy, stop, cancel, waiters, queued: Vec::new(), resumed, failure })}/// It continues a conversation kept in its transcript file.pub fn resumed(&self) -> bool {self.resumed}-}-impl Conversation for ApiConversation {- fn send(&mut self, text: &str) {+ fn submit(&mut self, text: Option<String>) {+ *self.failure.lock().unwrap_or_else(PoisonError::into_inner) = None;self.busy.store(true, Ordering::SeqCst);- if self.to_worker.send(text.to_string()).is_err() {+ if self.to_worker.send(text).is_err() {self.busy.store(false, Ordering::SeqCst);self.queued.push(AgentEvent::Error {message: "The assistant isn't running any more.".into(),@@ -768,6 +782,20 @@ impl Conversation for ApiConversation {});}}+}++impl Conversation for ApiConversation {+ fn send(&mut self, text: &str) {+ self.submit(Some(text.to_string()));+ }++ fn retry(&mut self, _text: &str) {+ self.submit(None);+ }++ fn last_failure(&self) -> Option<TurnFailure> {+ *self.failure.lock().unwrap_or_else(PoisonError::into_inner)+ }fn poll(&mut self) -> Vec<AgentEvent> {let mut v = std::mem::take(&mut self.queued);diff --git a/crates/ai/src/anthropic.rs b/crates/ai/src/anthropic.rsindex 9a6f5871..432b0262 100644--- a/crates/ai/src/anthropic.rs+++ b/crates/ai/src/anthropic.rs@@ -253,9 +253,9 @@ impl StreamState {"authentication_error" | "permission_error" => ErrorCode::NoPermission,_ => ErrorCode::ProgramFailed,};- return Err(- ToolError::new(code, format!("Anthropic stream error ({et}): {msg}")).with_remediation("Try again; if it keeps happening, switch backend.")- );+ return Err(ToolError::new(code, format!("Anthropic stream error ({et}): {msg}"))+ .with_remediation("Try again; if it keeps happening, switch backend.")+ .with_details(json!({"provider_error": et})));}_ => {}}diff --git a/crates/ai/src/cli_agents/acp.rs b/crates/ai/src/cli_agents/acp.rsindex 0b0a6f6b..ec88c2ea 100644--- a/crates/ai/src/cli_agents/acp.rs+++ b/crates/ai/src/cli_agents/acp.rs@@ -8,7 +8,7 @@//! process; [`AcpConversation`] runs it against the agent through `craft_anything_platform`.use super::{McpCommand, display_for};-use crate::{AgentEvent, Conversation};+use crate::{AgentEvent, Conversation, FailureKind, TurnFailure};use craft_anything_platform::sync::Mutex;use craft_anything_platform::{Process, RunningProcess, StdinMode};use serde_json::{Value, json};@@ -120,6 +120,7 @@ pub struct AcpCore {tools: HashMap<String, String>,/// Permission requests go to an approver ([`Out::Decide`]) instead of the person's chat.decides: bool,+ failure: Option<TurnFailure>,}impl AcpCore {@@ -138,6 +139,7 @@ impl AcpCore {asks: HashMap::new(),tools: HashMap::new(),decides: false,+ failure: None,}}@@ -171,6 +173,7 @@ impl AcpCore {/// A message from the person: sent now, or once the session is ready.pub fn prompt(&mut self, text: &str) -> Vec<Out> {+ self.failure = None;self.queued.push_back(text.to_string());// A start that failed earlier (an `initialize` error) left nothing under way: start again,// or the message would wait for a session that never comes (#271).@@ -226,6 +229,11 @@ impl AcpCore {let params = json!({"cwd": self.cwd, "mcpServers": self.mcp_servers()});return vec![self.request("session/new", params, Pending::NewSession)];}+ self.failure = Some(TurnFailure::new(match err.get("code").and_then(Value::as_i64) {+ Some(-32000) => FailureKind::Authentication,+ Some(-32602..=-32600) => FailureKind::Configuration,+ _ => FailureKind::Unknown,+ }));let fix = matches!(kind, Pending::Initialize | Pending::NewSession).then(|| format!("Check that {} is installed and signed in: run it once in a terminal.", self.agent_name));return self.fail(format!("{}: {text}", self.agent_name), fix);@@ -447,6 +455,7 @@ pub struct AcpConversation {approver: Option<Approver>,preamble: String,logs: Option<PathBuf>,+ failure: Option<TurnFailure>,}impl AcpConversation {@@ -488,6 +497,7 @@ impl AcpConversation {approver: None,preamble: String::new(),logs: None,+ failure: None,}}@@ -558,6 +568,10 @@ impl AcpConversation {}impl Conversation for AcpConversation {+ fn last_failure(&self) -> Option<TurnFailure> {+ self.failure.or_else(|| self.lock().as_ref().and_then(|s| s.core.failure))+ }+fn send(&mut self, text: &str) {if self.busy.swap(true, Ordering::SeqCst) {let _ = self.tx.send(AgentEvent::Error {@@ -567,11 +581,13 @@ impl Conversation for AcpConversation {});return;}+ self.failure = None;let running = self.proc.as_mut().is_some_and(|p| !p.is_finished());if !running {let _ = self.tx.send(AgentEvent::Status { text: format!("Starting {}…", display_for(&self.agent)) });if let Err(e) = self.spawn() {self.busy.store(false, Ordering::SeqCst);+ self.failure = Some(TurnFailure::from_process(&e));let _ = self.tx.send(AgentEvent::Error { message: e.message, remediation: e.remediation, log: None });let _ = self.tx.send(AgentEvent::TurnDone {backend: self.backend.clone(),@@ -596,6 +612,8 @@ impl Conversation for AcpConversation {// An agent that exits mid-message would otherwise leave the conversation busy forever.if self.busy.load(Ordering::SeqCst) && self.proc.as_mut().is_some_and(RunningProcess::is_finished) {self.busy.store(false, Ordering::SeqCst);+ let ready = self.lock().as_ref().is_some_and(|s| s.core.ready);+ self.failure = Some(TurnFailure::new(if ready { FailureKind::Transient } else { FailureKind::Unknown }));let tail = self.lock().as_ref().map(|s| s.stderr.iter().cloned().collect::<Vec<_>>().join("\n")).unwrap_or_default();events.extend(self.rx.try_iter());let command = craft_anything_platform::process::printable_command(std::ffi::OsStr::new(&self.program), &self.args);@@ -818,6 +836,7 @@ mod tests {c.start();c.prompt("hi");let outs = c.line(r#"{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"Authentication required"}}"#);+ assert_eq!(c.failure, Some(TurnFailure::new(FailureKind::Authentication)));assert!(outs.contains(&Out::TurnEnded));assert!(matches!(&events(outs)[0], AgentEvent::Error { message, remediation: Some(_), .. } if message.contains("Authentication required")));let refused = sent(&c.line(r#"{"jsonrpc":"2.0","id":9,"method":"fs/read_text_file","params":{"path":"/etc/passwd"}}"#));diff --git a/crates/ai/src/cli_agents/adapters.rs b/crates/ai/src/cli_agents/adapters.rsindex 4b7033a2..01b9644d 100644--- a/crates/ai/src/cli_agents/adapters.rs+++ b/crates/ai/src/cli_agents/adapters.rs@@ -287,7 +287,39 @@ impl AgentAdapter for ClaudeCode {p.session_id = Some(id.to_string());}match s("type").as_str() {+ "system" if s("subtype") == "init" => {+ let failed = v.get("mcp_servers").and_then(Value::as_array).and_then(|servers| {+ servers.iter().find(|server| {+ server.get("name").and_then(Value::as_str) == Some(MCP_NAME)+ && matches!(server.get("status").and_then(Value::as_str), Some("failed" | "needs-auth"))+ })+ });+ if let Some(server) = failed {+ p.failure = Some(TurnFailure::new(FailureKind::ToolsUnavailable));+ let status = server.get("status").and_then(Value::as_str).unwrap_or_default();+ out.push(AgentEvent::Error {+ message: format!("Craft Anything's tools did not start in Claude Code (server status: {status})."),+ remediation: Some(+ "Check the Craft Anything CLI and the agent failure log, then start a new build attempt after fixing the tool connection.".into(),+ ),+ log: None,+ });+ }+ }"assistant" => {+ if let Some(code) = v.get("error").and_then(Value::as_str) {+ let failure = TurnFailure::claude(code);+ p.failure = Some(failure);+ let why = v.pointer("/message/content").map(content_text).unwrap_or_else(|| code.to_string());+ let (message, hint) = claude_code_failure(&why);+ let remediation = hint.or_else(|| match failure.kind {+ FailureKind::Authentication => self.sign_in_fix(),+ FailureKind::Billing => Some("Check the account's credits, plan and spending limit before trying again.".into()),+ FailureKind::Configuration => Some("Check the chosen model and Claude Code settings before trying again.".into()),+ _ => None,+ });+ return vec![AgentEvent::Error { message, remediation, log: None }];+ }for b in v.pointer("/message/content").and_then(Value::as_array).cloned().unwrap_or_default() {match b.get("type").and_then(Value::as_str).unwrap_or_default() {"text" => out.push(AgentEvent::TextDelta { text: b.get("text").and_then(Value::as_str).unwrap_or_default().to_string() }),@@ -332,6 +364,7 @@ impl AgentAdapter for ClaudeCode {log: None,});} else if v.get("is_error").and_then(Value::as_bool).unwrap_or(false) {+ p.failure.get_or_insert(TurnFailure::new(FailureKind::Unknown));let why = if s("result").is_empty() { s("subtype") } else { s("result") };let (message, remediation) = claude_code_failure(&why);out.push(AgentEvent::Error { message, remediation, log: None });diff --git a/crates/ai/src/cli_agents/adapters/opencode.rs b/crates/ai/src/cli_agents/adapters/opencode.rsindex f5d33ef0..68a9a4ef 100644--- a/crates/ai/src/cli_agents/adapters/opencode.rs+++ b/crates/ai/src/cli_agents/adapters/opencode.rs@@ -21,13 +21,107 @@ impl AgentAdapter for OpenCode {a.extend(["--model".into(), m.clone()]);}a.extend(cx.options.args.iter().cloned());+ a.extend(["--format".into(), "json".into()]);a.push(format!("{}{}", cx.preamble, cx.prompt));a}- fn streams_json(&self) -> bool {+ fn required_flags(&self) -> (&'static [&'static str], &'static [&'static str]) {+ (&["run", "--help"], &["--format"])+ }++ fn plain_fallback(&self) -> bool {false}++ fn parse(&self, p: &mut StreamParser, v: &Value) -> Vec<AgentEvent> {+ let mut events = Vec::new();+ let part = v.get("part").unwrap_or(&Value::Null);+ match v.get("type").and_then(Value::as_str) {+ Some("text") => {+ if let Some(text) = part.get("text").and_then(Value::as_str) {+ events.push(AgentEvent::TextDelta { text: text.into() });+ }+ }+ Some("reasoning") => {+ if let Some(text) = part.get("text").and_then(Value::as_str) {+ events.push(AgentEvent::Status { text: summary_of(text) });+ }+ }+ Some("tool_use") => {+ if let (Some(id), Some(name), Some(state)) =+ (part.get("callID").and_then(Value::as_str), part.get("tool").and_then(Value::as_str), part.get("state"))+ {+ if !p.tool_names.contains_key(id) {+ events.push(p.started(id, name, state.get("input").cloned().unwrap_or_else(|| json!({}))));+ }+ match state.get("status").and_then(Value::as_str) {+ Some("completed") => events.push(p.finished(id, true, &content_text(state.get("output").unwrap_or(&Value::Null)))),+ Some("error") => events.push(p.finished(id, false, &content_text(state.get("error").unwrap_or(&Value::Null)))),+ _ => {}+ }+ }+ }+ Some("step_finish") => {+ let tokens = part.get("tokens").unwrap_or(&Value::Null);+ let count = |path| tokens.pointer(path).and_then(Value::as_u64).unwrap_or(0);+ // OpenCode's step-finish counters exclude cache from input and reasoning from+ // output, so these components make the inclusive totals the feed records.+ p.input_tokens = p.input_tokens.saturating_add(count("/input")).saturating_add(count("/cache/read")).saturating_add(count("/cache/write"));+ p.output_tokens = p.output_tokens.saturating_add(count("/output")).saturating_add(count("/reasoning"));+ if !p.cost_incomplete {+ p.cost_usd = part.get("cost").and_then(Value::as_f64).filter(|n| n.is_finite() && *n >= 0.0).and_then(|cost| {+ let total = p.cost_usd.unwrap_or(0.0) + cost;+ total.is_finite().then_some(total)+ });+ p.cost_incomplete = p.cost_usd.is_none();+ }+ }+ Some("error") => {+ let error = v.get("error").unwrap_or(&Value::Null);+ let data = error.get("data").unwrap_or(&Value::Null);+ let why = data.get("message").and_then(Value::as_str).unwrap_or("the service returned no error details");+ p.failure = Some(opencode_failure(error));+ p.saw_result = true;+ events.push(AgentEvent::Error { message: format!("OpenCode stopped: {why}"), remediation: None, log: None });+ }+ _ => {}+ }+ events+ }+}++/// OpenCode's JSON error carries the provider status and its own retry decision. In particular,+/// an unclassified exit or an unsupported CLI flag cannot become retryable from display text.+fn opencode_failure(error: &Value) -> TurnFailure {+ let data = error.get("data").unwrap_or(&Value::Null);+ match error.get("name").and_then(Value::as_str) {+ Some("ProviderAuthError") => TurnFailure::new(FailureKind::Authentication),+ Some("APIError") => {+ let status = data.get("statusCode").and_then(Value::as_u64).and_then(|n| u16::try_from(n).ok());+ let mut failure = status.map_or_else(+ || TurnFailure::new(FailureKind::Unknown),+ |status| {+ TurnFailure::from_api(&crate::http::status_error("OpenCode", status, data.get("responseBody").and_then(Value::as_str).unwrap_or_default()))+ },+ );+ if failure.kind == FailureKind::Unknown && data.get("isRetryable").and_then(Value::as_bool) == Some(true) {+ failure.kind = FailureKind::Transient;+ } else if failure.retryable() && data.get("isRetryable").and_then(Value::as_bool) == Some(false) {+ failure.kind = FailureKind::Unknown;+ }+ failure.retry_after_secs = data.get("responseHeaders").and_then(Value::as_object).and_then(|headers| {+ headers+ .iter()+ .find(|(name, _)| name.eq_ignore_ascii_case("retry-after"))+ .and_then(|(_, value)| value.as_str())+ .and_then(|n| n.trim().parse::<u64>().ok())+ .map(|n| n.clamp(1, crate::failure::MAX_RETRY_WAIT))+ });+ failure+ }+ _ => TurnFailure::new(FailureKind::Unknown),+ }}#[cfg(test)]@@ -41,10 +135,87 @@ mod tests {#[test]fn the_prompt_goes_to_run_after_the_flags() {- assert_eq!(args(false, &BackendOptions::default()), ["run", "PRE make a game"]);+ assert_eq!(args(false, &BackendOptions::default()), ["run", "--format", "json", "PRE make a game"]);let options =BackendOptions { cli_model: Some("anthropic/claude-sonnet-4-5".into()), args: vec!["--agent".into(), "build".into()], ..Default::default() };- assert_eq!(args(true, &options), ["run", "--auto", "--model", "anthropic/claude-sonnet-4-5", "--agent", "build", "PRE make a game"]);+ assert_eq!(+ args(true, &options),+ ["run", "--auto", "--model", "anthropic/claude-sonnet-4-5", "--agent", "build", "--format", "json", "PRE make a game"]+ );assert!(AgentAdapters::built_in().for_agent("opencode").ids().contains(&"opencode"), "registered, not the plain adapter");}++ #[test]+ fn structured_errors_select_retry_without_reading_english() {+ let error = |status, retryable| json!({"name":"APIError","data":{"message":"服务限流。Please retry in 19.05s.","statusCode":status,"isRetryable":retryable,"responseHeaders":{"Retry-After":"4"}}});+ let rate = opencode_failure(&error(429, true));+ assert_eq!(rate, TurnFailure { kind: FailureKind::RateLimited, retry_after_secs: Some(4) });+ for (status, kind) in+ [(401, FailureKind::Authentication), (402, FailureKind::Billing), (400, FailureKind::Configuration), (503, FailureKind::Transient)]+ {+ assert_eq!(opencode_failure(&error(status, true)).kind, kind);+ }+ assert!(!opencode_failure(&error(429, false)).retryable(), "an explicit nonretryable response is kept");+ assert!(!opencode_failure(&json!({"name":"UnknownError","data":{"message":"Please retry in 2s"}})).retryable());+ assert_eq!(opencode_failure(&json!({"name":"ProviderAuthError","data":{"message":"重新登录"}})).kind, FailureKind::Authentication);+ assert_eq!(opencode_failure(&json!({"name":"APIError","data":{"isRetryable":true}})).kind, FailureKind::Transient);+ let quota = json!({"name":"APIError","data":{"statusCode":429,"isRetryable":true,"responseBody":"{\"error\":{\"code\":\"insufficient_quota\"}}"}});+ assert_eq!(opencode_failure("a).kind, FailureKind::Billing);+ }++ #[test]+ fn json_output_keeps_text_tools_and_usage_without_ending_the_process_turn() {+ let mut p = StreamParser::new(Arc::new(OpenCode));+ let lines = [+ json!({"type":"step_start","sessionID":"reported-but-not-resumable","part":{}}),+ json!({"type":"text","part":{"text":"Built a page"}}),+ json!({"type":"tool_use","part":{"callID":"write-1","tool":"write","state":{"status":"completed","input":{"path":"index.html"},"output":"wrote the page"}}}),+ json!({"type":"step_finish","part":{"tokens":{"input":10,"output":5,"reasoning":2,"cache":{"read":3,"write":1}},"cost":0.01}}),+ json!({"type":"step_finish","part":{"tokens":{"input":1,"output":2},"cost":0.02}}),+ ];+ let events: Vec<_> = lines.iter().flat_map(|v| p.line(&v.to_string())).collect();+ assert!(+ matches!(events.as_slice(), [AgentEvent::TextDelta { text }, AgentEvent::ToolStarted { tool, .. }, AgentEvent::ToolFinished { ok:true, .. }] if text=="Built a page" && tool=="write")+ );+ assert_eq!((p.input_tokens, p.output_tokens), (15, 9));+ assert_eq!(p.cost_usd, Some(0.03));+ assert!(p.session_id.is_none(), "the adapter does not pretend it can resume");+ let events = p.line(&json!({"type":"error","error":{"name":"APIError","data":{"message":"暂时失败","statusCode":429,"isRetryable":true}}}).to_string());+ assert!(matches!(events.as_slice(), [AgentEvent::Error { .. }]), "run_cli alone owns TurnDone");+ assert_eq!(p.failure.unwrap().kind, FailureKind::RateLimited);+ }++ #[test]+ fn missing_invalid_and_overflowed_costs_stay_unknown_for_the_turn() {+ let step = |cost| json!({"type":"step_finish","part":{"cost":cost,"tokens":{}}}).to_string();+ for invalid in [Value::Null, json!(-1), json!("unknown")] {+ for unknown_first in [true, false] {+ let mut p = StreamParser::new(Arc::new(OpenCode));+ let costs = if unknown_first { [invalid.clone(), json!(0.02)] } else { [json!(0.02), invalid.clone()] };+ for cost in costs {+ p.line(&step(cost));+ }+ p.line(&step(json!(0.01)));+ assert_eq!(p.cost_usd, None, "a later known step cannot fill an earlier gap");+ }+ }+ for unknown_first in [true, false] {+ let mut missing = StreamParser::new(Arc::new(OpenCode));+ if !unknown_first {+ missing.line(&step(json!(0.02)));+ }+ missing.line(r#"{"type":"step_finish","part":{"tokens":{}}}"#);+ missing.line(&step(json!(0.01)));+ assert_eq!(missing.cost_usd, None);+ }+ let mut overflow = StreamParser::new(Arc::new(OpenCode));+ overflow.line(&step(json!(f64::MAX)));+ overflow.line(&step(json!(f64::MAX)));+ overflow.line(&step(json!(0)));+ assert_eq!(overflow.cost_usd, None);+ let mut zero = StreamParser::new(Arc::new(OpenCode));+ zero.line(&step(json!(0)));+ zero.line(&step(json!(0)));+ assert_eq!(zero.cost_usd, Some(0.0), "an explicit zero is a known cost");+ }}diff --git a/crates/ai/src/cli_agents/mod.rs b/crates/ai/src/cli_agents/mod.rsindex 1798b3b0..8fa4c6ce 100644--- a/crates/ai/src/cli_agents/mod.rs+++ b/crates/ai/src/cli_agents/mod.rs@@ -16,7 +16,7 @@ pub use adapters::{AgentAdapter, AgentAdapters, ArgsCx};use crate::config::BackendOptions;use crate::model::Stop;-use crate::{AgentEvent, BackendConfig, Conversation};+use crate::{AgentEvent, BackendConfig, Conversation, FailureKind, TurnFailure};use craft_anything_engine::CancelToken;use craft_anything_evidence::{ErrorCode, ToolError};use craft_anything_platform::process::Process;@@ -258,10 +258,14 @@ pub struct StreamParser {pub input_tokens: u64,pub output_tokens: u64,pub cost_usd: Option<f64>,+ /// A missing or invalid per-step cost cannot become a complete turn total later.+ cost_incomplete: bool,pub saw_result: bool,/// The run was asked to continue a saved session the agent doesn't have (it lives on another/// computer, or under another folder), so it did nothing.pub session_missing: bool,+ /// A protocol fact about the failed turn, independent of the text shown to the person.+ pub failure: Option<TurnFailure>,}fn short_tool(name: &str) -> String {@@ -303,8 +307,10 @@ impl StreamParser {input_tokens: 0,output_tokens: 0,cost_usd: None,+ cost_incomplete: false,saw_result: false,session_missing: false,+ failure: None,}}@@ -452,6 +458,7 @@ pub struct CliConversation {verified: Arc<Mutex<Option<bool>>>,/// The last run's saved session wasn't there (see [`Conversation::session_lost`]).lost: Arc<AtomicBool>,+ failure: Arc<Mutex<Option<TurnFailure>>>,}impl CliConversation {@@ -460,7 +467,17 @@ impl CliConversation {pub fn new(spec: CliSpec) -> Self {let (tx, rx) = std::sync::mpsc::channel();let session = Arc::new(Mutex::new(spec.resume.clone()));- CliConversation { spec, tx, rx, busy: Arc::default(), stop: Stop::default(), session, verified: Arc::default(), lost: Arc::default() }+ CliConversation {+ spec,+ tx,+ rx,+ busy: Arc::default(),+ stop: Stop::default(),+ session,+ verified: Arc::default(),+ lost: Arc::default(),+ failure: Arc::default(),+ }}/// The agent's session id once a run has reported it (to continue later).@@ -504,6 +521,7 @@ struct RunState {session: Arc<Mutex<Option<String>>>,verified: Arc<Mutex<Option<bool>>>,lost: Arc<AtomicBool>,+ failure: Arc<Mutex<Option<TurnFailure>>>,}/// End a turn: take the next message first, then say the turn is done (`send(done)`), since@@ -525,7 +543,7 @@ fn nothing_done(backend: &str) -> AgentEvent {/// next message; one it was asked to continue but doesn't have is forgotten and flagged in/// `state.lost` before the error that says so is sent.fn run_cli(spec: &CliSpec, text: &str, tx: &Sender<AgentEvent>, stop: &Stop, state: &RunState) -> AgentEvent {- let RunState { session, verified, lost } = state;+ let RunState { session, verified, lost, failure } = state;let name = spec.config.name.clone();let agent = spec.config.model.clone();let program = spec.options.command.clone().unwrap_or_else(|| program_for(&agent));@@ -592,7 +610,9 @@ fn run_cli(spec: &CliSpec, text: &str, tx: &Sender<AgentEvent>, stop: &Stop, stalet _ =tx.send(AgentEvent::Status { text: format!("Asking {}{}…", display_for(&agent), remote.as_ref().map(|h| format!(" on {h}")).unwrap_or_default()) });let parser = Arc::new(Mutex::new(StreamParser::new(adapter.clone())));- let (p2, tx2, tx_err, lost2) = (parser.clone(), tx.clone(), tx.clone(), lost.clone());+ let (p2, tx2, tx_err, lost2, failure2) = (parser.clone(), tx.clone(), tx.clone(), lost.clone(), failure.clone());+ let failed_tools = Arc::new(AtomicBool::new(false));+ let (failed_tools2, failed_tools3) = (failed_tools.clone(), failed_tools.clone());let stop2 = stop.clone();let mut proc = (if remote.is_none() { adapter.process(&prog) } else { Process::new(&prog) }).args(&argv)@@ -603,14 +623,23 @@ fn run_cli(spec: &CliSpec, text: &str, tx: &Sender<AgentEvent>, stop: &Stop, sta}let mut proc = proc.stdout_cap(4 * 1024 * 1024)- .cancel_when(move || stop2.is_stopped())+ .cancel_when(move || stop2.is_stopped() || failed_tools2.load(Ordering::SeqCst)).on_stdout_line(move |l| {+ if failed_tools3.load(Ordering::SeqCst) {+ return;+ }let mut p = p2.lock().unwrap_or_else(PoisonError::into_inner);let events = p.line(l);// Said before the error that follows it, so whoever reads the error can tell.if p.session_missing {lost2.store(true, Ordering::SeqCst);}+ if let Some(failed) = p.failure {+ *failure2.lock().unwrap_or_else(PoisonError::into_inner) = Some(failed);+ if failed.kind == FailureKind::ToolsUnavailable {+ failed_tools3.store(true, Ordering::SeqCst);+ }+ }for e in events {let _ = tx2.send(e);}@@ -635,7 +664,26 @@ fn run_cli(spec: &CliSpec, text: &str, tx: &Sender<AgentEvent>, stop: &Stop, sta} else if let Some(id) = &p.session_id {*session.lock().unwrap_or_else(PoisonError::into_inner) = Some(id.clone());}+ if p.failure.is_none() {+ let failed = match &result {+ Ok(out) if !out.success => Some(TurnFailure::process_exit(out.code)),+ Err(e) if e.code != ErrorCode::Cancelled => Some(TurnFailure::from_process(e)),+ _ => None,+ };+ *failure.lock().unwrap_or_else(PoisonError::into_inner) = failed;+ }match result {+ // The protocol already named the required tool server. Stop the owned process without+ // replacing that useful error with the cancellation used to stop it.+ result if failed_tools.load(Ordering::SeqCst) => {+ let (out, error) = match &result {+ Ok(out) => (Some(out), None),+ Err(e) => (None, Some(e.message.as_str())),+ };+ if let Some(path) = shown.save_with(&agent, out, Some("Craft Anything's tools did not start."), error).0 {+ let _ = tx.send(AgentEvent::Status { text: format!("The details are in {}", path.display()) });+ }+ }// It said why already, and there's nothing more in its output to keep.Ok(_) if p.session_missing => {}Ok(out) if !out.success && !p.saw_result => {@@ -775,6 +823,15 @@ impl FailureContext {}impl Conversation for CliConversation {+ fn retry(&mut self, text: &str) {+ let resumable = self.spec.adapters.for_agent(&self.spec.config.model).resumes() && self.session_id().is_some();+ self.send(if resumable { crate::failure::CONTINUE_TURN } else { text });+ }++ fn last_failure(&self) -> Option<TurnFailure> {+ *self.failure.lock().unwrap_or_else(PoisonError::into_inner)+ }+fn session_id(&self) -> Option<String> {CliConversation::session_id(self)}@@ -799,8 +856,9 @@ impl Conversation for CliConversation {}self.stop.reset();self.lost.store(false, Ordering::SeqCst);+ *self.failure.lock().unwrap_or_else(PoisonError::into_inner) = None;let (spec, tx, stop, busy) = (self.spec.clone(), self.tx.clone(), self.stop.clone(), self.busy.clone());- let state = RunState { session: self.session.clone(), verified: self.verified.clone(), lost: self.lost.clone() };+ let state = RunState { session: self.session.clone(), verified: self.verified.clone(), lost: self.lost.clone(), failure: self.failure.clone() };let text = text.to_string();let spawned = std::thread::Builder::new().name("ai-cli".into()).spawn(move || {let done = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run_cli(&spec, &text, &tx, &stop, &state))).unwrap_or_else(|_| {@@ -846,6 +904,47 @@ impl Drop for CliConversation {mod tests {use super::*;+ #[test]+ fn failed_craft_tool_server_is_reported_from_its_init_status() {+ let mut p = StreamParser::new(adapter_for("claude-code"));+ let events = p.line(r#"{"type":"system","subtype":"init","session_id":"cannot-build","mcp_servers":[{"name":"craft-anything","status":"failed"}]}"#);+ assert!(+ events.iter().any(|e| matches!(e, AgentEvent::Error { message, .. } if message.contains("tools") && message.contains("start"))),+ "a failed required tool server must end the turn with an actionable error: {events:?}"+ );+ assert_eq!(p.failure, Some(TurnFailure::new(FailureKind::ToolsUnavailable)));+ }++ #[test]+ fn optional_or_pending_mcp_servers_do_not_block_the_builder() {+ for status in ["connected", "pending"] {+ let mut p = StreamParser::new(adapter_for("claude-code"));+ let init = json!({"type":"system","subtype":"init","mcp_servers":[{"name":"craft-anything","status":status},{"name":"optional-service","status":"failed"}]});+ assert!(p.line(&init.to_string()).is_empty());+ assert!(p.failure.is_none());+ }+ let mut p = StreamParser::new(adapter_for("claude-code"));+ assert!(!p.line(r#"{"type":"system","subtype":"init","mcp_servers":[{"name":"craft-anything","status":"needs-auth"}]}"#).is_empty());+ assert_eq!(p.failure, Some(TurnFailure::new(FailureKind::ToolsUnavailable)));+ }++ #[test]+ fn claude_assistant_error_codes_survive_localized_display_messages() {+ for (code, kind) in [+ ("authentication_failed", FailureKind::Authentication),+ ("billing_error", FailureKind::Billing),+ ("rate_limit", FailureKind::RateLimited),+ ("server_error", FailureKind::Transient),+ ] {+ let mut p = StreamParser::new(adapter_for("claude-code"));+ let events = p.line(&json!({"type":"assistant","error":code,"message":{"content":[{"type":"text","text":"服务暂时不可用"}]}}).to_string());+ assert!(matches!(events.as_slice(), [AgentEvent::Error { .. }]));+ assert_eq!(p.failure, Some(TurnFailure::new(kind)));+ p.line(r#"{"type":"result","is_error":true,"result":"the turn ended"}"#);+ assert_eq!(p.failure, Some(TurnFailure::new(kind)), "result must keep the more specific code");+ }+ }+/// #506: under a Flatpak `~/.claude` is read-only, so Claude Code gets a folder of ours to save/// its conversations in (so `--resume` finds them); the person's own `CLAUDE_CONFIG_DIR` and a/// normal install are left as they are, and the sign-in hint says how to sign in there.diff --git a/crates/ai/src/failure.rs b/crates/ai/src/failure.rsnew file mode 100644index 00000000..bb94e99f--- /dev/null+++ b/crates/ai/src/failure.rs@@ -0,0 +1,175 @@+//! A failed turn's retry policy, kept apart from the words shown to the person. Protocol and+//! process boundaries supply these facts; display text never decides whether to retry.++use craft_anything_evidence::{ErrorCode, ToolError};++/// Why a conversation's last turn failed. Unknown failures need attention: repeating an+/// unclassified setup error can spend quota without doing any work.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum FailureKind {+ Authentication,+ Billing,+ MissingProgram,+ ToolsUnavailable,+ Configuration,+ Transient,+ RateLimited,+ Unknown,+}++/// A typed failure, and a provider's optional wait in seconds.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct TurnFailure {+ pub kind: FailureKind,+ pub retry_after_secs: Option<u64>,+}++/// No failed turn waits more than fifteen minutes before its next attempt.+pub(crate) const MAX_RETRY_WAIT: u64 = 15 * 60;+pub(crate) const CONTINUE_TURN: &str =+ "Your last turn was cut off. Carry on from where you stopped: look at what's already in app/ and your checklist, then finish the task.";++impl TurnFailure {+ pub const fn new(kind: FailureKind) -> Self {+ Self { kind, retry_after_secs: None }+ }++ pub fn retryable(self) -> bool {+ matches!(self.kind, FailureKind::Transient | FailureKind::RateLimited)+ }++ pub(crate) fn from_api(error: &ToolError) -> Self {+ let status = error.details.as_ref().and_then(|d| d.get("status")).and_then(serde_json::Value::as_u64);+ let provider = error.details.as_ref().and_then(|d| d.get("provider_error")).and_then(serde_json::Value::as_str);+ let kind = match (status, provider) {+ (Some(401 | 403), _) | (_, Some("authentication_error" | "permission_error" | "invalid_api_key")) => FailureKind::Authentication,+ (Some(402), _) | (_, Some("billing_error" | "insufficient_quota")) => FailureKind::Billing,+ (Some(429), _) | (_, Some("rate_limit_error" | "rate_limit_exceeded")) => FailureKind::RateLimited,+ (Some(408 | 500..=599), _) | (_, Some("overloaded_error" | "api_error")) => FailureKind::Transient,+ (Some(400..=499), _) => FailureKind::Configuration,+ (_, Some(_)) => FailureKind::Unknown,+ _ => match error.code {+ ErrorCode::TimedOut | ErrorCode::EngineMissing | ErrorCode::Truncated => FailureKind::Transient,+ ErrorCode::NoPermission => FailureKind::Authentication,+ ErrorCode::BadInput | ErrorCode::BadSettings | ErrorCode::NotSupported | ErrorCode::FeatureMissing => FailureKind::Configuration,+ _ => FailureKind::Unknown,+ },+ };+ let retry_after_secs =+ error.details.as_ref().and_then(|d| d.get("retry_after_secs")).and_then(serde_json::Value::as_u64).map(|n| n.clamp(1, MAX_RETRY_WAIT));+ Self { kind, retry_after_secs }+ }++ pub(crate) fn from_process(error: &ToolError) -> Self {+ Self::new(match error.code {+ ErrorCode::EngineMissing | ErrorCode::NotFound => FailureKind::MissingProgram,+ ErrorCode::TimedOut => FailureKind::Transient,+ ErrorCode::NoPermission | ErrorCode::BadSettings | ErrorCode::BadInput => FailureKind::Configuration,+ _ => FailureKind::Unknown,+ })+ }++ /// Claude Code's documented `assistant.error` values, not the assistant's display text.+ pub(crate) fn claude(code: &str) -> Self {+ Self::new(match code {+ "authentication_failed" | "oauth_org_not_allowed" | "cloud_credential_error" => FailureKind::Authentication,+ "billing_error" | "account_on_hold" => FailureKind::Billing,+ "invalid_request" | "model_not_found" => FailureKind::Configuration,+ "rate_limit" => FailureKind::RateLimited,+ "server_error" | "overloaded" => FailureKind::Transient,+ _ => FailureKind::Unknown,+ })+ }++ /// Without a protocol error, a process may have crashed or lost its connection. Signals+ /// and nonzero exits keep the same bounded retry policy; a successful exit never opts in.+ pub(crate) fn process_exit(code: Option<i32>) -> Self {+ Self::new(if code == Some(0) { FailureKind::Unknown } else { FailureKind::Transient })+ }+}++/// Compatibility for a provider's display-only wait hint, after a typed status already made+/// the failure retryable. This never decides whether to retry. The number is bounded even when+/// an error asks for hours, or sends infinity or a negative delay.+pub(crate) fn retry_after(message: &str) -> Option<u64> {+ let m = message.to_ascii_lowercase();+ let tail = ["retry in ", "retry after ", "try again in "].iter().find_map(|s| m.find(s).and_then(|at| m.get(at + s.len()..)))?;+ let number: String = tail.chars().take_while(|c| c.is_ascii_digit() || *c == '.').collect();+ let n: f64 = number.parse().ok()?;+ if !n.is_finite() || n < 0.0 {+ return None;+ }+ let unit = tail.get(number.len()..)?.trim_start();+ let scale = if unit.starts_with("ms") || unit.starts_with("millisecond") {+ 0.001+ } else if unit.starts_with("min") || unit.starts_with('m') {+ 60.0+ } else if unit.starts_with('h') {+ 3600.0+ } else {+ 1.0+ };+ Some((n * scale).ceil().clamp(1.0, MAX_RETRY_WAIT as f64) as u64)+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn typed_failures_do_not_depend_on_their_display_language() {+ for message in ["请更新凭证", "The provider is unavailable", "anything"] {+ let auth = ToolError::new(ErrorCode::NoPermission, message).with_details(json!({"status":401}));+ assert_eq!(TurnFailure::from_api(&auth).kind, FailureKind::Authentication);+ assert!(!TurnFailure::from_api(&auth).retryable());+ let bill = ToolError::new(ErrorCode::BadInput, message).with_details(json!({"status":402}));+ assert_eq!(TurnFailure::from_api(&bill).kind, FailureKind::Billing);+ let absent = ToolError::new(ErrorCode::EngineMissing, message);+ assert_eq!(TurnFailure::from_process(&absent).kind, FailureKind::MissingProgram);+ assert!(!TurnFailure::from_process(&absent).retryable());+ }+ let transient = ToolError::new(ErrorCode::TimedOut, "The authentication service is restarting; billing and login are temporarily unavailable.");+ assert!(TurnFailure::from_api(&transient).retryable(), "English words don't override a typed timeout");+ let limit = ToolError::new(ErrorCode::OverLimit, "provider response").with_details(json!({"status":429,"retry_after_secs":2}));+ assert_eq!(TurnFailure::from_api(&limit), TurnFailure { kind: FailureKind::RateLimited, retry_after_secs: Some(2) });+ let quota = crate::http::status_error("Provider", 429, r#"{"error":{"code":"insufficient_quota","message":"余额不足"}}"#);+ assert_eq!(TurnFailure::from_api("a).kind, FailureKind::Billing);+ assert!(!TurnFailure::from_api(&ToolError::new(ErrorCode::OverLimit, "oversized input")).retryable(), "a local size limit is not rate limiting");+ }++ #[test]+ fn streamed_provider_errors_keep_their_type() {+ for (provider, kind) in [+ ("rate_limit_exceeded", FailureKind::RateLimited),+ ("invalid_api_key", FailureKind::Authentication),+ ("insufficient_quota", FailureKind::Billing),+ ("unknown", FailureKind::Unknown),+ ] {+ let mut stream = crate::openai::ChatStream::default();+ let event = crate::http::SseEvent { event: None, data: json!({"error":{"code":provider,"message":"服务返回错误"}}).to_string() };+ let error = stream.apply(&event, &mut |_| {}).unwrap_err();+ assert_eq!(TurnFailure::from_api(&error).kind, kind);+ }+ }++ #[test]+ fn structured_claude_errors_and_plain_waits_have_bounded_retry_advice() {+ assert!(!TurnFailure::claude("billing_error").retryable());+ assert!(!TurnFailure::claude("authentication_failed").retryable());+ assert!(TurnFailure::claude("server_error").retryable());+ assert!(TurnFailure::claude("rate_limit").retryable());+ assert_eq!(retry_after("Please retry in 19.052906498s."), Some(20));+ assert_eq!(retry_after("Rate limit reached. Retry after 2 minutes"), Some(120));+ assert_eq!(retry_after("try again in 500ms"), Some(1));+ assert_eq!(retry_after("Retry in 45 seconds"), Some(45));+ for code in [None, Some(1), Some(2), Some(255)] {+ assert!(TurnFailure::process_exit(code).retryable());+ }+ assert!(!TurnFailure::process_exit(Some(0)).retryable());+ assert_eq!(retry_after("retry after 3 h"), Some(MAX_RETRY_WAIT));+ for text in ["retry after NaN", "retry after inf", "retry after -1s", "invalid command", "please retry in a moment", "retry in ."] {+ assert_eq!(retry_after(text), None);+ }+ }+}diff --git a/crates/ai/src/http.rs b/crates/ai/src/http.rsindex b5d57db0..87caafc9 100644--- a/crates/ai/src/http.rs+++ b/crates/ai/src/http.rs@@ -55,7 +55,12 @@ pub fn status_error(what: &str, status: u16, body: &str) -> ToolError {500..=599 => (ErrorCode::EngineMissing, "The service is having trouble. Try again soon, or use another backend."),_ => (ErrorCode::BadInput, "The request was rejected; see the message."),};- ToolError::new(code, format!("{what} answered {status}: {detail}")).with_remediation(remediation).with_details(serde_json::json!({"status": status}))+ let provider = serde_json::from_str::<Value>(body)+ .ok()+ .and_then(|v| v.pointer("/error/code").and_then(Value::as_str).or_else(|| v.pointer("/error/type").and_then(Value::as_str)).map(str::to_string));+ ToolError::new(code, format!("{what} answered {status}: {detail}"))+ .with_remediation(remediation)+ .with_details(serde_json::json!({"status": status, "provider_error": provider}))}/// Read only the human-readable fields, never serialise an arbitrary provider payload.@@ -112,8 +117,13 @@ pub fn post_json(agent: &ureq::Agent, what: &str, url: &str, headers: &[(&str, Slet resp = req.send(body.to_string()).map_err(|e| net_error(what, e))?;let status = resp.status().as_u16();if !(200..300).contains(&status) {+ let retry_after = resp.headers().get("retry-after").and_then(|h| h.to_str().ok()).and_then(|s| s.trim().parse::<u64>().ok());let text = resp.into_body().read_to_string().unwrap_or_default();- return Err(status_error(what, status, &text));+ let mut error = status_error(what, status, &text);+ if let (Some(seconds), Some(Value::Object(details))) = (retry_after, error.details.as_mut()) {+ details.insert("retry_after_secs".into(), Value::from(seconds.min(crate::failure::MAX_RETRY_WAIT)));+ }+ return Err(error);}Ok(Box::new(resp.into_body().into_reader()))}@@ -337,4 +347,39 @@ mod tests {assert!(e.message.contains("invalid x-api-key"));assert_eq!(status_error("x", 429, "slow down").code, ErrorCode::OverLimit);}++ #[test]+ fn a_post_keeps_a_structured_retry_after_header() {+ use std::io::Write;+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let url = format!("http://{}", listener.local_addr().unwrap());+ let server = std::thread::spawn(move || {+ let (mut stream, _) = listener.accept().unwrap();+ stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap();+ let mut reader = BufReader::new(&mut stream);+ let mut len = 0;+ loop {+ let mut line = String::new();+ assert!(reader.read_line(&mut line).unwrap() > 0);+ if line == "\r\n" {+ break;+ }+ if let Some((name, value)) = line.split_once(':')+ && name.eq_ignore_ascii_case("content-length")+ {+ len = value.trim().parse::<usize>().unwrap();+ }+ }+ reader.read_exact(&mut vec![0; len]).unwrap();+ let body = r#"{"error":{"type":"rate_limit_error","message":"服务限流"}}"#;+ write!(stream, "HTTP/1.1 429 Too Many Requests\r\nRetry-After: 7\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).unwrap();+ });+ let result = post_json(&quick_agent(Duration::from_secs(5)), "Stand-in", &url, &[], &serde_json::json!({}));+ let error = match result {+ Err(error) => error,+ Ok(_) => panic!("a rate limit is not a response stream"),+ };+ assert_eq!(crate::TurnFailure::from_api(&error), crate::TurnFailure { kind: crate::FailureKind::RateLimited, retry_after_secs: Some(7) });+ server.join().unwrap();+ }}diff --git a/crates/ai/src/lib.rs b/crates/ai/src/lib.rsindex d53dffb5..d58c496d 100644--- a/crates/ai/src/lib.rs+++ b/crates/ai/src/lib.rs@@ -12,6 +12,7 @@ pub mod cli_agents;pub mod config;pub mod context;pub mod detect;+mod failure;pub mod failure_log;pub mod http;pub mod key_tools;@@ -24,6 +25,7 @@ pub mod rebuild;pub mod ssh;pub use config::{AiConfig, BackendEntry, BackendOptions, Price};+pub use failure::{FailureKind, TurnFailure};pub use keys::KeyStore;pub use model::Effort;@@ -213,6 +215,17 @@ pub enum AgentEvent {/// A chat with the agent about the current session.pub trait Conversation: Send {fn send(&mut self, text: &str);+ /// Retry an interrupted turn. Stateless adapters resend `text`; a conversation that keeps+ /// its own history can continue it without appending the user's message again.+ fn retry(&mut self, text: &str) {+ let prompt = if self.session_id().is_some() { failure::CONTINUE_TURN } else { text };+ self.send(prompt);+ }+ /// Why the last turn failed, supplied by its protocol or process boundary. An unknown+ /// failure is never retried automatically. Cleared when the next turn starts.+ fn last_failure(&self) -> Option<TurnFailure> {+ None+ }/// A saved session to continue later, for agents that have one (agent CLIs).fn session_id(&self) -> Option<String> {Nonediff --git a/crates/ai/src/openai.rs b/crates/ai/src/openai.rsindex 50bdd24f..3313e78a 100644--- a/crates/ai/src/openai.rs+++ b/crates/ai/src/openai.rs@@ -235,7 +235,11 @@ impl ChatStream {serde_json::from_str(data).map_err(|e| ToolError::new(ErrorCode::GarbledOutput, format!("The server sent a chunk we couldn't read: {e}")))?;if let Some(err) = v.get("error") {let msg = err.get("message").and_then(Value::as_str).unwrap_or("unknown error");- return Err(ToolError::new(ErrorCode::EngineMissing, format!("Stream error: {msg}")));+ let provider = err.get("code").and_then(Value::as_str).or_else(|| err.get("type").and_then(Value::as_str));+ let status = err.get("code").and_then(Value::as_u64);+ return Err(+ ToolError::new(ErrorCode::ProgramFailed, format!("Stream error: {msg}")).with_details(json!({"provider_error": provider, "status": status}))+ );}if let Some(m) = v.get("model").and_then(Value::as_str) {self.model = m.to_string();diff --git a/crates/ai/src/rebuild/mod.rs b/crates/ai/src/rebuild/mod.rsindex b1b6ef91..45cc1947 100644--- a/crates/ai/src/rebuild/mod.rs+++ b/crates/ai/src/rebuild/mod.rs@@ -848,11 +848,29 @@ impl Worker {/// `fresh`: what to send instead when the agent's saved session turns out not to be on this/// computer.- fn turn(&mut self, ws: &mut Workspace, agent: &mut dyn Conversation, prompt: &str, mut fresh: Option<String>) -> Result<String, ToolError> {+ fn turn(&mut self, ws: &mut Workspace, agent: &mut dyn Conversation, prompt: &str, fresh: Option<String>) -> Result<String, ToolError> {+ self.turn_with_wait(ws, agent, prompt, fresh, |stop, seconds| {+ for _ in 0..seconds {+ stop.check()?;+ std::thread::sleep(Duration::from_secs(1));+ }+ stop.check()+ })+ }++ fn turn_with_wait(+ &mut self,+ ws: &mut Workspace,+ agent: &mut dyn Conversation,+ prompt: &str,+ mut fresh: Option<String>,+ mut wait_for: impl FnMut(&CancelToken, u64) -> Result<(), ToolError>,+ ) -> Result<String, ToolError> {let mut prompt = prompt.to_string();let mut tries = 0u64;+ let mut retry = false;loop {- let result = self.converse(ws, agent, &prompt);+ let result = self.converse_turn(ws, agent, &prompt, retry);let lost = result.is_err() && agent.session_lost();if lost {// The id is for a session this computer doesn't have: never ask for it again.@@ -871,30 +889,31 @@ impl Worker {"The builder's earlier conversation isn't on this computer. Starting a new one from what the project already has…",)?;prompt = f;+ retry = false;}- Err(e) if e.code != ErrorCode::Cancelled && tries < 3 && agent.session_id().is_some() && worth_retrying(&e.message) => {+ // A missing session can only start fresh above, never resume. Only backend+ // Error events become ProgramFailed here; a simultaneous backend failure must+ // not make an inbox or workspace write failure retryable.+ Err(e) if !lost && e.code == ErrorCode::ProgramFailed && tries < 3 && agent.last_failure().is_some_and(crate::TurnFailure::retryable) => {tries += 1;let why = e.message.chars().take(160).collect::<String>();- // A service that says how long to wait is waited for; otherwise a dropped- // connection or a busy service gets a moment, longer each time.- let wait = match retry_after(&e.message) {+ // Typed status decides whether to retry. A display-only wait remains a+ // compatibility hint, and never overrides a structured provider delay.+ let delay = agent.last_failure().and_then(|f| f.retry_after_secs).or_else(|| crate::failure::retry_after(&e.message));+ let seconds = match delay {Some(secs) => {let note = format!("The AI service is busy or its rate limit was reached, and asked to wait {secs} s. Trying again then…");self.step(ws, Step::Build, Status::Building, ¬e)?;- secs + 1+ secs.saturating_add(1).min(crate::failure::MAX_RETRY_WAIT)}None => {self.step(ws, Step::Build, Status::Building, &format!("The builder stopped ({why}). Picking up where it left off…"))?;30 * tries}};- for _ in 0..wait {- self.stop.check()?;- std::thread::sleep(Duration::from_secs(1));- }- prompt = format!(- "Your last turn was cut off ({why}). Carry on from where you stopped: look at what's already in app/ and your checklist, then finish the task."- );+ wait_for(&self.stop, seconds)?;+ self.stop.check()?;+ retry = true;}r => return r,}@@ -904,7 +923,15 @@ impl Worker {/// Send one message and pump the agent until its turn ends, relaying events and inbox/// requests. Returns the agent's reply text.fn converse(&mut self, ws: &mut Workspace, agent: &mut dyn Conversation, text: &str) -> Result<String, ToolError> {- agent.send(text);+ self.converse_turn(ws, agent, text, false)+ }++ fn converse_turn(&mut self, ws: &mut Workspace, agent: &mut dyn Conversation, text: &str, retry: bool) -> Result<String, ToolError> {+ if retry {+ agent.retry(text);+ } else {+ agent.send(text);+ }let mut reply = String::new();let mut failed = None;loop {@@ -1028,36 +1055,3 @@ pub fn builder_backend(cfg: &crate::AiConfig, named: Option<&str>) -> Result<cra.with_remediation("Install a coding agent or add an API key in Settings → AI, then try again.")})}--/// Whether a failed builder turn is worth resuming, rather than needing the person (sign-in,-/// credit, a missing, broken or too old install).-fn worth_retrying(message: &str) -> bool {- let m = message.to_ascii_lowercase();- !["sign in", "log in", "login", "logged in", "auth", "credit", "billing", "usage limit", "not installed", "install ", crate::cli_agents::TOO_OLD_FOR_MODEL]- .iter()- .any(|k| m.contains(k))-}--/// The longest wait a service's "retry in …" is followed for.-const MAX_RETRY_WAIT: u64 = 15 * 60;--/// How long a busy or rate-limited service asked to wait ("Please retry in 19.05s", "retry after-/// 2 minutes"), in whole seconds rounded up and at most [`MAX_RETRY_WAIT`] (#142).-fn retry_after(message: &str) -> Option<u64> {- let m = message.to_ascii_lowercase();- let at = ["retry in ", "retry after ", "try again in "].iter().find_map(|p| m.find(p).map(|i| i + p.len()))?;- let rest = m.get(at..)?;- let number: String = rest.chars().take_while(|c| c.is_ascii_digit() || *c == '.').collect();- let value: f64 = number.parse().ok()?;- let unit = rest.get(number.len()..).unwrap_or_default().trim_start();- let seconds = if unit.starts_with("ms") || unit.starts_with("millisecond") {- value / 1000.0- } else if unit.starts_with("min") || (unit.starts_with('m') && !unit.starts_with("ms")) {- value * 60.0- } else if unit.starts_with('h') {- value * 3600.0- } else {- value- };- Some((seconds.ceil().max(1.0) as u64).min(MAX_RETRY_WAIT))-}diff --git a/crates/ai/src/rebuild/tests.rs b/crates/ai/src/rebuild/tests.rsindex 73aefe31..f22b23bf 100644--- a/crates/ai/src/rebuild/tests.rs+++ b/crates/ai/src/rebuild/tests.rs@@ -323,35 +323,432 @@ fn pictures_of_the_original_reach_the_builder() {assert!(pictures_reminder(&ws).contains("1 picture(s)"));}-#[test]-fn a_cut_off_turn_is_resumed_unless_the_person_must_act() {- for resumable in ["Claude Code took longer than 86400 s and was stopped.", "API Error: 529 overloaded", "connection reset by peer"] {- assert!(worth_retrying(resumable), "{resumable}");+mod typed_retries {+ use super::*;+ use crate::{FailureKind, TurnFailure};+ use std::collections::VecDeque;++ struct FailingConversation {+ failures: VecDeque<Option<TurnFailure>>,+ failure: Option<TurnFailure>,+ queued: Vec<AgentEvent>,+ sent: Vec<String>,+ session: Option<String>,+ message: &'static str,}- for needs_you in [- "Not logged in. Please run /login",- "Your credit balance is too low",- "Claude Code is not installed",- "Invalid API key · Please run /login",- "Claude Code 2.1.81 does not support this model",- ] {- assert!(!worth_retrying(needs_you), "{needs_you}");++ impl FailingConversation {+ fn new(failures: impl IntoIterator<Item = Option<TurnFailure>>, session: bool) -> Self {+ Self {+ failures: failures.into_iter().collect(),+ failure: None,+ queued: Vec::new(),+ sent: Vec::new(),+ session: session.then(|| "saved".into()),+ message: "provider response",+ }+ }}- assert_eq!(RunOptions::default().max_rounds, 6, "room to keep fixing until it works");-}-/// #142: a rate-limited service's own wait is followed (rounded up, capped), not a fixed one.-#[test]-fn a_services_requested_wait_is_read() {- assert_eq!(retry_after("OpenCode stopped with exit code 1: Please retry in 19.052906498s."), Some(20));- assert_eq!(retry_after("Rate limit reached. Retry after 2 minutes"), Some(120));- assert_eq!(retry_after("try again in 500ms"), Some(1));- assert_eq!(retry_after("Please try again in 3h"), Some(MAX_RETRY_WAIT));- assert_eq!(retry_after("Retry in 45 seconds"), Some(45));- for no_wait in ["API Error: 529 overloaded", "please retry in a moment", "retry in ."] {- assert_eq!(retry_after(no_wait), None, "{no_wait}");+ impl Conversation for FailingConversation {+ fn send(&mut self, text: &str) {+ self.sent.push(text.into());+ self.failure = self.failures.pop_front().flatten();+ self.queued.push(if self.failure.is_some() {+ AgentEvent::Error { message: self.message.into(), remediation: None, log: None }+ } else {+ AgentEvent::TextDelta { text: "Done".into() }+ });+ self.queued.push(AgentEvent::TurnDone { backend: "scripted".into(), input_tokens: 0, output_tokens: 0, cost_usd: None, cost_is_estimate: false });+ }+ fn session_id(&self) -> Option<String> {+ self.session.clone()+ }+ fn last_failure(&self) -> Option<TurnFailure> {+ self.failure+ }+ fn poll(&mut self) -> Vec<AgentEvent> {+ std::mem::take(&mut self.queued)+ }+ fn answer_consent(&mut self, _: &str, _: bool) {}+ fn stop(&mut self) {}+ fn is_busy(&self) -> bool {+ false+ }+ }++ #[test]+ fn transient_failures_retry_without_requiring_an_external_session() {+ for saved in [false, true] {+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ let failure = TurnFailure { kind: FailureKind::RateLimited, retry_after_secs: Some(2) };+ let mut agent = FailingConversation::new([Some(failure), None], saved);+ agent.message = "Authentication and billing service restarting. Please retry in 90s.";+ let mut waits = Vec::new();+ let result = worker.turn_with_wait(&mut ws, &mut agent, "Build the requested app", None, |_, seconds| {+ waits.push(seconds);+ Ok(())+ });+ assert_eq!(result.unwrap(), "Done");+ assert_eq!(waits, [3], "typed provider wait overrides the text hint");+ assert_eq!(agent.sent.len(), 2);+ assert_eq!(agent.sent[0], "Build the requested app");+ assert_eq!(agent.sent[1], if saved { crate::failure::CONTINUE_TURN } else { "Build the requested app" });+ }+ }++ #[test]+ fn setup_errors_do_not_retry_even_with_a_wait_in_the_message() {+ for kind in [+ FailureKind::Authentication,+ FailureKind::Billing,+ FailureKind::MissingProgram,+ FailureKind::ToolsUnavailable,+ FailureKind::Configuration,+ FailureKind::Unknown,+ ] {+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ let mut agent = FailingConversation::new([Some(TurnFailure::new(kind))], true);+ agent.message = "需要修改设置。Please retry in 1s.";+ let result = worker.turn_with_wait(&mut ws, &mut agent, "Build it", None, |_, _| panic!("a setup error must not wait and retry"));+ assert!(result.is_err(), "{kind:?}");+ assert_eq!(agent.sent.len(), 1, "{kind:?}");+ }+ }++ #[test]+ fn retries_are_bounded_and_cancelled_waits_send_nothing_more() {+ assert_eq!(RunOptions::default().max_rounds, 6, "room to keep fixing until it works");+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ let failure = TurnFailure::new(FailureKind::Transient);+ let mut agent = FailingConversation::new([Some(failure); 5], false);+ let mut waits = Vec::new();+ assert!(+ worker+ .turn_with_wait(&mut ws, &mut agent, "Build it", None, |_, seconds| {+ waits.push(seconds);+ Ok(())+ })+ .is_err()+ );+ assert_eq!(agent.sent.len(), 4, "initial attempt and at most three retries");+ assert_eq!(waits, [30, 60, 90]);+ let mut agent = FailingConversation::new([Some(failure)], false);+ let error = worker+ .turn_with_wait(&mut ws, &mut agent, "Build it", None, |stop, _| {+ stop.cancel();+ stop.check()+ })+ .unwrap_err();+ assert_eq!(error.code, ErrorCode::Cancelled);+ assert_eq!(agent.sent.len(), 1);+ }++ #[test]+ fn a_local_self_check_write_error_is_not_retried_as_a_backend_rate_limit() {+ #[derive(Clone)]+ struct BlockReply {+ answer: PathBuf,+ }+ impl Check for BlockReply {+ fn id(&self) -> &'static str {+ "blocked-reply"+ }+ fn run(&self, cx: &mut CheckCx) -> Result<CheckResult, ToolError> {+ // The request was pending when the run read it; its answer now cannot be+ // written. The workspace itself remains writable, including retry status.+ std::fs::create_dir_all(&self.answer).unwrap();+ Ok(CheckResult::new(Check::id(self), cx.round, true, "The check finished."))+ }+ }+ impl TargetKind for BlockReply {+ fn id(&self) -> &'static str {+ "blocked-reply"+ }+ fn words(&self) -> KindWords {+ SlidesKind.words()+ }+ fn detect(&self, _: &DetectCx) -> Option<u32> {+ Some(1)+ }+ fn default_stack(&self, cx: &DetectCx) -> &'static str {+ SlidesKind.default_stack(cx)+ }+ fn goal(&self) -> &'static str {+ SlidesKind.goal()+ }+ fn checks(&self, _: &Workspace) -> Vec<Box<dyn Check>> {+ vec![Box::new(self.clone())]+ }+ }+ struct RateLimited {+ sent: usize,+ first_poll: bool,+ }+ impl Conversation for RateLimited {+ fn send(&mut self, _: &str) {+ self.sent += 1;+ self.first_poll = true;+ }+ fn poll(&mut self) -> Vec<AgentEvent> {+ vec![if std::mem::take(&mut self.first_poll) {+ AgentEvent::Error { message: "The provider is rate limited.".into(), remediation: None, log: None }+ } else {+ AgentEvent::TurnDone { backend: "scripted".into(), input_tokens: 0, output_tokens: 0, cost_usd: None, cost_is_estimate: false }+ }]+ }+ fn last_failure(&self) -> Option<TurnFailure> {+ Some(TurnFailure::new(FailureKind::RateLimited))+ }+ fn answer_consent(&mut self, _: &str, _: bool) {}+ fn stop(&mut self) {}+ fn is_busy(&self) -> bool {+ false+ }+ }+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let asks = Inbox::new(self_check::asks_dir(&ws.root));+ let id = asks.post(craft_anything_engine::inbox::RequestKind::Question, "Check the build", "", Vec::new(), false, None).unwrap();+ let mut worker = check_worker(&ws, false);+ worker.self_check = Some((Arc::new(BlockReply { answer: asks.dir().join(format!("{id}.answer.json")) }), 90.0));+ let mut agent = RateLimited { sent: 0, first_poll: false };+ let mut waits = Vec::new();+ let error = worker+ .turn_with_wait(&mut ws, &mut agent, "Build the app", None, |_, seconds| {+ waits.push(seconds);+ Ok(())+ })+ .unwrap_err();+ assert_eq!(agent.sent, 1, "a backend's retry hint cannot retry a local persistence failure");+ assert!(waits.is_empty());+ assert_ne!(error.code, ErrorCode::ProgramFailed);+ assert!(error.message.contains(".answer.json"), "the original local failure is retained: {error:?}");+ }++ #[test]+ fn an_api_rate_limit_continues_history_without_repeating_users_or_completed_tools() {+ use crate::agent::tests::{Scripted, turn};+ use crate::agent::{ApiConversation, EngineHost, LoopConfig, ModelSlot};+ use crate::model::{Block, Speaker, StopReason};+ for write_first in [false, true] {+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a notes app", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ worker.attach(&ws).unwrap();+ let (tx, rx) = std::sync::mpsc::channel();+ worker.tx = tx;+ let seen = Arc::default();+ let mut script = Scripted { turns: VecDeque::new(), seen: Arc::clone(&seen) };+ if write_first {+ script.turns.push_back(turn(+ vec![Block::ToolUse {+ id: "write-once".into(),+ name: "build_file_write".into(),+ input: json!({"path":"app/README.md","content":"Built once"}),+ }],+ StopReason::ToolUse,+ ));+ }+ script.turns.push_back(Err(ToolError::new(ErrorCode::OverLimit, "服务限流").with_details(json!({"status":429,"retry_after_secs":3}))));+ script.turns.push_back(turn(vec![Block::Text { text: "Built the notes app".into() }], StopReason::EndTurn));+ let slot = ModelSlot { name: "Scripted".into(), model: Box::new(script), local: false, effort: None, max_turns: 10, context_tokens: 0 };+ let enabled = std::collections::BTreeSet::from([craft_anything_engine::Toolset::Core, craft_anything_engine::Toolset::Build]);+ let mut api = ApiConversation::start_with(Arc::new(EngineHost(worker.engine.clone())), vec![slot], LoopConfig::default(), enabled).unwrap();+ let mut waits = Vec::new();+ assert_eq!(+ worker+ .turn_with_wait(&mut ws, &mut api, "Build the notes app", None, |_, seconds| {+ waits.push(seconds);+ Ok(())+ })+ .unwrap(),+ "Built the notes app"+ );+ assert_eq!(waits, [4]);+ assert!(ws.manifest.agent_session.is_none());+ let requests = seen.lock().unwrap();+ assert_eq!(requests.len(), if write_first { 3 } else { 2 });+ let before_retry = &requests[requests.len() - 2];+ let after_retry = requests.last().unwrap();+ assert_eq!(before_retry, after_retry, "retry must continue the same model request");+ assert_eq!(after_retry.iter().filter(|m| m.role == Speaker::User && m.text() == "Build the notes app").count(), 1);+ let calls = rx.try_iter().filter(|e| matches!(e, RunEvent::Agent(AgentEvent::ToolStarted { tool, .. }) if tool == "build_file_write")).count();+ assert_eq!(calls, usize::from(write_first), "a completed write is not replayed");+ if write_first {+ assert_eq!(std::fs::read_to_string(ws.app_dir().join("README.md")).unwrap(), "Built once");+ }+ }+ }++ #[cfg(unix)]+ fn executable_cli(ws: &Workspace, model: &str, script: &str) -> crate::cli_agents::CliConversation {+ use std::os::unix::fs::PermissionsExt;+ let program = ws.root.join("agent-stand-in.sh");+ std::fs::write(&program, script).unwrap();+ std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap();+ let config = crate::BackendConfig {+ name: "Scripted CLI".into(),+ kind: BackendKind::AgentCli,+ model: model.into(),+ base_url: None,+ ssh_host: None,+ enabled: true,+ };+ let options = crate::BackendOptions { command: Some(program.display().to_string()), timeout_secs: Some(10), ..Default::default() };+ let mut spec = CliSpec::new(config, options, false, None);+ spec.project = Some(ws.root.clone());+ spec.preamble = Some(String::new());+ crate::cli_agents::CliConversation::new(spec)+ }++ #[cfg(unix)]+ #[test]+ fn a_failed_required_tool_server_stops_the_owned_cli_without_resuming_it() {+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ let (tx, rx) = std::sync::mpsc::channel();+ worker.tx = tx;+ let mut cli = executable_cli(+ &ws,+ "claude-code",+ concat!(+ "#!/bin/sh\ncat >/dev/null\nprintf 'attempt\\n' >> attempts\n",+ "printf '%s\\n' '{\"type\":\"system\",\"subtype\":\"init\",\"session_id\":\"cannot-build\",\"mcp_servers\":[{\"name\":\"craft-anything\",\"status\":\"failed\"}]}'\n",+ "sleep 5\nprintf 'unreachable' > continued-without-tools\n"+ ),+ );+ let error = worker.turn_with_wait(&mut ws, &mut cli, "Build the app", None, |_, _| panic!("missing tools must not be retried")).unwrap_err();+ assert!(error.message.contains("tools did not start"), "{error:?}");+ assert_eq!(cli.last_failure().unwrap().kind, FailureKind::ToolsUnavailable);+ assert!(!cli.is_busy());+ assert_eq!(std::fs::read_to_string(ws.root.join("attempts")).unwrap(), "attempt\n");+ assert!(!ws.root.join("continued-without-tools").exists(), "the owned child was stopped at the failed init");+ assert_eq!(rx.try_iter().filter(|e| matches!(e, RunEvent::Agent(AgentEvent::TurnDone { .. }))).count(), 1);+ }++ #[cfg(unix)]+ #[test]+ fn a_plain_nonzero_cli_exit_resumes_its_saved_session() {+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ let (tx, rx) = std::sync::mpsc::channel();+ worker.tx = tx;+ let mut cli = executable_cli(+ &ws,+ "claude-code",+ concat!(+ "#!/bin/sh\ncat >> prompts\nprintf '\\n' >> prompts\nprintf 'attempt\\n' >> attempts\n",+ "case \" $* \" in *\" --resume interrupted-session \"*)\n",+ "printf '%s\\n' '{\"type\":\"assistant\",\"message\":{\"content\":[{\"type\":\"text\",\"text\":\"Built after the interruption\"}]}}'\n",+ "printf '%s\\n' '{\"type\":\"result\",\"is_error\":false,\"result\":\"Built after the interruption\"}'\nexit 0;; esac\n",+ "printf '%s\\n' '{\"type\":\"system\",\"subtype\":\"init\",\"session_id\":\"interrupted-session\"}'\nexit 1\n"+ ),+ );+ let mut waits = Vec::new();+ let result = worker.turn_with_wait(&mut ws, &mut cli, "Make a notes app", None, |_, seconds| {+ waits.push(seconds);+ Ok(())+ });+ assert_eq!(std::fs::read_to_string(ws.root.join("attempts")).unwrap(), "attempt\nattempt\n", "a plain nonzero exit must resume the reported session");+ assert_eq!(result.unwrap(), "Built after the interruption");+ assert_eq!(waits, [30]);+ assert_eq!(std::fs::read_to_string(ws.root.join("prompts")).unwrap(), format!("Make a notes app\n{}\n", crate::failure::CONTINUE_TURN));+ assert_eq!(cli.session_id().as_deref(), Some("interrupted-session"));+ assert!(cli.last_failure().is_none(), "successful recovery clears the process failure");+ assert!(!cli.is_busy());+ assert_eq!(rx.try_iter().filter(|e| matches!(e, RunEvent::Agent(AgentEvent::TurnDone { .. }))).count(), 2);+ }++ #[cfg(unix)]+ #[test]+ fn typed_cli_setup_errors_take_precedence_over_a_nonzero_exit() {+ for (code, kind) in+ [("authentication_failed", FailureKind::Authentication), ("billing_error", FailureKind::Billing), ("model_not_found", FailureKind::Configuration)]+ {+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ let error = json!({"type":"assistant","error":code,"message":{"content":[{"type":"text","text":"Please retry in 1s."}]}});+ let script = format!("#!/bin/sh\ncat >/dev/null\nprintf 'attempt\\n' >> attempts\nprintf '%s\\n' '{error}'\nexit 1\n");+ let mut cli = executable_cli(&ws, "claude-code", &script);+ let result = worker.turn_with_wait(&mut ws, &mut cli, "Make a notes app", None, |_, _| panic!("typed setup failures must not retry"));+ assert!(result.is_err(), "{kind:?}");+ assert_eq!(cli.last_failure().unwrap().kind, kind);+ assert_eq!(std::fs::read_to_string(ws.root.join("attempts")).unwrap(), "attempt\n");+ assert!(!cli.is_busy());+ }+ }++ #[cfg(unix)]+ #[test]+ fn opencode_retries_a_typed_rate_limit_with_its_whole_original_request() {+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ let (tx, rx) = std::sync::mpsc::channel();+ worker.tx = tx;+ let mut cli = executable_cli(+ &ws,+ "opencode",+ concat!(+ "#!/bin/sh\nif [ \"$2\" = '--help' ]; then printf '%s\\n' '--format'; exit 0; fi\n",+ "for arg do last=\"$arg\"; done\nprintf '%s\\n' \"$last\" >> prompts\n",+ "if [ ! -f rate-was-returned ]; then touch rate-was-returned\n",+ "printf '%s\\n' '{\"type\":\"error\",\"error\":{\"name\":\"APIError\",\"data\":{\"message\":\"Please retry in 19.052906498s.\",\"statusCode\":429,\"isRetryable\":true}}}'\nexit 1\nfi\n",+ "printf '%s\\n' '{\"type\":\"text\",\"part\":{\"text\":\"Built the app\"}}'\n"+ ),+ );+ let mut waits = Vec::new();+ assert_eq!(+ worker+ .turn_with_wait(&mut ws, &mut cli, "Make a notes app", None, |_, seconds| {+ waits.push(seconds);+ Ok(())+ })+ .unwrap(),+ "Built the app"+ );+ assert_eq!(waits, [21]);+ assert_eq!(std::fs::read_to_string(ws.root.join("prompts")).unwrap(), "Make a notes app\nMake a notes app\n");+ assert!(cli.session_id().is_none());+ assert!(cli.last_failure().is_none(), "successful retry clears the prior failure");+ assert!(!cli.is_busy());+ assert_eq!(rx.try_iter().filter(|e| matches!(e, RunEvent::Agent(AgentEvent::TurnDone { .. }))).count(), 2, "one final event per process attempt");+ }++ #[cfg(unix)]+ #[test]+ fn an_untyped_opencode_exit_keeps_its_error_and_the_three_retry_cap() {+ let dir = tempfile::tempdir().unwrap();+ let mut ws = Workspace::create(dir.path(), "x", "Build a thing", TargetRef::None).unwrap();+ let mut worker = check_worker(&ws, false);+ let mut cli = executable_cli(+ &ws,+ "opencode",+ "#!/bin/sh\nif [ \"$2\" = '--help' ]; then printf 'old help\\n'; exit 0; fi\nprintf 'attempt\\n' >> attempts\nprintf 'Unknown option: --format. Please retry in 1s.\\n' >&2\nexit 2\n",+ );+ let mut waits = Vec::new();+ let error = worker+ .turn_with_wait(&mut ws, &mut cli, "Make a notes app", None, |_, seconds| {+ waits.push(seconds);+ Ok(())+ })+ .unwrap_err();+ assert!(error.message.contains("Unknown option"), "{error:?}");+ assert_eq!(cli.last_failure().unwrap().kind, FailureKind::Transient);+ assert!(!cli.is_busy());+ assert_eq!(waits, [2, 2, 2], "the display wait hint affects only the delay after a retryable exit");+ assert_eq!(std::fs::read_to_string(ws.root.join("attempts")).unwrap(), "attempt\nattempt\nattempt\nattempt\n");}- assert!(worth_retrying("Please retry in 19.05s"), "a rate limit is waited out, not handed to the person");}#[test]@@ -1874,7 +2271,7 @@ fn a_missing_session_with_nothing_to_start_from_is_said_plainly() {usage: None,self_check: None,};- let e = w.turn(&mut ws, &mut agent, "Carry on.", None).unwrap_err();+ let e = w.turn_with_wait(&mut ws, &mut agent, "Carry on.", None, |_, _| panic!("a missing session without a fresh start must not be retried")).unwrap_err();assert!(e.message.contains("isn't on this computer"), "{e:?}");assert!(ws.manifest.agent_session.is_none());}diff --git a/docs/ROADMAP.md b/docs/ROADMAP.mdindex 25c62a97..b010cf2b 100644--- a/docs/ROADMAP.md+++ b/docs/ROADMAP.md@@ -170,18 +170,18 @@ Every round spends the person's money or quota. One that can't do anything shoulacross runs: a run ends at 7 checks (`max_rounds`) but check numbers carry on, so #44's check28 and #417's 35+ look like several runs (a guess: Try again is the only restart found, andAuto mode doesn't start runs)-- [ ] A builder whose tools didn't start isn't resumed. `turn` (`rebuild/mod.rs`) tries 3 more times+- [x] A builder whose tools didn't start isn't resumed. `turn` (`rebuild/mod.rs`) tries 3 more times("Picking up where it left off", #377) because `worth_retrying` doesn't know this error. Giveit a typed signal, as `Conversation::session_lost` does, not one more word in that function's- list. Say what failed and stop (pitfalls: never retry what can't succeed)-- [ ] `worth_retrying` decides on English words in a message (AGENTS.md: structured results). Move- its other cases (sign-in, credit, a missing install) to the same typed signal+ list. Say what failed and stop (pitfalls: never retry what can't succeed) — commit `f80a16a66834`+- [x] `worth_retrying` decides on English words in a message (AGENTS.md: structured results). Move+ its other cases (sign-in, credit, a missing install) to the same typed signal — commit `f80a16a66834`- [ ] Find what repeated #64's attempts and stop it: it logged 41, about 38 s apart, on 0.2.0, but`turn` allows 3 retries 30, 60 and 90 s apart, there too, so something else starts them-- [ ] A rate limit is waited for only on agents with a session: `turn` retries only when+- [x] A rate limit is waited for only on agents with a session: `turn` retries only when`agent.session_id()` is set, which API backends and OpenCode's plain adapter never set (readin the code, not run). So #148 didn't reach #142, which was OpenCode. Wait and send the turn- again for them too, then close #142+ again for them too, then close #142 — commit `f80a16a66834`- [ ] **decision** Whether a build gets a token or cost limit of its own, beyond rounds: #399 used9.4M tokens in 19 minutes of the first stage, then failed to compilediff --git a/docs/STATUS.md b/docs/STATUS.mdindex fdd9b072..f8c98c25 100644--- a/docs/STATUS.md+++ b/docs/STATUS.md@@ -41,6 +41,39 @@ workspace clippy, layers, fmt and the secret scan of all history were clean. Chi| SQLite database files (`db_sqlite_show`) | A rowid alias column (`INTEGER PRIMARY KEY`, including the table-constraint form) shows the rowid; `DESC`, non-`INTEGER` and composite keys keep their stored values | Hand-built fixtures; generated files compared with the official SQLite library || SQLite WITHOUT ROWID tables | Read from the index b-tree, interior records and overflow included; rows in primary-key order, values in declaration order, `rowid` null | Hand-built fixtures; generated files compared with the official SQLite library |+### Builder retry policy++Builder failures carry a typed reason through `Conversation::last_failure`. Typed authentication,+billing, missing programs, unavailable tools, configuration and unknown protocol failures stop+for attention. Transport interruptions, rate limits and nonzero process exits without a typed+error get at most three retries; the decision no longer searches English error messages.+Structured provider waits win; the earlier numeric “retry in”+hint is only a delay fallback after a failure has already been classified as retryable. All waits+remain cancellable and bounded to fifteen minutes.+Local inbox and workspace write failures are not retried using a concurrent backend failure's+retry advice; the original local error is returned.+A missing saved session starts afresh only when a fresh request is available; otherwise the run+stops without retrying the unavailable session.++Claude Code's failed or unauthenticated `craft-anything` MCP server in its init event stops the+owned CLI process. Pending servers and failures in unrelated optional servers do not stop it.+Claude assistant error codes, HTTP status/provider errors and ACP authentication errors retain+their typed meanings. A later generic result does not erase a more specific error.++API retries continue the existing history without inserting the user's message again or replaying+completed tools. Stateless CLI retries receive the whole original request. OpenCode now uses+`run --format json`: its API error status and retry flag decide retryability while text, tools and+usage still reach the feed. Missing or invalid step costs keep the turn's cost unknown; explicit+zero remains a known cost. A CLI that fails with only a numeric exit also uses bounded retries;+its display error remains available if recovery fails. The process wrapper sends one final turn+event and becomes available for the next request.++Protocol fixtures, a loopback HTTP server, scripted API turns and executable CLI stand-ins cover+these paths, including cancellation and the retry cap. An executable that exits 1 without a+structured error resumes its saved session, while typed setup errors retain priority over the+same exit status. No live provider or paid-model run was performed. The separate source of #64's+41 attempts remains unverified.+### Bun stored-text encodingPacked Bun members with encoding value 2 are labelled `utf16le`, matching their stored UTF-16LE