The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(runner): durably reconcile Codex tools (#12384)
Dotta6c85fa060e96
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rsindex 1095e28495..81763e1043 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs@@ -120,6 +120,66 @@ fn emit_ambiguous_turn_evidence(}}+fn emit_ambiguous_turn_item() -> io::Result<()> {+ send(json!({+ "method": "item/completed",+ "params": {"item": {+ "id": "replacement-message-before-terminal",+ "type": "agentMessage",+ "status": "completed",+ "text": "Replacement output before terminal authority."+ }}+ }))+}++fn send_question(state: &FakeState) -> io::Result<()> {+ let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1");+ send(json!({+ "id": "runtime-request-1",+ "method": "item/tool/requestUserInput",+ "params": {+ "threadId": state.thread_id,+ "turnId": turn_id,+ "itemId": "question-item-1",+ "isBlocking": true,+ "title": "Deployment input",+ "questions": [{+ "id": "environment",+ "header": "Environment",+ "question": "Where should we deploy?",+ "options": [+ {"label": "Staging", "description": "Deploy safely."},+ {"label": "Production", "description": "Deploy directly."}+ ]+ }]+ }+ }))+}++fn send_runtime_request_flood(state: &FakeState, interrupt_count: u64) -> io::Result<()> {+ let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1");+ for index in 0..160_u64 {+ send(json!({+ "id": format!("runtime-flood-{interrupt_count}-{index}"),+ "method": "item/tool/requestUserInput",+ "params": {+ "threadId": state.thread_id,+ "turnId": turn_id,+ "itemId": format!("question-item-{interrupt_count}-{index}"),+ "isBlocking": true,+ "title": "Bounded cleanup input",+ "questions": [{+ "id": "environment",+ "header": "Environment",+ "question": "Where should we deploy?",+ "options": [{"label": "Staging", "description": "Deploy safely."}],+ }],+ },+ }))?;+ }+ Ok(())+}+fn run() -> Result<(), Box<dyn std::error::Error>> {let args = std::env::args().skip(1).collect::<Vec<_>>();let state_path =@@ -136,6 +196,19 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {let exit_after_tool_call_completion = args.iter().any(|value| value == "--exit-after-tool-call-completion");+ let emit_tool_call_on_resume = args+ .iter()+ .any(|value| value == "--emit-tool-call-on-resume");+ let resume_unowned_turn_when_marked = args+ .iter()+ .any(|value| value == "--resume-unowned-turn-when-marked");+ let replay_completed_tool_call_count = argument(&args, "--replay-completed-tool-call-count")+ .map(|value| value.parse::<u64>())+ .transpose()?+ .unwrap_or_default();+ let finish_turn_with_pending_tool = args+ .iter()+ .any(|value| value == "--finish-turn-with-pending-tool");let require_dynamic_tool = args.iter().any(|value| value == "--require-dynamic-tool");let hold_turn = args.iter().any(|value| value == "--hold-turn");let exit_after_turn_start = args.iter().any(|value| value == "--exit-after-turn-start");@@ -163,6 +236,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {let missing_id_second_turn_start = args.iter().any(|value| value == "--missing-id-second-turn-start");+ let missing_id_live_turn_start = args+ .iter()+ .any(|value| value == "--missing-id-live-turn-start");let fail_after_accepting_second_turn_before_response = args.iter().any(|value| value == "--fail-after-accepting-second-turn-before-response");@@ -184,32 +260,93 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {let conflicting_ambiguous_second_turn = args.iter().any(|value| value == "--conflicting-ambiguous-second-turn");+ let ambiguous_older_reused_turn = args+ .iter()+ .any(|value| value == "--ambiguous-older-reused-turn");let omit_ambiguous_turn_started = args.iter().any(|value| value == "--omit-ambiguous-turn-started");let fail_after_thread_read = args.iter().any(|value| value == "--fail-after-thread-read");+ let fail_first_interrupt = args.iter().any(|value| value == "--fail-first-interrupt");+ let accept_interrupt_without_terminal_once = args+ .iter()+ .any(|value| value == "--accept-interrupt-without-terminal-once");+ let accept_interrupt_without_terminal = args+ .iter()+ .any(|value| value == "--accept-interrupt-without-terminal");+ let flood_runtime_requests_on_interrupt = args+ .iter()+ .any(|value| value == "--flood-runtime-requests-on-interrupt");+ let interrupt_terminal_delay_ms = argument(&args, "--interrupt-terminal-delay-ms")+ .map(|value| value.parse::<u64>())+ .transpose()?;let exit_after_thread_read = args.iter().any(|value| value == "--exit-after-thread-read");let fail_after_turn_completion_delay_ms =argument(&args, "--fail-after-turn-completion-delay-ms").map(|value| value.parse::<u64>()).transpose()?;+ let delayed_tool_after_failed_turn = args+ .iter()+ .any(|value| value == "--delayed-tool-after-failed-turn");+ let delayed_tool_after_next_turn_start = args+ .iter()+ .any(|value| value == "--delayed-tool-after-next-turn-start");+ let delayed_tool_after_third_turn_start = args+ .iter()+ .any(|value| value == "--delayed-tool-after-third-turn-start");+ let delayed_tool_after_second_turn_completion = args+ .iter()+ .any(|value| value == "--delayed-tool-after-second-turn-completion");+ let tool_after_reused_turn_start = args+ .iter()+ .any(|value| value == "--tool-after-reused-turn-start");+ let tool_after_older_reused_turn_start = args+ .iter()+ .any(|value| value == "--tool-after-older-reused-turn-start");+ let question_before_failed_turn = args+ .iter()+ .any(|value| value == "--question-before-failed-turn");+ let reuse_question_id = args.iter().any(|value| value == "--reuse-question-id");let pre_response_notification = args.iter().any(|value| value == "--notification-before-response");let mut state = load_state(&state_path);let mut turn_start_count = 0_u64;+ let mut interrupt_count = 0_u64;+ let mut delayed_interrupt_terminal_scheduled = false;+ let mut answered_questions = 0u8;+ let mut replayed_completed_tool_calls = 0_u64;for line in io::stdin().lock().lines() {let message: Value = serde_json::from_str(&line?)?;+ if message.get("method").is_none()+ && message+ .get("id")+ .and_then(Value::as_str)+ .is_some_and(|id| id.starts_with("runtime-flood-"))+ {+ let outcome = if message.get("error").is_some() {+ "runtime-response:rejected"+ } else {+ "runtime-response:cancelled"+ };+ log_call(call_log.as_deref(), outcome)?;+ continue;+ }if message.get("method").is_none() && message.get("id") == Some(&json!("runtime-request-1")){+ if reuse_question_id && answered_questions == 0 {+ answered_questions = 1;+ send_question(&state)?;+ continue;+ }finish_turn(&state_path, &mut state, "completed")?;continue;}if message.get("method").is_none() && message.get("id") == Some(&json!("tool-request-1")) {if message.pointer("/result/success") == Some(&json!(false)) {log_call(call_log.as_deref(), "tool-response:failure")?;- if state.active_turn_id.is_some() {+ if state.active_turn_id.is_some() && !hold_turn {finish_turn(&state_path, &mut state, "failed")?;}continue;@@ -225,7 +362,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {if result != json!({"ok": true, "task": {"id": "task-1"}}) {return Err("semantic tool response changed the operation result".into());}- if replay_completed_tool_call {+ log_call(call_log.as_deref(), &format!("tool-response:{text}"))?;+ if replay_completed_tool_call && replayed_completed_tool_calls == 0 {+ replayed_completed_tool_calls += 1;send(json!({"id": "tool-request-replay","method": "item/tool/call",@@ -237,9 +376,22 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {"arguments": {}}}))?;- continue;+ } else if replayed_completed_tool_calls < replay_completed_tool_call_count {+ replayed_completed_tool_calls += 1;+ send(json!({+ "id": "tool-request-1",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": state.active_turn_id,+ "callId": "semantic-call-1",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))?;+ } else if !hold_turn {+ finish_turn(&state_path, &mut state, "completed")?;}- finish_turn(&state_path, &mut state, "completed")?;continue;}let Some(method) = message.get("method").and_then(Value::as_str) else {@@ -275,10 +427,31 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {if require_dynamic_tool && !has_task_context_tool(&message) {return Err("thread/resume omitted the authorized dynamic tool".into());}+ let unowned_turn_marker = state_path.with_file_name("resume-unowned-turn");+ if resume_unowned_turn_when_marked && unowned_turn_marker.exists() {+ state.active_turn_id = Some("provider-turn-unowned".to_owned());+ save_state(&state_path, &state)?;+ fs::remove_file(unowned_turn_marker)?;+ }send(json!({"id": id,"result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}}}))?;+ if emit_tool_call_on_resume {+ if let Some(turn_id) = state.active_turn_id.as_deref() {+ send(json!({+ "id": "tool-request-1",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": turn_id,+ "callId": "semantic-call-1",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))?;+ }+ }}"thread/read" => {let turns = state@@ -319,17 +492,31 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {&& (complete_ambiguous_second_turn|| complete_ambiguous_second_turn_before_response|| conflicting_ambiguous_second_turn);- let provider_turn_id = if emits_ambiguous_turn_evidence+ let provider_turn_id = if turn_start_count == 3 && ambiguous_older_reused_turn {+ "provider-turn-1".to_owned()+ } else if emits_ambiguous_turn_evidence|| (turn_start_count == 2&& (retain_ambiguous_second_turn_active|| hold_ambiguous_second_turn_after_item)){- "provider-turn-2"+ "provider-turn-2".to_owned()+ } else if tool_after_reused_turn_start+ || (tool_after_older_reused_turn_start && turn_start_count == 3)+ {+ "provider-turn-1".to_owned()} else {- "provider-turn-1"+ format!("provider-turn-{turn_start_count}")};- state.active_turn_id = Some(provider_turn_id.to_owned());+ state.active_turn_id = Some(provider_turn_id.clone());save_state(&state_path, &state)?;+ if ambiguous_older_reused_turn && turn_start_count == 3 {+ send(json!({+ "method": "turn/started",+ "params": {"turn": {"id": provider_turn_id}}+ }))?;+ send(json!({"id": id, "error": {}}))?;+ continue;+ }if complete_ambiguous_second_turn_before_response && turn_start_count == 2 {emit_ambiguous_turn_evidence(&state_path,@@ -355,6 +542,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {return Ok(());}if malformed_error_second_turn_start && turn_start_count == 2 {+ if hold_ambiguous_second_turn_after_item {+ emit_ambiguous_turn_item()?;+ }send(json!({"id": id, "error": {}}))?;if emits_ambiguous_turn_evidence&& !complete_ambiguous_second_turn_before_response@@ -374,15 +564,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {"result": {"turn": {"status": "inProgress"}}}))?;if hold_ambiguous_second_turn_after_item {- send(json!({- "method": "item/completed",- "params": {"item": {- "id": "replacement-message-before-terminal",- "type": "agentMessage",- "status": "completed",- "text": "Replacement output before terminal authority."- }}- }))?;+ emit_ambiguous_turn_item()?;continue;}if emits_ambiguous_turn_evidence@@ -397,16 +579,103 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {}return Err("configured failure after missing turn identity".into());}+ if missing_id_live_turn_start {+ send(json!({+ "id": id,+ "result": {"turn": {"status": "inProgress"}}+ }))?;+ continue;+ }send(json!({"id": id,- "result": {"turn": {"id": "provider-turn-1", "status": "inProgress"}}+ "result": {"turn": {"id": provider_turn_id, "status": "inProgress"}}}))?;send(json!({"method": "turn/started",- "params": {"turn": {"id": "provider-turn-1"}}+ "params": {"turn": {"id": provider_turn_id}}}))?;if fail_after_second_turn_start && turn_start_count == 2 {return Err("configured failure after second turn start".into());+ } else if question_before_failed_turn {+ send_question(&state)?;+ send(json!({+ "method": "turn/completed",+ "params": {+ "threadId": state.thread_id,+ "turn": {"id": provider_turn_id, "status": "failed"}+ }+ }))?;+ state.active_turn_id = None;+ save_state(&state_path, &state)?;+ } else if delayed_tool_after_failed_turn {+ send(json!({+ "method": "turn/failed",+ "params": {+ "threadId": state.thread_id,+ "turn": {"id": provider_turn_id, "status": "failed"}+ }+ }))?;+ state.active_turn_id = None;+ save_state(&state_path, &state)?;+ send(json!({+ "id": "tool-request-delayed",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": provider_turn_id,+ "callId": "semantic-call-delayed",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))?;+ } else if delayed_tool_after_next_turn_start && turn_start_count == 2 {+ send(json!({+ "id": "tool-request-delayed",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": "provider-turn-1",+ "callId": "semantic-call-delayed",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))?;+ } else if delayed_tool_after_third_turn_start && turn_start_count == 3 {+ send(json!({+ "id": "tool-request-two-turns-delayed",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": "provider-turn-1",+ "callId": "semantic-call-two-turns-delayed",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))?;+ } else if tool_after_reused_turn_start && turn_start_count == 2 {+ send(json!({+ "id": "tool-request-reused-turn",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": "provider-turn-1",+ "callId": "semantic-call-reused-turn",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))?;+ } else if tool_after_older_reused_turn_start && turn_start_count == 3 {+ send(json!({+ "id": "tool-request-older-reused-turn",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": "provider-turn-1",+ "callId": "semantic-call-older-reused-turn",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))?;} else if exit_after_turn_start {return Ok(());} else if emit_tool_call {@@ -415,41 +684,35 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {"method": "item/tool/call","params": {"threadId": state.thread_id,- "turnId": "provider-turn-1",+ "turnId": provider_turn_id,"callId": "semantic-call-1","tool": "get_task_context","arguments": {}}}))?;- if complete_after_tool_call {+ if complete_after_tool_call || finish_turn_with_pending_tool {finish_turn(&state_path, &mut state, "completed")?;if exit_after_tool_call_completion {return Ok(());}}} else if emit_question {- send(json!({- "id": "runtime-request-1",- "method": "item/tool/requestUserInput",- "params": {- "threadId": state.thread_id,- "turnId": "provider-turn-1",- "itemId": "question-item-1",- "isBlocking": true,- "title": "Deployment input",- "questions": [{- "id": "environment",- "header": "Environment",- "question": "Where should we deploy?",- "options": [- {"label": "Staging", "description": "Deploy safely."},- {"label": "Production", "description": "Deploy directly."}- ]- }]- }- }))?;+ send_question(&state)?;} else if !hold_turn {finish_turn(&state_path, &mut state, "completed")?;+ if delayed_tool_after_second_turn_completion && turn_start_count == 2 {+ send(json!({+ "id": "tool-request-idle-two-turns-delayed",+ "method": "item/tool/call",+ "params": {+ "threadId": state.thread_id,+ "turnId": "provider-turn-1",+ "callId": "semantic-call-idle-two-turns-delayed",+ "tool": "get_task_context",+ "arguments": {}+ }+ }))?;+ }if emit_post_completion_warning {send(json!({"method": "warning",@@ -477,8 +740,44 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {}"turn/steer" => send(json!({"id": id, "result": {"accepted": true}}))?,"turn/interrupt" => {- send(json!({"id": id, "result": {"accepted": true}}))?;- finish_turn(&state_path, &mut state, "interrupted")?;+ interrupt_count += 1;+ if fail_first_interrupt && interrupt_count == 1 {+ send(json!({+ "id": id,+ "error": {"code": -32001, "message": "configured interrupt failure"}+ }))?;+ } else {+ send(json!({"id": id, "result": {"accepted": true}}))?;+ if flood_runtime_requests_on_interrupt {+ send_runtime_request_flood(&state, interrupt_count)?;+ }+ if !accept_interrupt_without_terminal+ && !(accept_interrupt_without_terminal_once+ && interrupt_count == if fail_first_interrupt { 2 } else { 1 })+ {+ if let Some(delay_ms) = interrupt_terminal_delay_ms {+ if !delayed_interrupt_terminal_scheduled {+ delayed_interrupt_terminal_scheduled = true;+ let delayed_state_path = state_path.clone();+ let mut delayed_state = state.clone();+ thread::spawn(move || {+ thread::sleep(Duration::from_millis(delay_ms));+ if let Err(error) = finish_turn(+ &delayed_state_path,+ &mut delayed_state,+ "interrupted",+ ) {+ eprintln!(+ "failed to emit delayed interrupt terminal: {error}"+ );+ }+ });+ }+ } else {+ finish_turn(&state_path, &mut state, "interrupted")?;+ }+ }+ }}_ if id.is_some() => send(json!({"id": id,diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rsindex 4764ed1e88..62aed56ea1 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs@@ -50,28 +50,26 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {optional_u64(args, name).map(|value| Duration::from_millis(value.unwrap_or(default)))};let state_dir = PathBuf::from(value(args, "--state-dir")?);- run_durable_runner(- DurableRunnerConfig {- connect_url: value(args, "--connect-url")?,- state_dir: state_dir.clone(),- runner_instance_id: value(args, "--runner-id")?,- environment_lease_id: value(args, "--environment-lease-id")?,- run_id: value(args, "--run-id")?,- normalized_session_id: value(args, "--session-id")?,- turn_id: value(args, "--turn-id")?,- item_id: value(args, "--item-id")?,- runner_version: value(args, "--runner-version")?,- runner_digest: value(args, "--runner-digest")?,- max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 16 * 1024 * 1024)?,- p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 1024 * 1024)?,- max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?,- reconnect_delay: duration("--reconnect-delay-ms", 250)?,- max_runtime: duration("--max-runtime-ms", 60 * 60 * 1000)?,- },- ticket,- CodexCommandExecutor::new(state_dir),- )- .map_err(|error| LocalRunnerError::invalid(error.to_string()))+ let config = DurableRunnerConfig {+ connect_url: value(args, "--connect-url")?,+ state_dir: state_dir.clone(),+ runner_instance_id: value(args, "--runner-id")?,+ environment_lease_id: value(args, "--environment-lease-id")?,+ run_id: value(args, "--run-id")?,+ normalized_session_id: value(args, "--session-id")?,+ turn_id: value(args, "--turn-id")?,+ item_id: value(args, "--item-id")?,+ runner_version: value(args, "--runner-version")?,+ runner_digest: value(args, "--runner-digest")?,+ max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 16 * 1024 * 1024)?,+ p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 1024 * 1024)?,+ max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?,+ reconnect_delay: duration("--reconnect-delay-ms", 250)?,+ max_runtime: duration("--max-runtime-ms", 60 * 60 * 1000)?,+ };+ let executor = CodexCommandExecutor::with_runner_config(state_dir, &config);+ run_durable_runner(config, ticket, executor)+ .map_err(|error| LocalRunnerError::invalid(error.to_string()))}fn run() -> Result<(), LocalRunnerError> {diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rsindex 322236013d..249c13f0ae 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs@@ -4,18 +4,24 @@ use std::time::Duration;use serde::{Deserialize, Serialize};use serde_json::{json, Value};+use sha2::{Digest, Sha256};use crate::durable::redact_text;use crate::local_runner::LocalRunnerError;use crate::process_supervisor::SupervisedProcess;-use crate::provider_bridge::{AuthorizedTool, ToolResult};+use crate::provider_bridge::{AuthorizedTool, DurableReplayFilter, ToolResult};+use crate::provider_events::normalized_codex_terminal_event_type;pub const CODEX_APP_SERVER_MAX_FRAME_BYTES: usize = 4 * 1024 * 1024;const MAX_BUFFERED_MESSAGES: usize = 1_024;+const MAX_BUFFERED_MESSAGE_BYTES: usize = 16 * 1024 * 1024;const MAX_INSTRUCTIONS_BYTES: usize = 1024 * 1024;const MAX_PENDING_TOOL_REQUESTS: usize = 4_096;const MAX_PENDING_TOOL_REQUEST_BYTES: usize = 16 * 1024 * 1024;const MAX_COMPLETED_TOOL_CALL_IDS: usize = 4_096;+const MAX_PENDING_RUNTIME_REQUESTS: usize = 128;+const MAX_PENDING_RUNTIME_REQUEST_BYTES: usize = 4 * 1024 * 1024;+pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096;type QuestionOptionLabels = BTreeMap<String, BTreeMap<String, String>>;type QuestionSetMapping = (String, Value, QuestionOptionLabels);@@ -135,11 +141,71 @@ struct CompletedTurnAuthority {provider_turn_id: String,}+#[derive(Default)]+struct SettledProviderTurnIds {+ ids: BTreeSet<String>,+ filter: DurableReplayFilter,+}++impl SettledProviderTurnIds {+ fn insert(&mut self, provider_turn_id: String) -> bool {+ if self.contains(&provider_turn_id) {+ return true;+ }+ if self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS {+ return false;+ }+ self.ids.insert(provider_turn_id)+ }++ fn contains(&self, provider_turn_id: &str) -> bool {+ self.ids.contains(provider_turn_id)+ }++ fn at_capacity(&self) -> bool {+ self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS || !self.filter.is_empty()+ }++ fn restore(&mut self, provider_turn_id: String) -> Result<(), LocalRunnerError> {+ if !self.insert(provider_turn_id) {+ return Err(LocalRunnerError::invalid(+ "Codex restored provider turn identity epoch exceeded its exact capacity",+ ));+ }+ Ok(())+ }++ fn restore_all(+ &mut self,+ provider_turn_ids: impl IntoIterator<Item = String>,+ replay_filter: DurableReplayFilter,+ ) -> Result<(), LocalRunnerError> {+ replay_filter+ .validate()+ .map_err(|error| LocalRunnerError::invalid(error.to_string()))?;+ self.filter = replay_filter;+ for provider_turn_id in provider_turn_ids {+ if !self.insert(provider_turn_id) {+ return Err(LocalRunnerError::invalid(+ "Codex restored provider turn identity epoch exceeded its exact capacity",+ ));+ }+ }+ Ok(())+ }+}+enum ProviderRequestError {Rejected(LocalRunnerError),Ambiguous(LocalRunnerError),}+#[derive(Clone, Debug, PartialEq, Eq)]+pub(crate) enum RejectedAcceptedTurn {+ ReusedIdentity(String),+ InvalidIdentity,+}+impl ProviderRequestError {fn into_inner(self) -> LocalRunnerError {match self {@@ -159,10 +225,12 @@ struct PendingToolRequest {#[derive(Clone, Debug, PartialEq)]struct PendingRuntimeRequest {rpc_id: Value,+ turn_id: String,method: String,params: Value,question_set: Value,option_labels: QuestionOptionLabels,+ retained_bytes: usize,}struct BufferedProviderMessage {@@ -178,22 +246,32 @@ enum AmbiguousTurnMessage {pub struct CodexProvider {process: SupervisedProcess,+ config: CodexProviderConfig,+ authorized_tools: Vec<AuthorizedTool>,next_request_id: u64,thread_id: String,provider_session_id: Option<String>,active_provider_turn_id: Option<String>,pending_messages: VecDeque<BufferedProviderMessage>,deferred_ambiguous_messages: VecDeque<BufferedProviderMessage>,+ pending_message_bytes: usize,authorized_tool_ids: BTreeSet<String>,pending_tool_requests: BTreeMap<String, PendingToolRequest>,completed_tool_call_ids: BTreeSet<String>,+ durable_tool_call_replays: bool,pending_tool_request_bytes: usize,pending_runtime_requests: BTreeMap<String, PendingRuntimeRequest>,+ pending_runtime_request_bytes: usize,+ runtime_request_scope: [u8; 16],+ next_runtime_request_sequence: u64,expected_shutdown: bool,process_generation: u64,completed_turn_authority: Option<CompletedTurnAuthority>,completion_reconciliation_pending: bool,ambiguous_turn_start_pending: bool,+ settled_provider_turn_ids: SettledProviderTurnIds,+ rejected_accepted_turn: Option<RejectedAcceptedTurn>,+ quarantined: bool,}impl CodexProvider {@@ -212,20 +290,7 @@ impl CodexProvider {Self::start_with_tools_for_generation(config, authorized_tools, resume_thread_id, 1)}- pub(crate) fn start_for_generation(- config: &CodexProviderConfig,- resume_thread_id: Option<&str>,- process_generation: u64,- ) -> Result<Self, LocalRunnerError> {- Self::start_with_tools_for_generation(- config,- std::iter::empty(),- resume_thread_id,- process_generation,- )- }-- fn start_with_tools_for_generation(+ pub(crate) fn start_with_tools_for_generation(config: &CodexProviderConfig,authorized_tools: impl IntoIterator<Item = AuthorizedTool>,resume_thread_id: Option<&str>,@@ -237,7 +302,9 @@ impl CodexProvider {"Codex process generation must be positive",));}- let (dynamic_tools, authorized_tool_ids) = codex_dynamic_tools(authorized_tools)?;+ let authorized_tools = authorized_tools.into_iter().collect::<Vec<_>>();+ let (dynamic_tools, authorized_tool_ids) =+ codex_dynamic_tools(authorized_tools.iter().cloned())?;let mut provider = Self {process: SupervisedProcess::spawn(&config.command,@@ -245,22 +312,32 @@ impl CodexProvider {Duration::from_secs(2),CODEX_APP_SERVER_MAX_FRAME_BYTES,)?,+ config: config.clone(),+ authorized_tools,next_request_id: 1,thread_id: String::new(),provider_session_id: None,active_provider_turn_id: None,pending_messages: VecDeque::new(),deferred_ambiguous_messages: VecDeque::new(),+ pending_message_bytes: 0,authorized_tool_ids,pending_tool_requests: BTreeMap::new(),completed_tool_call_ids: BTreeSet::new(),+ durable_tool_call_replays: false,pending_tool_request_bytes: 0,pending_runtime_requests: BTreeMap::new(),+ pending_runtime_request_bytes: 0,+ runtime_request_scope: new_runtime_request_scope()?,+ next_runtime_request_sequence: 1,expected_shutdown: false,process_generation,completed_turn_authority: None,completion_reconciliation_pending: false,ambiguous_turn_start_pending: false,+ settled_provider_turn_ids: SettledProviderTurnIds::default(),+ rejected_accepted_turn: None,+ quarantined: false,};let initialized = provider.request("initialize",@@ -320,7 +397,7 @@ impl CodexProvider {if resume_thread_id.is_some() {let snapshot = provider.read_thread()?;provider.active_provider_turn_id = latest_active_turn_id(&snapshot)- .map(|turn_id| bounded_identifier(Some(&turn_id), "Codex turn id"))+ .map(|provider_turn_id| bounded_provider_turn_id(Some(&provider_turn_id))).transpose()?;}Ok(provider)@@ -330,6 +407,10 @@ impl CodexProvider {self.process.id()}+ pub(crate) fn process_generation(&self) -> u64 {+ self.process_generation+ }+pub fn thread_id(&self) -> &str {&self.thread_id}@@ -346,12 +427,19 @@ impl CodexProvider {self.ambiguous_turn_start_pending}+ pub(crate) fn enable_durable_tool_call_replays(&mut self) {+ // The durable backend validates the call id, operation, and input+ // against its persisted receipt before returning a stored result.+ // Direct provider consumers retain the stricter one-shot behavior.+ self.durable_tool_call_replays = true;+ }+pub(crate) fn restore_completed_turn_authority(&mut self,authoritative: bool,process_generation: Option<u64>,provider_turn_id: Option<&str>,- ) {+ ) -> Result<(), LocalRunnerError> {self.completed_turn_authority = authoritative.then(|| CompletedTurnAuthority {// Legacy state did not record the generation. Generation zero is// deliberately older than every supervised process generation.@@ -360,6 +448,10 @@ impl CodexProvider {.unwrap_or("durable-completed-turn").to_owned(),});+ if let Some(authority) = self.completed_turn_authority.as_ref() {+ self.settled_provider_turn_ids+ .restore(authority.provider_turn_id.clone())?;+ }// Resuming a completed durable thread and reading its provider state// is recovery, not new turn work. Keep the prior terminal authoritative// until start_turn explicitly revokes it.@@ -369,6 +461,16 @@ impl CodexProvider {// of them supersedes a completed result. Only accepting a replacement// turn identity revokes this authority.self.completion_reconciliation_pending = false;+ Ok(())+ }++ pub(crate) fn restore_settled_turn_identities(+ &mut self,+ provider_turn_ids: impl IntoIterator<Item = String>,+ replay_filter: DurableReplayFilter,+ ) -> Result<(), LocalRunnerError> {+ self.settled_provider_turn_ids+ .restore_all(provider_turn_ids, replay_filter)}pub(crate) fn completed_turn_authority(&self) -> Option<(u64, &str)> {@@ -380,7 +482,87 @@ impl CodexProvider {})}+ pub(crate) fn take_rejected_accepted_turn(&mut self) -> Option<RejectedAcceptedTurn> {+ self.rejected_accepted_turn.take()+ }++ pub(crate) fn restart_idle_identity_epoch(&mut self) -> Result<(), LocalRunnerError> {+ if self.active_provider_turn_id.is_some() || self.ambiguous_turn_start_pending {+ return Err(LocalRunnerError::invalid(+ "Codex provider identity epoch cannot rotate while work is active",+ ));+ }++ let next_generation = self.process_generation.checked_add(1).ok_or_else(|| {+ LocalRunnerError::invalid("Codex process generation exhausted during epoch rollover")+ })?;+ let config = self.config.clone();+ let authorized_tools = self.authorized_tools.clone();+ let thread_id = self.thread_id.clone();+ let completed_turn_authority = self.completed_turn_authority.clone();+ let completion_reconciliation_pending = self.completion_reconciliation_pending;+ let durable_tool_call_replays = self.durable_tool_call_replays;++ // Exact turn identities may be forgotten only after the provider+ // process that could emit them is gone. Resume the same thread in a+ // fresh process generation, then preserve prior completion authority+ // until a replacement turn identity is actually accepted.+ self.shutdown()?;+ let mut replacement = Self::start_with_tools_for_generation(+ &config,+ authorized_tools,+ Some(&thread_id),+ next_generation,+ )?;+ replacement.durable_tool_call_replays = durable_tool_call_replays;+ if replacement.active_provider_turn_id.is_some() {+ // A terminal can race the provider's own durable idle-state write.+ // This process has work Paperclip never dispatched in the new+ // epoch, so revoke its request authority and reap it. Retain the+ // exact ledger for diagnostics, but never expose the unexpected+ // turn as ordinary active work or admit a replacement.+ replacement.settled_provider_turn_ids =+ std::mem::take(&mut self.settled_provider_turn_ids);+ replacement.active_provider_turn_id = None;+ replacement.ambiguous_turn_start_pending = true;+ replacement.rejected_accepted_turn = Some(RejectedAcceptedTurn::InvalidIdentity);+ replacement.quarantined = true;+ replacement.pending_messages.clear();+ replacement.deferred_ambiguous_messages.clear();+ replacement.pending_message_bytes = 0;+ let _ = replacement.cancel_pending_requests();+ let _ = replacement.process.terminate_group();+ replacement.expected_shutdown = false;+ *self = replacement;+ return Err(LocalRunnerError::invalid(+ "Codex provider epoch rollover resumed unowned active work; the provider was terminated",+ ));+ }+ if let Some(authority) = completed_turn_authority.as_ref() {+ replacement.restore_completed_turn_authority(+ true,+ Some(authority.process_generation),+ Some(&authority.provider_turn_id),+ )?;+ }+ replacement.completion_reconciliation_pending = completion_reconciliation_pending;+ *self = replacement;+ Ok(())+ }++ fn rollover_settled_turn_epoch_if_needed(&mut self) -> Result<(), LocalRunnerError> {+ if !self.settled_provider_turn_ids.at_capacity() {+ return Ok(());+ }+ self.restart_idle_identity_epoch()+ }+pub fn start_turn(&mut self, message: &str, cwd: &str) -> Result<Value, LocalRunnerError> {+ if self.quarantined {+ return Err(LocalRunnerError::invalid(+ "Codex provider is quarantined after unsafe recovered work",+ ));+ }if self.active_provider_turn_id.is_some() {return Err(LocalRunnerError::invalid("Codex already has an active provider turn",@@ -396,6 +578,7 @@ impl CodexProvider {"Codex turn text is empty or exceeds the 1 MiB limit",));}+ self.rollover_settled_turn_epoch_if_needed()?;// Preserve the prior durable result until a replacement turn identity// is accepted. A rejected, ambiguous, or transport-failed attempt does// not prove that replacement work superseded the completed turn.@@ -403,6 +586,7 @@ impl CodexProvider {self.completion_reconciliation_pending = false;let prior_buffered_message_count = self.pending_messages.len();self.ambiguous_turn_start_pending = true;+ let runtime_request_scope = new_runtime_request_scope()?;let result = match self.request_classified("turn/start",json!({@@ -433,16 +617,32 @@ impl CodexProvider {}Err(ProviderRequestError::Ambiguous(error)) => return Err(error),};- let provider_turn_id = bounded_identifier(- result- .pointer("/turn/id")- .or_else(|| result.get("turnId"))- .and_then(Value::as_str),- "Codex turn id",- )?;+ let provider_turn_id = result+ .pointer("/turn/id")+ .or_else(|| result.get("turnId"))+ .and_then(Value::as_str);+ let provider_turn_id = match bounded_provider_turn_id(provider_turn_id) {+ Ok(provider_turn_id) => provider_turn_id,+ Err(error) => {+ // A successful turn/start response means the provider may+ // already be executing the work. Without a bounded identity,+ // runnerd cannot durably bind, interrupt, or reconcile it.+ // Terminate the process and let the durable backend close the+ // run before returning the protocol error.+ self.rejected_accepted_turn = Some(RejectedAcceptedTurn::InvalidIdentity);+ self.expected_shutdown = true;+ self.completed_turn_authority = None;+ let _ = self.cancel_pending_requests();+ let _ = self.process.terminate_group();+ return Err(error);+ }+ };+ if self.settled_provider_turn_ids.contains(&provider_turn_id) {+ return Err(self.reject_accepted_reused_turn_identity(provider_turn_id));+ }// Only a validated provider turn identity proves that replacement// work exists and supersedes the prior completed result.- self.accept_replacement_turn(provider_turn_id);+ self.accept_replacement_turn(provider_turn_id, runtime_request_scope);Ok(result)}@@ -471,17 +671,12 @@ impl CodexProvider {))})?;- if self- .completed_turn_authority- .as_ref()- .is_some_and(|authority| authority.provider_turn_id == provider_turn_id)- {- return Err(LocalRunnerError::invalid(format!(- "Codex {method} notification reused the previously completed turn id while resolving an ambiguous turn start"- )));+ if self.settled_provider_turn_ids.contains(&provider_turn_id) {+ return Err(self.reject_accepted_reused_turn_identity(provider_turn_id));}- self.accept_replacement_turn(provider_turn_id.clone());+ let runtime_request_scope = new_runtime_request_scope()?;+ self.accept_replacement_turn(provider_turn_id.clone(), runtime_request_scope);if method == "turn/started" && message.get("id").is_none() {Ok(AmbiguousTurnMessage::ReconciledWithStart)} else {@@ -489,13 +684,39 @@ impl CodexProvider {}}- fn accept_replacement_turn(&mut self, provider_turn_id: String) {+ fn accept_replacement_turn(+ &mut self,+ provider_turn_id: String,+ runtime_request_scope: [u8; 16],+ ) {self.ambiguous_turn_start_pending = false;self.expected_shutdown = false;self.completed_turn_authority = None;self.completion_reconciliation_pending = false;self.completed_tool_call_ids.clear();+ // Retain the prior settled identity while the next turn runs. Besides+ // recognizing delayed prior-turn requests, this fails closed if a+ // provider ambiguously reuses the same turn id for fresh work.self.active_provider_turn_id = Some(provider_turn_id);+ self.runtime_request_scope = runtime_request_scope;+ }++ fn reject_accepted_reused_turn_identity(+ &mut self,+ provider_turn_id: String,+ ) -> LocalRunnerError {+ // Both a successful response and identity-bearing evidence after an+ // ambiguous response prove the provider accepted work. A settled+ // identity cannot durably own that work, so terminate its process+ // before returning instead of leaving an untracked turn alive.+ self.rejected_accepted_turn = Some(RejectedAcceptedTurn::ReusedIdentity(provider_turn_id));+ self.expected_shutdown = true;+ self.completed_turn_authority = None;+ let _ = self.cancel_pending_requests();+ let _ = self.process.terminate_group();+ LocalRunnerError::invalid(+ "Codex reused a settled provider turn identity after accepting work; the provider was terminated",+ )}pub fn steer_turn(&mut self, message: &str) -> Result<Value, LocalRunnerError> {@@ -554,15 +775,84 @@ impl CodexProvider {.ok_or_else(|| {LocalRunnerError::invalid("runtime response has no pending Codex request")})?;+ if self.active_provider_turn_id.as_deref() != Some(pending.turn_id.as_str()) {+ return Err(LocalRunnerError::invalid(+ "runtime response belongs to another Codex turn",+ ));+ }let result = codex_question_response(&pending, response)?;self.process.send(&json!({"id": pending.rpc_id, "result": result}))?;- self.pending_runtime_requests.remove(request_id);+ if let Some(completed) = self.pending_runtime_requests.remove(request_id) {+ self.pending_runtime_request_bytes = self+ .pending_runtime_request_bytes+ .saturating_sub(completed.retained_bytes);+ }Ok(())}+ fn reject_post_terminal_request(+ &mut self,+ rpc_id: Value,+ method: &str,+ ) -> Result<Option<CodexProviderEvent>, LocalRunnerError> {+ let message = format!(+ "ignored delayed {} request after the Codex turn terminated",+ bounded_method(method)+ );+ let response = if method == "item/tool/call" {+ json!({+ "id": rpc_id,+ "result": codex_tool_failure("the Codex turn has already terminated"),+ })+ } else {+ json!({+ "id": rpc_id,+ "error": {"code": -32000, "message": "the Codex turn has already terminated"},+ })+ };+ // The terminal notification is already authoritative and may be+ // waiting in the durable outbox. A courtesy rejection must not turn a+ // provider that has closed stdin into a fatal polling error.+ let _ = self.process.send(&response);+ Ok(Some(CodexProviderEvent::Notification {+ method: "warning".to_owned(),+ params: json!({"message": message, "providerMethod": bounded_method(method)}),+ }))+ }+pub fn poll(&mut self) -> Result<Option<CodexProviderEvent>, LocalRunnerError> {+ if self.quarantined {+ // Never interpret provider-originated requests after fail-closed+ // quarantine. Drain output only so process termination cannot+ // deadlock on a full pipe, then surface an unequivocal failure.+ if self+ .process+ .receive_stdout_line(Duration::from_millis(1))?+ .is_some()+ {+ return Ok(None);+ }+ return Ok(self+ .process+ .try_wait()?+ .map(|exit| CodexProviderEvent::Exited {+ exit_code: exit.exit_code,+ success: false,+ completed_turn_authoritative: false,+ completed_turn_observed_by_process: false,+ completion_reconciles_exit: false,+ process_generation: self.process_generation,+ completed_turn_process_generation: None,+ }));+ }let buffered = self.pending_messages.pop_front();+ if let Some(buffered) = buffered.as_ref() {+ self.pending_message_bytes = self.pending_message_bytes.saturating_sub(json_size(+ &buffered.value,+ "buffered Codex provider message",+ )?);+ }let message = if let Some(buffered) = buffered {buffered.value} else {@@ -625,11 +915,24 @@ impl CodexProvider {match self.classify_ambiguous_turn_message(&message)? {AmbiguousTurnMessage::Ready => {}AmbiguousTurnMessage::Deferred => {- if self.deferred_ambiguous_messages.len() >= MAX_BUFFERED_MESSAGES {+ if self+ .pending_messages+ .len()+ .saturating_add(self.deferred_ambiguous_messages.len())+ >= MAX_BUFFERED_MESSAGES+ {return Err(LocalRunnerError::invalid("Codex emitted too many messages before resolving an ambiguous turn start",));}+ let retained_bytes = json_size(&message, "buffered Codex provider message")?;+ self.pending_message_bytes =+ retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes)+ .ok_or_else(|| {+ LocalRunnerError::invalid(+ "Codex buffered messages exceed the 16 MiB aggregate limit",+ )+ })?;self.deferred_ambiguous_messages.push_back(BufferedProviderMessage { value: message });return Ok(None);@@ -641,6 +944,14 @@ impl CodexProvider {}AmbiguousTurnMessage::ReconciledNeedsStart { provider_turn_id } => {let mut replay = std::mem::take(&mut self.deferred_ambiguous_messages);+ let retained_bytes = json_size(&message, "buffered Codex provider message")?;+ self.pending_message_bytes =+ retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes)+ .ok_or_else(|| {+ LocalRunnerError::invalid(+ "Codex buffered messages exceed the 16 MiB aggregate limit",+ )+ })?;replay.push_back(BufferedProviderMessage { value: message });replay.append(&mut self.pending_messages);self.pending_messages = replay;@@ -666,6 +977,13 @@ impl CodexProvider {"Codex tool call named another thread",));}+ if request_targets_non_active_turn(+ self.active_provider_turn_id.as_deref(),+ &self.settled_provider_turn_ids,+ ¶ms,+ ) {+ return self.reject_post_terminal_request(rpc_id, method);+ }let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| {LocalRunnerError::invalid("Codex tool call arrived outside an active turn")})?;@@ -719,7 +1037,8 @@ impl CodexProvider {input: input.clone(),retained_bytes,};- if self.completed_tool_call_ids.contains(&call_id) {+ let completed_replay = self.completed_tool_call_ids.contains(&call_id);+ if completed_replay && !self.durable_tool_call_replays {return Err(LocalRunnerError::invalid("Codex reused a completed tool call id",));@@ -746,7 +1065,9 @@ impl CodexProvider {"Codex emitted too many pending tool calls",));}- if self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS {+ if !completed_replay+ && self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS+ {return Err(LocalRunnerError::invalid("Codex emitted too many completed tool calls in one turn",));@@ -770,36 +1091,93 @@ impl CodexProvider {"Codex runtime request named another thread",));}- let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| {+ if request_targets_non_active_turn(+ self.active_provider_turn_id.as_deref(),+ &self.settled_provider_turn_ids,+ ¶ms,+ ) {+ return self.reject_post_terminal_request(rpc_id, method);+ }+ let active_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| {LocalRunnerError::invalid("Codex runtime request arrived outside an active turn",)})?;- if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id) {+ if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id.as_str()) {return Err(LocalRunnerError::invalid("Codex runtime request named another turn",));}- let (request_id, question_set, option_labels) =+ let (provider_request_id, question_set, option_labels) =codex_question_set(&rpc_id, ¶ms)?;+ let retained_bytes = pending_runtime_request_size(+ &rpc_id,+ &active_turn_id,+ method,+ ¶ms,+ &question_set,+ &option_labels,+ )?;let pending = PendingRuntimeRequest {- rpc_id,+ rpc_id: rpc_id.clone(),+ turn_id: active_turn_id.clone(),method: method.to_owned(),params,question_set: question_set.clone(),option_labels,+ retained_bytes,};- if let Some(existing) = self.pending_runtime_requests.get(&request_id) {+ if let Some(existing) = self+ .pending_runtime_requests+ .values()+ .find(|existing| existing.rpc_id == rpc_id)+ {if existing != &pending {return Err(LocalRunnerError::invalid("Codex reused a runtime request id with different input",));}return Ok(None);- } else {- self.pending_runtime_requests- .insert(request_id.clone(), pending);}+ let retained_request_bytes = retain_pending_runtime_request_bytes(+ self.pending_runtime_request_bytes,+ retained_bytes,+ );+ if self.pending_runtime_requests.len() >= MAX_PENDING_RUNTIME_REQUESTS+ || retained_request_bytes.is_none()+ {+ self.process.send(&json!({+ "id": rpc_id,+ "error": {+ "code": -32000,+ "message": "Paperclip rejected this runtime request because the pending input capacity was reached",+ },+ }))?;+ return Ok(Some(CodexProviderEvent::Notification {+ method: "warning".to_owned(),+ params: json!({+ "message": "rejected a Codex runtime request at the bounded pending-input limit",+ "providerMethod": "item/tool/requestUserInput",+ }),+ }));+ }+ let request_sequence = self.next_runtime_request_sequence;+ self.next_runtime_request_sequence = self+ .next_runtime_request_sequence+ .checked_add(1)+ .ok_or_else(|| {+ LocalRunnerError::invalid("Codex runtime request sequence overflowed")+ })?;+ let request_id = scoped_runtime_request_id(+ &self.runtime_request_scope,+ &active_turn_id,+ &provider_request_id,+ request_sequence,+ );+ self.pending_runtime_requests+ .insert(request_id.clone(), pending);+ self.pending_runtime_request_bytes =+ retained_request_bytes.expect("bounded runtime request bytes checked above");return Ok(Some(CodexProviderEvent::RuntimeRequest {request_id,question_set,@@ -817,24 +1195,60 @@ impl CodexProvider {if let Some(method) = message.get("method").and_then(Value::as_str) {let params = message.get("params").cloned().unwrap_or(Value::Null);+ let terminal_event_type = normalized_codex_terminal_event_type(method, ¶ms);+ let notification_turn_id = params+ .get("turnId")+ .or_else(|| params.pointer("/turn/id"))+ .and_then(Value::as_str);+ if terminal_event_type.is_some()+ && notification_turn_id.is_some()+ && notification_turn_id != self.active_provider_turn_id.as_deref()+ && notification_turn_id+ .is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id))+ {+ return Ok(Some(CodexProviderEvent::Notification {+ method: "warning".to_owned(),+ params: json!({+ "message": "ignored a terminal notification for a non-active Codex turn",+ "providerMethod": bounded_method(method),+ }),+ }));+ }validate_notification_binding(&self.thread_id,self.active_provider_turn_id.as_deref(),¶ms,)?;- if method == "turn/completed" {- let provider_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| {- LocalRunnerError::invalid(- "Codex completion arrived outside an active provider turn",- )- })?;+ if let Some(terminal_event_type) = terminal_event_type {+ if self.active_provider_turn_id.is_none() {+ return Err(LocalRunnerError::invalid(+ "Codex terminal arrived outside an active provider turn",+ ));+ }+ let provider_turn_id = self+ .active_provider_turn_id+ .clone()+ .expect("active provider turn checked above");+ let completed_turn_authority = if terminal_event_type == "turn.completed" {+ Some(CompletedTurnAuthority {+ process_generation: self.process_generation,+ provider_turn_id: provider_turn_id.clone(),+ })+ } else {+ None+ };+ if !self+ .settled_provider_turn_ids+ .insert(provider_turn_id.clone())+ {+ return Err(LocalRunnerError::invalid(+ "Codex provider turn identity epoch reached its exact capacity",+ ));+ }self.active_provider_turn_id = None;self.expected_shutdown = true;- self.completed_turn_authority = Some(CompletedTurnAuthority {- process_generation: self.process_generation,- provider_turn_id,- });- self.completion_reconciliation_pending = true;+ self.completed_turn_authority = completed_turn_authority;+ self.completion_reconciliation_pending = terminal_event_type == "turn.completed";// The provider terminal is authoritative once received. Clear// local request ownership and attempt courtesy responses, but// a provider that already closed stdin must not turn the@@ -898,6 +1312,7 @@ impl CodexProvider {fn cancel_pending_requests(&mut self) -> Result<(), LocalRunnerError> {let pending_runtime = std::mem::take(&mut self.pending_runtime_requests);let pending = std::mem::take(&mut self.pending_tool_requests);+ self.pending_runtime_request_bytes = 0;self.pending_tool_request_bytes = 0;let mut first_error = None;for request in pending_runtime.into_values() {@@ -967,17 +1382,44 @@ impl CodexProvider {}return Ok(message.get("result").cloned().unwrap_or(Value::Null));}- if self.pending_messages.len() >= MAX_BUFFERED_MESSAGES {+ if self+ .pending_messages+ .len()+ .saturating_add(self.deferred_ambiguous_messages.len())+ >= MAX_BUFFERED_MESSAGES+ {return Err(ProviderRequestError::Ambiguous(LocalRunnerError::invalid("Codex emitted too many messages before a request response",)));}+ let retained_bytes = json_size(&message, "buffered Codex provider message")+ .map_err(ProviderRequestError::Ambiguous)?;+ let next_retained_bytes =+ retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes)+ .ok_or_else(|| {+ ProviderRequestError::Ambiguous(LocalRunnerError::invalid(+ "Codex buffered messages exceed the 16 MiB aggregate limit",+ ))+ })?;self.pending_messages.push_back(BufferedProviderMessage { value: message });+ self.pending_message_bytes = next_retained_bytes;}}}+fn json_size(value: &Value, label: &str) -> Result<usize, LocalRunnerError> {+ serde_json::to_vec(value)+ .map(|bytes| bytes.len())+ .map_err(|error| LocalRunnerError::invalid(format!("{label} is not serializable: {error}")))+}++fn retain_buffered_message_bytes(current: usize, incoming: usize) -> Option<usize> {+ current+ .checked_add(incoming)+ .filter(|total| *total <= MAX_BUFFERED_MESSAGE_BYTES)+}+fn pending_tool_request_size(parts: impl IntoIterator<Item = usize>,) -> Result<usize, LocalRunnerError> {@@ -1002,6 +1444,29 @@ fn retain_pending_tool_request_bytes(})}+fn pending_runtime_request_size(+ rpc_id: &Value,+ turn_id: &str,+ method: &str,+ params: &Value,+ question_set: &Value,+ option_labels: &QuestionOptionLabels,+) -> Result<usize, LocalRunnerError> {+ serde_json::to_vec(&(rpc_id, turn_id, method, params, question_set, option_labels))+ .map(|encoded| encoded.len())+ .map_err(|error| {+ LocalRunnerError::invalid(format!(+ "Codex pending runtime request is not serializable: {error}"+ ))+ })+}++fn retain_pending_runtime_request_bytes(current: usize, incoming: usize) -> Option<usize> {+ current+ .checked_add(incoming)+ .filter(|total| *total <= MAX_PENDING_RUNTIME_REQUEST_BYTES)+}+fn codex_dynamic_tools(authorized_tools: impl IntoIterator<Item = AuthorizedTool>,) -> Result<(Vec<Value>, BTreeSet<String>), LocalRunnerError> {@@ -1079,12 +1544,31 @@ fn codex_dynamic_tools(fn bounded_identifier(value: Option<&str>, label: &str) -> Result<String, LocalRunnerError> {let value = value.ok_or_else(|| LocalRunnerError::invalid(format!("{label} is required")))?;- if value.is_empty() || value.len() > 160 || value.chars().any(char::is_control) {+ let mut characters = value.chars();+ let valid_first = characters+ .next()+ .is_some_and(|character| character.is_ascii_alphanumeric());+ let valid_rest = characters.all(|character| {+ character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '.' | ':')+ });+ if value.len() > 160 || !valid_first || !valid_rest {return Err(LocalRunnerError::invalid(format!("{label} is invalid")));}Ok(value.to_owned())}+fn bounded_provider_turn_id(value: Option<&str>) -> Result<String, LocalRunnerError> {+ let value = value+ .filter(|value| !value.is_empty())+ .ok_or_else(|| LocalRunnerError::invalid("Codex turn/start omitted turn.id"))?;+ if value.len() > 240 || value.chars().any(char::is_control) {+ return Err(LocalRunnerError::invalid(+ "Codex turn/start returned an invalid turn.id",+ ));+ }+ Ok(value.to_owned())+}+fn codex_tool_failure(message: &str) -> Value {json!({"success": false,@@ -1160,6 +1644,17 @@ fn notification_turn_id(params: &Value) -> Option<&str> {.filter(|value| !value.is_empty())}+fn request_targets_non_active_turn(+ active_turn_id: Option<&str>,+ settled_turn_ids: &SettledProviderTurnIds,+ params: &Value,+) -> bool {+ let requested_turn_id = params.get("turnId").and_then(Value::as_str);+ requested_turn_id.is_some_and(|requested| {+ settled_turn_ids.contains(requested) || active_turn_id != Some(requested)+ })+}+fn latest_active_turn_id(snapshot: &Value) -> Option<String> {snapshot.pointer("/thread/turns")@@ -1296,6 +1791,33 @@ fn codex_question_set())}+fn new_runtime_request_scope() -> Result<[u8; 16], LocalRunnerError> {+ let mut scope = [0u8; 16];+ getrandom::fill(&mut scope).map_err(|error| {+ LocalRunnerError::invalid(format!(+ "failed to mint Codex runtime request scope: {error}"+ ))+ })?;+ Ok(scope)+}++fn scoped_runtime_request_id(+ scope: &[u8; 16],+ turn_id: &str,+ provider_request_id: &str,+ request_sequence: u64,+) -> String {+ let mut digest = Sha256::new();+ digest.update(scope);+ digest.update([0]);+ digest.update(turn_id.as_bytes());+ digest.update([0]);+ digest.update(provider_request_id.as_bytes());+ digest.update([0]);+ digest.update(request_sequence.to_be_bytes());+ format!("runtime-request-{:x}", digest.finalize())+}+fn codex_question_response(pending: &PendingRuntimeRequest,response: &Value,@@ -1436,10 +1958,12 @@ mod tests {assert_eq!(question_set["schema"], "paperclip.question_set.v1");let pending = PendingRuntimeRequest {rpc_id: json!(41),+ turn_id: "turn-1".to_owned(),method: "item/tool/requestUserInput".to_owned(),params: Value::Null,question_set,option_labels: labels,+ retained_bytes: 0,};let native = codex_question_response(&pending,@@ -1461,6 +1985,23 @@ mod tests {}),).is_err());+ let scope = [7u8; 16];+ assert_eq!(+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),+ );+ assert_ne!(+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),+ scoped_runtime_request_id(&scope, "turn-2", "41", 1),+ );+ assert_ne!(+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),+ scoped_runtime_request_id(&[8u8; 16], "turn-1", "41", 1),+ );+ assert_ne!(+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),+ scoped_runtime_request_id(&scope, "turn-1", "41", 2),+ );assert!(codex_question_response(&pending,&json!({@@ -1503,6 +2044,87 @@ mod tests {.is_ok());}+ #[test]+ fn rejects_requests_bound_to_any_non_active_turn_nonfatally() {+ let mut turn_one_settled = SettledProviderTurnIds::default();+ turn_one_settled.insert("turn-1".to_owned());+ let mut turn_two_settled = SettledProviderTurnIds::default();+ turn_two_settled.insert("turn-2".to_owned());+ let no_settled_turns = SettledProviderTurnIds::default();+ assert!(request_targets_non_active_turn(+ Some("turn-2"),+ &turn_one_settled,+ &json!({"turnId": "turn-1"}),+ ));+ assert!(request_targets_non_active_turn(+ Some("turn-2"),+ &turn_one_settled,+ &json!({"turnId": "turn-0"}),+ ));+ assert!(request_targets_non_active_turn(+ None,+ &turn_two_settled,+ &json!({"turnId": "turn-1"}),+ ));+ assert!(request_targets_non_active_turn(+ Some("turn-1"),+ &turn_one_settled,+ &json!({"turnId": "turn-1"}),+ ));+ assert!(!request_targets_non_active_turn(+ Some("turn-2"),+ &turn_one_settled,+ &json!({"turnId": "turn-2"}),+ ));+ assert!(!request_targets_non_active_turn(+ None,+ &no_settled_turns,+ &json!({}),+ ));+ }++ #[test]+ fn settled_provider_turn_history_never_evicts_exact_identities() {+ let mut settled = SettledProviderTurnIds::default();+ for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS {+ assert!(settled.insert(format!("turn-{index}")));+ }++ assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS);+ assert!(settled.contains("turn-0"));+ assert!(settled.contains("turn-1"));+ assert!(settled.at_capacity());+ assert!(!settled.insert(format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}")));+ assert!(!settled.contains(&format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}")));+ assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS);+ assert!(settled.contains("turn-0"));+ assert!(settled.filter.is_empty());+ }++ #[test]+ fn restored_provider_turn_identity_is_retained_in_release_builds() {+ let mut settled = SettledProviderTurnIds::default();++ settled.restore("turn-restored".to_owned()).unwrap();++ assert!(settled.contains("turn-restored"));+ }++ #[test]+ fn restores_the_complete_durable_provider_turn_ledger() {+ let mut settled = SettledProviderTurnIds::default();++ settled+ .restore_all(+ ["turn-older".to_owned(), "turn-latest".to_owned()],+ DurableReplayFilter::default(),+ )+ .unwrap();++ assert!(settled.contains("turn-older"));+ assert!(settled.contains("turn-latest"));+ }+#[test]fn bounds_all_retained_pending_tool_request_data_in_aggregate() {let request_bytes = pending_tool_request_size([1, 2, 3, 4]).unwrap();@@ -1516,4 +2138,30 @@ mod tests {assert!(retain_pending_tool_request_bytes(usize::MAX, 1).is_err());assert!(pending_tool_request_size([usize::MAX, 1]).is_err());}++ #[test]+ fn bounds_all_retained_runtime_request_data_in_aggregate() {+ assert_eq!(+ retain_pending_runtime_request_bytes(MAX_PENDING_RUNTIME_REQUEST_BYTES - 10, 10,),+ Some(MAX_PENDING_RUNTIME_REQUEST_BYTES)+ );+ assert_eq!(+ retain_pending_runtime_request_bytes(MAX_PENDING_RUNTIME_REQUEST_BYTES, 1),+ None+ );+ assert_eq!(retain_pending_runtime_request_bytes(usize::MAX, 1), None);+ }++ #[test]+ fn bounds_messages_buffered_while_waiting_for_a_response() {+ assert_eq!(+ retain_buffered_message_bytes(MAX_BUFFERED_MESSAGE_BYTES - 10, 10),+ Some(MAX_BUFFERED_MESSAGE_BYTES)+ );+ assert_eq!(+ retain_buffered_message_bytes(MAX_BUFFERED_MESSAGE_BYTES, 1),+ None+ );+ assert_eq!(retain_buffered_message_bytes(usize::MAX, 1), None);+ }}diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rsindex ef3ec61456..3bbf2f9311 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs@@ -9,12 +9,14 @@ use std::time::Duration;pub use runner::{run_durable_runner, CommandExecution, CommandExecutor, PolledEvent};pub(crate) use state::{- create_private_temporary_file, open_private_regular_file, redact_text, verify_private_directory,+ create_private_temporary_file, open_private_regular_file, redact_text, sanitize_value,+ verify_private_directory,};pub use state::{Command, CommandDisposition, DurableState, DurableStateStore, EventPriority,StoredCommandResult, StoredOutboxEvent,};+pub(crate) use transport::current_unix_ms;pub const PROTOCOL: &str = "paperclip.runner";pub const PROTOCOL_VERSION: u64 = 1;diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rsindex 50c6df265c..0e27f28f3c 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs@@ -1078,7 +1078,23 @@ fn sensitive_key(key: &str) -> bool {.any(|needle| normalized.contains(needle))}-fn sanitize_value(value: &Value) -> Value {+fn protocol_authorization_boundary(key: &str, value: &Value) -> bool {+ key.eq_ignore_ascii_case("authorizationBoundary")+ && value.as_str().is_some_and(|boundary| {+ matches!(+ boundary,+ "company"+ | "actor"+ | "active_task"+ | "grant"+ | "governed_action"+ | "lock"+ | "revision"+ )+ })+}++pub(crate) fn sanitize_value(value: &Value) -> Value {match value {Value::Object(object) => Value::Object(object@@ -1086,7 +1102,9 @@ fn sanitize_value(value: &Value) -> Value {.map(|(key, value)| {(key.clone(),- if sensitive_key(key) {+ if protocol_authorization_boundary(key, value) {+ value.clone()+ } else if sensitive_key(key) {Value::String("[REDACTED]".to_owned())} else {sanitize_value(value)@@ -1323,6 +1341,21 @@ mod tests {);}+ #[test]+ fn protocol_authorization_boundary_is_not_redacted_as_a_credential() {+ let sanitized = sanitize_value(&json!({+ "authorizationBoundary": "active_task",+ "nested": {"authorizationBoundary": "Bearer secret-value"},+ "authorization": "Bearer secret-value",+ }));+ assert_eq!(sanitized["authorizationBoundary"], json!("active_task"));+ assert_eq!(sanitized["authorization"], json!("[REDACTED]"));+ assert_eq!(+ sanitized["nested"]["authorizationBoundary"],+ json!("[REDACTED]")+ );+ }+#[test]fn outbox_reserves_capacity_for_p0_and_bounds_frames() {let mut bounds_config = config(PathBuf::from("unused"));diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rsindex 860362da03..5c037db769 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs@@ -11,17 +11,69 @@ use std::os::unix::fs::PermissionsExt;use serde::{Deserialize, Serialize};use serde_json::{json, Value};-use crate::codex_provider::{CodexProvider, CodexProviderConfig, CodexProviderEvent};+use crate::codex_provider::{+ CodexProvider, CodexProviderConfig, CodexProviderEvent, RejectedAcceptedTurn,+ MAX_SETTLED_PROVIDER_TURN_IDS,+};use crate::durable::{- create_private_temporary_file, open_private_regular_file, verify_private_directory, Command,- CommandExecution, CommandExecutor, DurableRunnerError, EventPriority, PolledEvent,+ create_private_temporary_file, current_unix_ms, open_private_regular_file, sanitize_value,+ verify_private_directory, Command, CommandExecution, CommandExecutor, DurableRunnerConfig,+ DurableRunnerError, EventPriority, PolledEvent,+};+use crate::provider_bridge::{+ authorized_tool_catalog_digest, semantic_value_digest, AuthorizedToolSet, DurableReplayFilter,+ PendingToolCall, ProviderBridgeError, ProviderToolBridge, ToolResult, MAX_PENDING_CALLS,+ TOOL_SET_SCHEMA,+};+use crate::provider_events::{+ normalize_codex_notification, normalized_codex_terminal_event_type, NormalizedProviderEvent,};-use crate::provider_events::{normalize_codex_notification, NormalizedProviderEvent};const PROVIDER_STATE_SCHEMA: &str = "paperclip.runner.codex-provider-state.v1";const PROVIDER_STATE_FILE: &str = "codex-provider-state.json";-const MAX_PROVIDER_STATE_BYTES: u64 = 2 * 1024 * 1024;+const MAX_PROVIDER_STATE_BYTES: u64 = 16 * 1024 * 1024;const MAX_EVENTS_PER_POLL: usize = 128;+// One accepted semantic call can produce an input and a result event. Normal+// traffic cannot consume the additional capacity required to diagnose a+// receipt-limit stop, settle every retained call, and record the provider plus+// run terminal events.+const MAX_REGULAR_QUEUED_PROVIDER_EVENTS: usize = 2 * MAX_PENDING_CALLS + 3;+const MAX_RECEIPT_LIMIT_TERMINAL_RESERVE: usize = MAX_PENDING_CALLS + 4;+// During a receipt-limit stop, provider polling continues even when older+// events remain unacknowledged so an already-buffered authoritative terminal+// wins over the deadline fallback. Reserve one complete poll of cleanup events+// in addition to the semantic-result and terminal envelopes.+const MAX_TERMINAL_SETTLEMENT_EVENTS: usize = MAX_PENDING_CALLS + MAX_EVENTS_PER_POLL + 4;+const MAX_QUEUED_PROVIDER_EVENTS: usize =+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS + MAX_TERMINAL_SETTLEMENT_EVENTS;+const MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS: u8 = 3;+const RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS: u64 = 2_000;+const RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS: u64 = 30_000;++fn receipt_limit_deadline_after(timeout_ms: u64) -> Result<u64, DurableRunnerError> {+ current_unix_ms()?.checked_add(timeout_ms).ok_or_else(|| {+ DurableRunnerError::invalid("Codex receipt-limit interruption deadline overflowed")+ })+}++#[derive(Clone, Debug)]+struct ProviderEventIdentity {+ run_id: String,+ normalized_session_id: String,+ turn_id: String,+ item_id: String,+}++impl ProviderEventIdentity {+ fn from_config(config: &DurableRunnerConfig) -> Self {+ Self {+ run_id: config.run_id.clone(),+ normalized_session_id: config.normalized_session_id.clone(),+ turn_id: config.turn_id.clone(),+ item_id: config.item_id.clone(),+ }+ }+}#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]#[serde(rename_all = "camelCase")]@@ -70,6 +122,106 @@ fn completion_contract(Ok(Some(binding))}+fn authorized_tool_set(payload: &Value) -> Result<AuthorizedToolSet, DurableRunnerError> {+ if let Some(value) = payload.get("authorizedTools") {+ return serde_json::from_value(value.clone()).map_err(|error| {+ DurableRunnerError::invalid(format!("run.prepare authorizedTools is invalid: {error}"))+ });+ }+ let operations = Vec::new();+ let catalog_digest = authorized_tool_catalog_digest(&operations).map_err(|error| {+ DurableRunnerError::invalid(format!("empty authorized tool set is invalid: {error}"))+ })?;+ Ok(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest,+ operations,+ })+}++fn semantic_correlation(identity: &ProviderEventIdentity) -> Value {+ json!({+ "runId": identity.run_id,+ "normalizedSessionId": identity.normalized_session_id,+ "turnId": identity.turn_id,+ "itemId": identity.item_id,+ })+}++fn semantic_input_event(+ identity: &ProviderEventIdentity,+ call: &PendingToolCall,+) -> NormalizedProviderEvent {+ let safe_input = sanitize_value(&call.input);+ NormalizedProviderEvent {+ event_type: "semantic_tool.input".to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "semantic_tool": {+ "schema": "paperclip.prp.semantic_tool.v1",+ "schemaVersion": 1,+ "phase": "input",+ "operationId": call.operation_id,+ "callId": call.call_id,+ "correlation": semantic_correlation(identity),+ "idempotencyKey": Value::Null,+ "content": {+ "digest": semantic_value_digest(&safe_input),+ "redactionDisposition": "digest_only",+ "references": [],+ },+ "input": safe_input,+ },+ }),+ }+}++fn semantic_result_event(+ identity: &ProviderEventIdentity,+ result: &ToolResult,+) -> NormalizedProviderEvent {+ let safe_result = sanitize_value(&result.result);+ let envelope = safe_result+ .get("resultReceipt")+ .filter(|receipt| {+ receipt.get("schema").and_then(Value::as_str)+ == Some("paperclip.prp.semantic_tool.v1")+ && receipt.get("phase").and_then(Value::as_str) == Some("result")+ && receipt.get("operationId").and_then(Value::as_str)+ == Some(result.operation_id.as_str())+ && receipt.get("callId").and_then(Value::as_str) == Some(result.call_id.as_str())+ && receipt.get("correlation") == Some(&semantic_correlation(identity))+ })+ .cloned()+ .unwrap_or_else(|| {+ json!({+ "schema": "paperclip.prp.semantic_tool.v1",+ "schemaVersion": 1,+ "phase": "result",+ "operationId": result.operation_id,+ "callId": result.call_id,+ "correlation": semantic_correlation(identity),+ "idempotencyKey": Value::Null,+ "content": {+ "digest": semantic_value_digest(&safe_result),+ "redactionDisposition": "digest_only",+ "references": [],+ },+ "outcome": if result.is_error { "failed" } else { "succeeded" },+ "code": if result.is_error { "semantic_tool_failed" } else { "semantic_tool_succeeded" },+ "retryable": false,+ "authorizationBoundary": "active_task",+ "operationReceiptId": format!("operation_{}", result.call_id),+ })+ });+ NormalizedProviderEvent {+ event_type: "semantic_tool.result".to_owned(),+ priority: EventPriority::P0,+ payload: json!({"semantic_tool": envelope}),+ }+}+fn terminal_events(state: &CodexProviderState, event_type: &str) -> Vec<NormalizedProviderEvent> {let Some(contract) = state.completion_contract.as_ref() else {return Vec::new();@@ -158,6 +310,8 @@ struct CodexProviderState {#[serde(default)]completion_contract: Option<CompletionContractBinding>,#[serde(default)]+ tool_bridge: ProviderToolBridge,+ #[serde(default)]thread_id: Option<String>,#[serde(default)]provider_session_id: Option<String>,@@ -173,17 +327,69 @@ struct CodexProviderState {completed_turn_process_generation: Option<u64>,#[serde(default)]completed_provider_turn_id: Option<String>,+ // Unlike the live provider process, the durable run survives restarts.+ // Terminal identities stay exact for one process-generation epoch. At the+ // bound, an idle process is reaped before this epoch is rotated.+ #[serde(default)]+ settled_provider_turn_ids: std::collections::BTreeSet<String>,+ #[serde(default)]+ settled_provider_turn_filter: DurableReplayFilter,+ #[serde(default)]+ receipt_limit_diagnostic_emitted: bool,+ #[serde(default)]+ receipt_limit_interrupt_pending: bool,+ #[serde(default)]+ receipt_limit_interrupt_accepted: bool,+ #[serde(default)]+ receipt_limit_interrupt_attempts: u8,+ #[serde(default)]+ receipt_limit_interrupt_deadline_unix_ms: Option<u64>,last_agent_message: Option<String>,#[serde(default)]pending_events: VecDeque<PolledEvent>,+ #[serde(default)]+ queued_events: VecDeque<PolledEvent>,#[serde(default = "initial_provider_event_seq")]next_provider_event_seq: u64,}+#[derive(Debug, PartialEq)]+enum ToolCallAdmission {+ CompletedReplay(ToolResult),+ PendingReplay,+ Pending(PendingToolCall),+}++fn settled_provider_turn_contains(+ identities: &std::collections::BTreeSet<String>,+ _filter: &DurableReplayFilter,+ provider_turn_id: &str,+) -> bool {+ identities.contains(provider_turn_id)+}++fn remember_settled_provider_turn(+ identities: &mut std::collections::BTreeSet<String>,+ _filter: &mut DurableReplayFilter,+ provider_turn_id: String,+) -> Result<(), DurableRunnerError> {+ if identities.contains(&provider_turn_id) {+ return Ok(());+ }+ if identities.len() >= MAX_SETTLED_PROVIDER_TURN_IDS {+ return Err(DurableRunnerError::invalid(+ "Codex provider turn identity epoch reached its exact capacity",+ ));+ }+ identities.insert(provider_turn_id);+ Ok(())+}+impl CodexProviderState {fn new(config: CodexProviderConfig,completion_contract: Option<CompletionContractBinding>,+ tool_bridge: ProviderToolBridge,) -> Self {let thread_id = config.provider_session_id.clone();Self {@@ -191,6 +397,7 @@ impl CodexProviderState {lifecycle: "prepared".to_owned(),config,completion_contract,+ tool_bridge,thread_id,provider_session_id: None,active_provider_turn_id: None,@@ -199,8 +406,16 @@ impl CodexProviderState {provider_process_generation: 0,completed_turn_process_generation: None,completed_provider_turn_id: None,+ settled_provider_turn_ids: std::collections::BTreeSet::new(),+ settled_provider_turn_filter: DurableReplayFilter::default(),+ receipt_limit_diagnostic_emitted: false,+ receipt_limit_interrupt_pending: false,+ receipt_limit_interrupt_accepted: false,+ receipt_limit_interrupt_attempts: 0,+ receipt_limit_interrupt_deadline_unix_ms: None,last_agent_message: None,pending_events: VecDeque::new(),+ queued_events: VecDeque::new(),next_provider_event_seq: initial_provider_event_seq(),}}@@ -209,6 +424,9 @@ impl CodexProviderState {self.config.validate().map_err(|error| DurableRunnerError::invalid(error.to_string()))?;+ self.tool_bridge.validate_recovered().map_err(|error| {+ DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}"))+ })?;let mut pending_event_ids = HashSet::new();if self.schema != PROVIDER_STATE_SCHEMA|| !matches!(@@ -262,24 +480,59 @@ impl CodexProviderState {|| (!self.completed_turn_authoritative&& (self.completed_turn_process_generation.is_some()|| self.completed_provider_turn_id.is_some()))+ || self.settled_provider_turn_ids.len() > MAX_SETTLED_PROVIDER_TURN_IDS+ || self.settled_provider_turn_filter.validate().is_err()+ || self+ .settled_provider_turn_ids+ .iter()+ .any(|provider_turn_id| {+ provider_turn_id.is_empty()+ || provider_turn_id.len() > 240+ || provider_turn_id.chars().any(char::is_control)+ })+ || self+ .active_provider_turn_id+ .as_ref()+ .is_some_and(|provider_turn_id| {+ settled_provider_turn_contains(+ &self.settled_provider_turn_ids,+ &self.settled_provider_turn_filter,+ provider_turn_id,+ )+ })|| self.completed_turn_process_generation.is_some_and(|generation| generation > self.provider_process_generation)+ || (self.receipt_limit_diagnostic_emitted && self.active_provider_turn_id.is_none())+ || (self.receipt_limit_interrupt_pending+ && (!self.receipt_limit_diagnostic_emitted+ || self.active_provider_turn_id.is_none()))+ || (self.receipt_limit_interrupt_accepted && !self.receipt_limit_interrupt_pending)+ || self.receipt_limit_interrupt_attempts > MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS+ || (!self.receipt_limit_interrupt_pending && self.receipt_limit_interrupt_attempts != 0)+ || self+ .receipt_limit_interrupt_deadline_unix_ms+ .is_some_and(|deadline| deadline == 0 || !self.receipt_limit_interrupt_pending)|| (matches!(self.lifecycle.as_str(),"prepared" | "session_open" | "closed") && self.active_provider_turn_id.is_some())|| self.next_provider_event_seq == 0|| self.pending_events.len() > MAX_EVENTS_PER_POLL + 3- || self.pending_events.iter().any(|event| {- provider_event_sequence(&event.executor_event_id)- .is_none_or(|sequence| sequence >= self.next_provider_event_seq)- || !pending_event_ids.insert(event.executor_event_id.as_str())- || event.event_type.is_empty()- || event.event_type.len() > 160- || event.event_type.chars().any(char::is_control)- || !event.payload.is_object()- })+ || self.queued_events.len() > MAX_QUEUED_PROVIDER_EVENTS+ || self+ .pending_events+ .iter()+ .chain(self.queued_events.iter())+ .any(|event| {+ provider_event_sequence(&event.executor_event_id)+ .is_none_or(|sequence| sequence >= self.next_provider_event_seq)+ || !pending_event_ids.insert(event.executor_event_id.as_str())+ || event.event_type.is_empty()+ || event.event_type.len() > 160+ || event.event_type.chars().any(char::is_control)+ || !event.payload.is_object()+ }){return Err(DurableRunnerError::invalid("Codex provider state is malformed or inconsistent",@@ -288,20 +541,131 @@ impl CodexProviderState {Ok(())}- fn push_event(&mut self, event: NormalizedProviderEvent) -> Result<(), DurableRunnerError> {+ fn push_event_with_limit(+ &mut self,+ event: NormalizedProviderEvent,+ max_queued_events: usize,+ ) -> Result<(), DurableRunnerError> {+ let queue_event =+ !self.queued_events.is_empty() || self.pending_events.len() >= MAX_EVENTS_PER_POLL;+ if queue_event && self.queued_events.len() >= max_queued_events {+ return Err(DurableRunnerError::invalid(+ "Codex provider event backlog exceeds its durable limit",+ ));+ }let sequence = self.next_provider_event_seq;self.next_provider_event_seq = sequence.checked_add(1).ok_or_else(|| DurableRunnerError::invalid("provider event sequence exhausted"))?;- self.pending_events.push_back(PolledEvent {+ let event = PolledEvent {executor_event_id: provider_event_id(sequence),event_type: event.event_type,priority: event.priority,payload: event.payload,- });+ };+ if queue_event {+ self.queued_events.push_back(event);+ } else {+ self.pending_events.push_back(event);+ }+ Ok(())+ }++ fn begin_receipt_limit_stop(+ &mut self,+ call_id: String,+ operation_id: String,+ deadline_unix_ms: u64,+ ) -> Result<bool, DurableRunnerError> {+ if self.receipt_limit_diagnostic_emitted {+ return Ok(self.receipt_limit_interrupt_pending);+ }+ self.push_terminal_event(NormalizedProviderEvent {+ event_type: "harness.diagnostic".to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "provider": "codex",+ "code": "semantic_tool_turn_receipt_limit",+ "operationId": operation_id,+ "callId": call_id,+ "message": "The active provider turn reached its durable semantic-tool receipt limit and was interrupted",+ "paperclipExecuted": false,+ }),+ })?;+ self.receipt_limit_diagnostic_emitted = true;+ self.receipt_limit_interrupt_pending = true;+ self.receipt_limit_interrupt_attempts = 0;+ self.receipt_limit_interrupt_deadline_unix_ms = Some(deadline_unix_ms);+ Ok(true)+ }++ fn record_receipt_limit_interrupt_attempt(&mut self) -> Result<(), DurableRunnerError> {+ if self.receipt_limit_interrupt_attempts >= MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS {+ return Err(DurableRunnerError::invalid(+ "Codex receipt-limit interruption exceeded its durable retry bound",+ ));+ }+ self.receipt_limit_interrupt_attempts += 1;Ok(())}+ fn mark_receipt_limit_interrupt_accepted(&mut self, accepted_deadline_unix_ms: u64) {+ // Provider restoration can reconcile the active turn as already+ // settled while an interruption command is in flight. In that case+ // `interrupt_turn` returns `already_settled` and recovery has already+ // cleared the durable retry marker. Do not recreate an accepted state+ // without a pending interruption or active turn.+ if !self.receipt_limit_interrupt_pending || self.active_provider_turn_id.is_none() {+ return;+ }+ if !self.receipt_limit_interrupt_accepted {+ self.receipt_limit_interrupt_deadline_unix_ms = Some(+ self.receipt_limit_interrupt_deadline_unix_ms+ .unwrap_or_default()+ .max(accepted_deadline_unix_ms),+ );+ }+ self.receipt_limit_interrupt_accepted = true;+ }++ fn push_event(&mut self, event: NormalizedProviderEvent) -> Result<(), DurableRunnerError> {+ self.push_event_with_limit(event, MAX_REGULAR_QUEUED_PROVIDER_EVENTS)+ }++ fn push_terminal_event(+ &mut self,+ event: NormalizedProviderEvent,+ ) -> Result<(), DurableRunnerError> {+ self.push_event_with_limit(event, MAX_QUEUED_PROVIDER_EVENTS)+ }++ fn push_receipt_limit_cleanup_event(+ &mut self,+ event: NormalizedProviderEvent,+ ) -> Result<(), DurableRunnerError> {+ let queue_event =+ !self.queued_events.is_empty() || self.pending_events.len() >= MAX_EVENTS_PER_POLL;+ if queue_event+ && self.queued_events.len()+ >= MAX_QUEUED_PROVIDER_EVENTS - MAX_RECEIPT_LIMIT_TERMINAL_RESERVE+ {+ // Continue draining the provider so an authoritative terminal can+ // still be observed, but never let cleanup chatter consume the+ // semantic-result and terminal-event reserve.+ return Ok(());+ }+ self.push_terminal_event(event)+ }++ fn refill_pending_events(&mut self) {+ while self.pending_events.len() < MAX_EVENTS_PER_POLL {+ let Some(event) = self.queued_events.pop_front() else {+ break;+ };+ self.pending_events.push_back(event);+ }+ }+fn extend_events(&mut self,events: impl IntoIterator<Item = NormalizedProviderEvent>,@@ -312,6 +676,43 @@ impl CodexProviderState {Ok(())}+ fn admit_tool_call(+ &mut self,+ call_id: &str,+ operation_id: &str,+ input: &Value,+ ) -> Result<ToolCallAdmission, ProviderBridgeError> {+ if let Some(result) = self+ .tool_bridge+ .replay_result(call_id, operation_id, input)?+ {+ // An exact completed replay is a transport retry. Its input and+ // result receipts are already durable, so recording another event+ // would make an otherwise idempotent replay consume bounded event+ // capacity and could prevent returning the stored result.+ return Ok(ToolCallAdmission::CompletedReplay(result));+ }++ let pending_replay = self+ .tool_bridge+ .pending_calls()+ .any(|pending| pending.call_id == call_id);+ let call = self.tool_bridge.begin_call(+ call_id.to_owned(),+ operation_id.to_owned(),+ input.clone(),+ )?;+ if pending_replay {+ // The first input receipt is already durable. An exact pending+ // replay is only the provider re-establishing its request after a+ // process restart; appending another event would make the retry+ // consume bounded backlog capacity without adding information.+ Ok(ToolCallAdmission::PendingReplay)+ } else {+ Ok(ToolCallAdmission::Pending(call))+ }+ }+fn reconcile_active_provider_turn(&mut self, active_provider_turn_id: Option<String>) {self.active_provider_turn_id = active_provider_turn_id;if self.active_provider_turn_id.is_some() {@@ -330,12 +731,53 @@ impl CodexProviderState {"session_open".to_owned()};}++ fn settle_active_provider_turn_identity(&mut self) -> Result<(), DurableRunnerError> {+ let provider_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| {+ DurableRunnerError::invalid("Codex terminal omitted its active provider turn identity")+ })?;+ remember_settled_provider_turn(+ &mut self.settled_provider_turn_ids,+ &mut self.settled_provider_turn_filter,+ provider_turn_id,+ )?;+ Ok(())+ }++ fn recovered_settled_provider_turn_ids(+ &self,+ ) -> Result<(std::collections::BTreeSet<String>, DurableReplayFilter), DurableRunnerError> {+ let mut settled_provider_turn_ids = self.settled_provider_turn_ids.clone();+ let mut settled_provider_turn_filter = self.settled_provider_turn_filter.clone();+ // State written before the durable set was introduced retained only+ // the latest completed identity. Fold that legacy authority into the+ // new ledger before the provider is allowed to accept replacement work.+ if let Some(provider_turn_id) = self.completed_provider_turn_id.clone() {+ remember_settled_provider_turn(+ &mut settled_provider_turn_ids,+ &mut settled_provider_turn_filter,+ provider_turn_id,+ )?;+ }+ Ok((settled_provider_turn_ids, settled_provider_turn_filter))+ }++ fn extend_terminal_events(+ &mut self,+ events: impl IntoIterator<Item = NormalizedProviderEvent>,+ ) -> Result<(), DurableRunnerError> {+ for event in events {+ self.push_terminal_event(event)?;+ }+ Ok(())+ }}pub struct CodexCommandExecutor {state_dir: PathBuf,state: Option<CodexProviderState>,provider: Option<CodexProvider>,+ event_identity: Option<ProviderEventIdentity>,restore_checked: bool,}@@ -345,10 +787,17 @@ impl CodexCommandExecutor {state_dir: state_dir.into(),state: None,provider: None,+ event_identity: None,restore_checked: false,}}+ pub fn with_runner_config(state_dir: impl Into<PathBuf>, config: &DurableRunnerConfig) -> Self {+ let mut executor = Self::new(state_dir);+ executor.event_identity = Some(ProviderEventIdentity::from_config(config));+ executor+ }+fn state_path(&self) -> PathBuf {self.state_dir.join(PROVIDER_STATE_FILE)}@@ -380,9 +829,14 @@ impl CodexCommandExecutor {file.read_to_end(&mut input).map_err(|error| {DurableRunnerError::invalid(format!("failed to read Codex provider state: {error}"))})?;- let state: CodexProviderState = serde_json::from_slice(&input).map_err(|error| {+ let mut state: CodexProviderState = serde_json::from_slice(&input).map_err(|error| {DurableRunnerError::invalid(format!("Codex provider state is malformed: {error}"))})?;+ state.tool_bridge.attach_existing_run().map_err(|error| {+ DurableRunnerError::invalid(format!(+ "Codex semantic tool state could not be reattached: {error}"+ ))+ })?;state.validate()?;self.state = Some(state);self.restore_provider_if_needed()@@ -413,15 +867,139 @@ impl CodexCommandExecutor {let completed_turn_process_generation = state.completed_turn_process_generation;let completed_provider_turn_id = state.completed_provider_turn_id.clone();let ambiguous_turn_start_pending = state.ambiguous_turn_start_pending;- let mut provider = CodexProvider::start_for_generation(+ let tool_replay_history_blocks_admission =+ state.tool_bridge.replay_history_blocks_admission();+ let tool_receipt_epoch_has_active_receipts = state.tool_bridge.has_active_receipts();+ let (settled_provider_turn_ids, settled_provider_turn_filter) =+ state.recovered_settled_provider_turn_ids()?;+ let provider_epoch_requires_rollover = settled_provider_turn_ids.len()+ >= MAX_SETTLED_PROVIDER_TURN_IDS+ || !settled_provider_turn_filter.is_empty();+ let mut provider = CodexProvider::start_with_tools_for_generation(&state.config,+ state.tool_bridge.authorized_tools().cloned(),Some(&thread_id),process_generation,).map_err(|error| {DurableRunnerError::invalid(format!("failed to resume Codex provider: {error}"))})?;+ provider.enable_durable_tool_call_replays();+ provider+ .restore_settled_turn_identities(+ settled_provider_turn_ids.iter().cloned(),+ settled_provider_turn_filter.clone(),+ )+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to restore Codex provider turn identities: {error}"+ ))+ })?;let recovered_active_turn_id = provider.active_provider_turn_id().map(str::to_owned);+ let legacy_epoch_is_ambiguous = (provider_epoch_requires_rollover+ && (ambiguous_turn_start_pending || recovered_active_turn_id.is_some()))+ || (tool_replay_history_blocks_admission+ && (ambiguous_turn_start_pending+ || recovered_active_turn_id.is_some()+ || (previous_active_turn_id.is_some()+ && tool_receipt_epoch_has_active_receipts)));+ if legacy_epoch_is_ambiguous {+ // A saturated legacy epoch cannot prove that recovered work can+ // be identified and settled exactly. Reap the resumed process+ // generation and close the run instead of risking duplicate work.+ let provider_reported_active = recovered_active_turn_id.is_some();+ let provider_shutdown_failed = provider.shutdown().is_err();+ drop(provider);+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available during legacy recovery");+ state.provider_process_generation = process_generation;+ state.settled_provider_turn_ids = settled_provider_turn_ids;+ state.settled_provider_turn_filter = settled_provider_turn_filter;+ state.active_provider_turn_id = None;+ state.ambiguous_turn_start_pending = false;+ state.completed_turn_authoritative = false;+ state.completed_turn_process_generation = None;+ state.completed_provider_turn_id = None;+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;+ state.last_agent_message = None;+ state.lifecycle = "closed".to_owned();+ let _ = state.push_terminal_event(NormalizedProviderEvent {+ event_type: "harness.diagnostic".to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "provider": "codex",+ "code": "legacy_provider_turn_epoch_ambiguous",+ "message": "Codex recovery could not safely identify and settle active work from a saturated legacy replay epoch; Paperclip terminated the provider and closed the durable run",+ "paperclipAccepted": false,+ "providerReportedActive": provider_reported_active,+ "ambiguousStartPending": ambiguous_turn_start_pending,+ "providerShutdownFailed": provider_shutdown_failed,+ }),+ });+ self.save_state()?;+ return Ok(());+ }+ if let Some(reused_provider_turn_id) = recovered_active_turn_id+ .as_ref()+ .filter(|provider_turn_id| {+ settled_provider_turn_contains(+ &settled_provider_turn_ids,+ &settled_provider_turn_filter,+ provider_turn_id,+ )+ })+ .cloned()+ {+ // The durable terminal ledger is authoritative. A resumed provider+ // that reports one of those identities as active is contradictory+ // and may still be mutating the workspace. Terminate that process+ // generation and persist the run closed before exposing recovery+ // to the controller; otherwise this path would reopen settled work.+ let provider_shutdown_failed = provider.shutdown().is_err();+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available during recovery");+ state.provider_process_generation = process_generation;+ state.settled_provider_turn_ids = settled_provider_turn_ids;+ state.settled_provider_turn_filter = settled_provider_turn_filter;+ state.active_provider_turn_id = None;+ state.ambiguous_turn_start_pending = false;+ state.completed_turn_authoritative = false;+ state.completed_turn_process_generation = None;+ state.completed_provider_turn_id = None;+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;+ state.last_agent_message = None;+ state.lifecycle = "closed".to_owned();+ // Closing the provider is the safety boundary. Preserve that+ // durable transition even when an already-full event backlog has+ // no room for an additional diagnostic.+ let _ = state.push_terminal_event(NormalizedProviderEvent {+ event_type: "harness.diagnostic".to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "provider": "codex",+ "code": "provider_turn_identity_reused",+ "providerTurnId": reused_provider_turn_id,+ "message": "Codex recovery reported a previously settled turn identity as active; Paperclip terminated the provider and closed the durable run",+ "paperclipAccepted": false,+ "providerReportedActive": true,+ "providerShutdownFailed": provider_shutdown_failed,+ }),+ });+ self.save_state()?;+ return Ok(());+ }if ambiguous_turn_start_pending {let recovered_turn_id = recovered_active_turn_id.as_deref().ok_or_else(|| {DurableRunnerError::invalid(@@ -434,37 +1012,99 @@ impl CodexCommandExecutor {));}}- provider.restore_completed_turn_authority(- completed_turn_authoritative- && recovered_active_turn_id.is_none()- && !ambiguous_turn_start_pending,- completed_turn_process_generation,- completed_provider_turn_id.as_deref(),- );+ provider+ .restore_completed_turn_authority(+ completed_turn_authoritative+ && recovered_active_turn_id.is_none()+ && !ambiguous_turn_start_pending,+ completed_turn_process_generation,+ completed_provider_turn_id.as_deref(),+ )+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to restore Codex completion authority: {error}"+ ))+ })?;self.provider = Some(provider);- self.state- .as_mut()- .expect("Codex state remains available during recovery")- .provider_process_generation = process_generation;+ {+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available during recovery");+ state.provider_process_generation = process_generation;+ state.settled_provider_turn_ids = settled_provider_turn_ids;+ state.settled_provider_turn_filter = settled_provider_turn_filter;+ }if provider_had_exited|| ambiguous_turn_start_pending|| recovered_active_turn_id != previous_active_turn_id{+ let recovered_turn_ended =+ previous_active_turn_id.is_some() && recovered_active_turn_id.is_none();+ let identity = self.event_identity.clone();let state = self.state.as_mut().expect("Codex state remains available during recovery");+ if recovered_turn_ended {+ if !provider_epoch_requires_rollover {+ state.settle_active_provider_turn_identity()?;+ }+ let settled = state+ .tool_bridge+ .settle_turn("provider_turn_terminated")+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to settle semantic tools during recovery: {error}"+ ))+ })?;+ if !settled.is_empty() {+ let identity = identity.as_ref().ok_or_else(|| {+ DurableRunnerError::invalid(+ "Codex semantic tool events require the durable runner identity",+ )+ })?;+ for result in settled {+ state.push_terminal_event(semantic_result_event(identity, &result))?;+ }+ }+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;+ }state.reconcile_active_provider_turn(recovered_active_turn_id.clone());- state.push_event(NormalizedProviderEvent {+ let reconciled = NormalizedProviderEvent {event_type: "session.reconciled".to_owned(),priority: EventPriority::P0,payload: json!({"provider": "codex","providerSessionId": thread_id,- "previousProviderTurnId": previous_active_turn_id,- "activeProviderTurnId": recovered_active_turn_id,+ "previousProviderTurnId": previous_active_turn_id.clone(),+ "activeProviderTurnId": recovered_active_turn_id.clone(),}),- })?;+ };+ if recovered_turn_ended {+ state.push_terminal_event(reconciled)?;+ // A turn that disappeared while runnerd was offline has no+ // trustworthy success notification to replay. Terminate it+ // conservatively so the controller cannot wait forever or+ // mistake an unknown outcome for success.+ state.push_terminal_event(NormalizedProviderEvent {+ event_type: "turn.failed".to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "provider": "codex",+ "providerTurnId": previous_active_turn_id,+ "status": "failed",+ "providerTerminalObserved": false,+ }),+ })?;+ state.extend_terminal_events(terminal_events(state, "turn.failed"))?;+ } else {+ state.push_event(reconciled)?;+ }}self.save_state()?;Ok(())@@ -502,7 +1142,7 @@ impl CodexCommandExecutor {})?;if bytes.len() as u64 > MAX_PROVIDER_STATE_BYTES {return Err(DurableRunnerError::invalid(- "Codex provider state exceeds the 2 MiB limit",+ "Codex provider state exceeds the 16 MiB limit",));}let (temporary, mut file) = create_private_temporary_file(&path)?;@@ -542,7 +1182,8 @@ impl CodexCommandExecutor {.validate().map_err(|error| DurableRunnerError::invalid(error.to_string()))?;let completion_contract = completion_contract(payload)?;- if let Some(state) = &self.state {+ let tool_set = authorized_tool_set(payload)?;+ if let Some(state) = self.state.as_mut() {if state.config != config || state.completion_contract != completion_contract {return Err(DurableRunnerError::invalid("Codex provider or completion contract changed across the durable run",@@ -553,8 +1194,33 @@ impl CodexCommandExecutor {"Codex provider session is already closed",));}+ if state.tool_bridge.has_catalog() {+ state+ .tool_bridge+ .verify_tool_set(&tool_set)+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "run.prepare tool contract changed: {error}"+ ))+ })?;+ } else {+ state.tool_bridge.prepare(tool_set).map_err(|error| {+ DurableRunnerError::invalid(format!(+ "run.prepare tool contract rejected: {error}"+ ))+ })?;+ self.save_state()?;+ }} else {- self.state = Some(CodexProviderState::new(config, completion_contract));+ let mut tool_bridge = ProviderToolBridge::default();+ tool_bridge.prepare(tool_set).map_err(|error| {+ DurableRunnerError::invalid(format!("run.prepare tool contract rejected: {error}"))+ })?;+ self.state = Some(CodexProviderState::new(+ config,+ completion_contract,+ tool_bridge,+ ));self.save_state()?;}Ok(CommandExecution::result(json!({@@ -579,24 +1245,50 @@ impl CodexCommandExecutor {.provider_process_generation.checked_add(1).ok_or_else(|| DurableRunnerError::invalid("Codex process generation exhausted"))?;- let mut provider = CodexProvider::start_for_generation(+ let (settled_provider_turn_ids, settled_provider_turn_filter) =+ state.recovered_settled_provider_turn_ids()?;+ let mut provider = CodexProvider::start_with_tools_for_generation(&state.config,+ state.tool_bridge.authorized_tools().cloned(),state.thread_id.as_deref(),process_generation,).map_err(|error| {DurableRunnerError::invalid(format!("failed to start Codex provider: {error}"))})?;- provider.restore_completed_turn_authority(- state.completed_turn_authoritative && provider.active_provider_turn_id().is_none(),- state.completed_turn_process_generation,- state.completed_provider_turn_id.as_deref(),- );+ provider.enable_durable_tool_call_replays();+ provider+ .restore_settled_turn_identities(+ settled_provider_turn_ids.iter().cloned(),+ settled_provider_turn_filter.clone(),+ )+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to restore Codex provider turn identities: {error}"+ ))+ })?;+ provider+ .restore_completed_turn_authority(+ state.completed_turn_authoritative+ && provider.active_provider_turn_id().is_none(),+ state.completed_turn_process_generation,+ state.completed_provider_turn_id.as_deref(),+ )+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to restore Codex completion authority: {error}"+ ))+ })?;self.provider = Some(provider);- self.state- .as_mut()- .expect("Codex state remains available after provider start")- .provider_process_generation = process_generation;+ {+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available after provider start");+ state.provider_process_generation = process_generation;+ state.settled_provider_turn_ids = settled_provider_turn_ids;+ state.settled_provider_turn_filter = settled_provider_turn_filter;+ }self.save_state()?;}self.provider@@ -604,6 +1296,21 @@ impl CodexCommandExecutor {.ok_or_else(|| DurableRunnerError::invalid("Codex provider is unavailable"))}+ fn verify_attached_tools(&self, payload: &Value) -> Result<(), DurableRunnerError> {+ if payload.get("authorizedTools").is_none() {+ return Ok(());+ }+ let tool_set = authorized_tool_set(payload)?;+ self.state+ .as_ref()+ .ok_or_else(|| DurableRunnerError::invalid("Codex provider has not been prepared"))?+ .tool_bridge+ .verify_tool_set(&tool_set)+ .map_err(|error| {+ DurableRunnerError::invalid(format!("run.attach tool contract changed: {error}"))+ })+ }+fn open_session(&mut self) -> Result<CommandExecution, DurableRunnerError> {self.restore_provider_if_needed()?;if self@@ -636,6 +1343,11 @@ impl CodexCommandExecutor {state.thread_id = Some(thread_id.clone());state.provider_session_id = provider_session_id.clone();state.active_provider_turn_id = None;+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;state.lifecycle = "session_open".to_owned();state.config.provider_version.clone()};@@ -667,47 +1379,244 @@ impl CodexCommandExecutor {})}- fn start_turn(&mut self, payload: &Value) -> Result<CommandExecution, DurableRunnerError> {- self.restore_provider_if_needed()?;- if self- .state+ fn close_after_rejected_provider_acceptance(+ &mut self,+ rejected_accepted_turn: &RejectedAcceptedTurn,+ ) -> Result<(), DurableRunnerError> {+ let provider_process_generation = self+ .provider.as_ref()- .is_some_and(|state| state.active_provider_turn_id.is_some())- {- return Err(DurableRunnerError::invalid(- "Codex already has an active provider turn",- ));+ .map(CodexProvider::process_generation);+ // The provider has already been terminated. Drop its quarantined+ // handle before persisting the closure so recovery can never resume+ // work that Codex accepted without Paperclip accepting its identity.+ self.provider = None;+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available after rejected provider acceptance");+ if let Some(provider_process_generation) = provider_process_generation {+ state.provider_process_generation = provider_process_generation;}- let text = payload- .get("text")- .and_then(Value::as_str)- .ok_or_else(|| DurableRunnerError::invalid("turn.start payload.text is required"))?;- let cwd = self+ state.active_provider_turn_id = None;+ state.ambiguous_turn_start_pending = false;+ state.completed_turn_authoritative = false;+ state.completed_turn_process_generation = None;+ state.completed_provider_turn_id = None;+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;+ state.last_agent_message = None;+ state.lifecycle = "closed".to_owned();+ // Closure is the safety boundary. Preserve it even if a saturated+ // event queue cannot retain this additional diagnostic.+ let _ = state.push_terminal_event(NormalizedProviderEvent {+ event_type: "harness.diagnostic".to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "provider": "codex",+ "code": match rejected_accepted_turn {+ RejectedAcceptedTurn::ReusedIdentity(_) => "provider_turn_identity_reused",+ RejectedAcceptedTurn::InvalidIdentity => "provider_turn_identity_invalid",+ },+ "providerTurnId": match rejected_accepted_turn {+ RejectedAcceptedTurn::ReusedIdentity(provider_turn_id) => json!(provider_turn_id),+ RejectedAcceptedTurn::InvalidIdentity => Value::Null,+ },+ "message": match rejected_accepted_turn {+ RejectedAcceptedTurn::ReusedIdentity(_) => "Codex accepted work with a previously settled turn identity; Paperclip terminated the provider and closed the durable run",+ RejectedAcceptedTurn::InvalidIdentity => "Codex accepted work without a valid bounded turn identity; Paperclip terminated the provider and closed the durable run",+ },+ "paperclipAccepted": false,+ "providerAccepted": true,+ }),+ });+ self.save_state()+ }++ fn rollover_provider_identity_epochs_if_needed(&mut self) -> Result<(), DurableRunnerError> {+ let tool_rollover_required = self.state.as_ref()- .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?- .config- .cwd- .clone();- self.ensure_provider()?;- {- let state = self- .state- .as_mut()- .expect("Codex state remains available before turn/start dispatch");- state.ambiguous_turn_start_pending = true;+ .is_some_and(|state| state.tool_bridge.replay_history_blocks_admission());+ let rollover_required = self.state.as_ref().is_some_and(|state| {+ state.settled_provider_turn_ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS+ || !state.settled_provider_turn_filter.is_empty()+ || tool_rollover_required+ });+ if !rollover_required {+ return Ok(());}- self.save_state()?;- let (start_result, completion_authority_retained, ambiguous_turn_start_pending) = {- let provider = self.ensure_provider()?;+ let rollover_is_safe = self.state.as_ref().is_some_and(|state| {+ state.active_provider_turn_id.is_none()+ && !state.ambiguous_turn_start_pending+ && state.lifecycle == "session_open"+ });+ if !rollover_is_safe {+ return Err(DurableRunnerError::invalid(+ "Codex provider identity epoch cannot rotate while work is active",+ ));+ }++ let (restart_result, process_generation, rejected_accepted_turn) = {+ let provider = self.provider.as_mut().ok_or_else(|| {+ DurableRunnerError::invalid(+ "Codex provider identity epoch cannot rotate without an attached process",+ )+ })?;+ let restart_result = provider.restart_idle_identity_epoch();+ (+ restart_result,+ provider.process_generation(),+ provider.take_rejected_accepted_turn(),+ )+ };+ if let Err(error) = restart_result {+ if let Some(rejected_accepted_turn) = rejected_accepted_turn {+ let failure_kind = match &rejected_accepted_turn {+ RejectedAcceptedTurn::ReusedIdentity(_) => "accepted identity reuse",+ RejectedAcceptedTurn::InvalidIdentity => "an invalid accepted identity",+ };+ self.close_after_rejected_provider_acceptance(&rejected_accepted_turn)?;+ return Err(DurableRunnerError::invalid(format!(+ "Codex identity epoch rollover failed closed after {failure_kind}: {error}"+ )));+ }+ return Err(DurableRunnerError::invalid(format!(+ "failed to rotate the completed Codex identity epoch: {error}"+ )));+ };+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available during identity epoch rollover");+ state.provider_process_generation = process_generation;+ state.settled_provider_turn_ids.clear();+ if let Some(completed_provider_turn_id) = state.completed_provider_turn_id.clone() {+ // The replacement process restored this still-authoritative+ // terminal into its fresh epoch. Mirror that one tombstone in the+ // durable ledger until accepting replacement work revokes the+ // completion authority.+ state+ .settled_provider_turn_ids+ .insert(completed_provider_turn_id);+ }+ state.settled_provider_turn_filter = DurableReplayFilter::default();+ if tool_rollover_required {+ state+ .tool_bridge+ .rollover_replay_epoch_after_provider_restart()+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to rotate Codex semantic tool replay authority: {error}"+ ))+ })?;+ }+ self.save_state()?;+ Ok(())+ }++ fn start_turn(&mut self, payload: &Value) -> Result<CommandExecution, DurableRunnerError> {+ self.restore_provider_if_needed()?;+ if self+ .state+ .as_ref()+ .is_some_and(|state| state.lifecycle == "closed")+ {+ return Err(DurableRunnerError::invalid(+ "Codex provider session is closed",+ ));+ }+ if self+ .state+ .as_ref()+ .is_some_and(|state| state.active_provider_turn_id.is_some())+ {+ return Err(DurableRunnerError::invalid(+ "Codex already has an active provider turn",+ ));+ }+ if self+ .state+ .as_ref()+ .is_some_and(|state| state.ambiguous_turn_start_pending)+ {+ return Err(DurableRunnerError::invalid(+ "Codex has an unresolved ambiguous provider turn start",+ ));+ }+ if self+ .state+ .as_ref()+ .is_some_and(|state| state.queued_events.len() >= MAX_REGULAR_QUEUED_PROVIDER_EVENTS)+ {+ return Err(DurableRunnerError::invalid(+ "cannot start a new Codex turn until terminal events are acknowledged",+ ));+ }+ self.state+ .as_mut()+ .expect("Codex state remains available before turn receipt preparation")+ .tool_bridge+ .prepare_turn()+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "Codex semantic tool receipts could not prepare the next turn: {error}"+ ))+ })?;+ self.rollover_provider_identity_epochs_if_needed()?;+ let text = payload+ .get("text")+ .and_then(Value::as_str)+ .ok_or_else(|| DurableRunnerError::invalid("turn.start payload.text is required"))?;+ let cwd = self+ .state+ .as_ref()+ .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?+ .config+ .cwd+ .clone();+ self.ensure_provider()?;+ {+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available before turn/start dispatch");+ state.ambiguous_turn_start_pending = true;+ }+ self.save_state()?;+ let (+ start_result,+ completion_authority_retained,+ ambiguous_turn_start_pending,+ rejected_accepted_turn,+ ) = {+ let provider = self.ensure_provider()?;let result = provider.start_turn(text, &cwd);(result,provider.completed_turn_authority().is_some(),provider.ambiguous_turn_start_pending(),+ provider.take_rejected_accepted_turn(),)};if let Err(error) = start_result {+ if let Some(rejected_accepted_turn) = rejected_accepted_turn {+ // Codex accepted this work before disclosing a usable durable+ // identity. The provider has already been terminated; close+ // this run before returning so recovery cannot resume the+ // untracked turn from the provider's thread snapshot.+ self.close_after_rejected_provider_acceptance(&rejected_accepted_turn)?;+ let failure_kind = match &rejected_accepted_turn {+ RejectedAcceptedTurn::ReusedIdentity(_) => "accepted identity reuse",+ RejectedAcceptedTurn::InvalidIdentity => "an invalid accepted identity",+ };+ return Err(DurableRunnerError::invalid(format!(+ "Codex turn/start failed closed after {failure_kind}: {error}"+ )));+ }let state = self.state.as_mut()@@ -748,6 +1657,11 @@ impl CodexCommandExecutor {state.completed_turn_authoritative = false;state.completed_turn_process_generation = None;state.completed_provider_turn_id = None;+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;state.last_agent_message = None;state.lifecycle = "turn_active".to_owned();self.save_state()?;@@ -773,6 +1687,30 @@ impl CodexCommandExecutor {"reason": reason,})));}+ let has_pending_tools = self+ .state+ .as_ref()+ .is_some_and(|state| state.tool_bridge.pending_calls().next().is_some());+ if has_pending_tools {+ let identity = self.event_identity()?;+ let mut next_state = self+ .state+ .clone()+ .expect("Codex state remains available during interruption");+ let cancelled = next_state+ .tool_bridge+ .cancel_pending_calls("provider_turn_stopped")+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to cancel pending semantic tools: {error}"+ ))+ })?;+ for result in cancelled {+ next_state.push_terminal_event(semantic_result_event(&identity, &result))?;+ }+ self.persist_state(&next_state)?;+ self.state = Some(next_state);+ }self.ensure_provider()?.interrupt_turn().map_err(|error| {DurableRunnerError::invalid(format!("Codex turn interrupt failed: {error}"))})?;@@ -799,6 +1737,15 @@ impl CodexCommandExecutor {.get("requestId").and_then(Value::as_str).ok_or_else(|| DurableRunnerError::invalid("request.resolve requires requestId"))?;+ if self+ .state+ .as_ref()+ .is_none_or(|state| state.active_provider_turn_id.is_none())+ {+ return Err(DurableRunnerError::invalid(+ "cannot resolve a Codex runtime request outside an active turn",+ ));+ }let response = payload.get("response").ok_or_else(|| DurableRunnerError::invalid("request.resolve requires response"))?;@@ -817,6 +1764,377 @@ impl CodexCommandExecutor {})}+ fn event_identity(&self) -> Result<ProviderEventIdentity, DurableRunnerError> {+ self.event_identity.clone().ok_or_else(|| {+ DurableRunnerError::invalid(+ "Codex semantic tool events require the durable runner identity",+ )+ })+ }++ fn reject_tool_call(+ &mut self,+ call_id: String,+ operation_id: String,+ reason: String,+ ) -> Result<(), DurableRunnerError> {+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available for a rejected tool call");+ let event = NormalizedProviderEvent {+ event_type: "harness.diagnostic".to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "provider": "codex",+ "code": "semantic_tool_denied",+ "operationId": operation_id,+ "callId": call_id,+ "message": reason,+ "paperclipExecuted": false,+ }),+ };+ if state.receipt_limit_interrupt_pending {+ state.push_receipt_limit_cleanup_event(event)?;+ } else {+ state.push_event(event)?;+ }+ self.save_state()?;+ let rejection = ToolResult {+ call_id,+ operation_id,+ result: json!({+ "error": {+ "code": "invalid_tool_call",+ "message": "Paperclip rejected this semantic tool call",+ "retryable": false,+ },+ }),+ is_error: true,+ };+ self.provider+ .as_mut()+ .expect("provider remains present while rejecting its tool call")+ .deliver_tool_result(&rejection)+ .map_err(|delivery_error| {+ DurableRunnerError::invalid(format!(+ "failed to return the semantic tool rejection: {delivery_error}"+ ))+ })+ }++ fn stop_turn_at_tool_receipt_limit(+ &mut self,+ call_id: String,+ operation_id: String,+ ) -> Result<(), DurableRunnerError> {+ let deadline_unix_ms =+ receipt_limit_deadline_after(RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS)?;+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available at its tool receipt limit");+ let interrupt_pending = state.begin_receipt_limit_stop(+ call_id.clone(),+ operation_id.clone(),+ deadline_unix_ms,+ )?;+ let first_interrupt_attempt =+ interrupt_pending && state.receipt_limit_interrupt_attempts == 0;+ if interrupt_pending {+ self.save_state()?;+ }+ // The durable diagnostic owns the turn-level failure, while every+ // buffered JSON-RPC call still receives an explicit provider error.+ // Deliver the rejection before requesting interruption: Codex may+ // close the transport as part of the interrupt, and a failed courtesy+ // RPC must never abort runnerd's terminal polling loop.+ let rejection = ToolResult {+ call_id,+ operation_id,+ result: json!({+ "error": {+ "code": "semantic_tool_turn_receipt_limit",+ "message": "Paperclip stopped this turn at its durable semantic-tool receipt limit",+ "retryable": false,+ },+ }),+ is_error: true,+ };+ if let Some(provider) = self.provider.as_mut() {+ let _ = provider.deliver_tool_result(&rejection);+ }+ if first_interrupt_attempt {+ // Keep the durable retry marker until a terminal notification is+ // observed. Provider acceptance acknowledges only this RPC; it+ // does not prove that the turn stopped. Later buffered calls may+ // therefore retry the idempotent interruption instead of leaving+ // a still-active receipt-exhausted turn permanently unstopped.+ self.state+ .as_mut()+ .expect("Codex state remains available before receipt-limit interruption")+ .record_receipt_limit_interrupt_attempt()?;+ self.save_state()?;+ match self.interrupt_turn("semantic_tool_turn_receipt_limit") {+ Ok(_) => {+ let accepted_deadline_unix_ms =+ receipt_limit_deadline_after(RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS)?;+ self.state+ .as_mut()+ .expect("Codex state remains available after receipt-limit interruption")+ .mark_receipt_limit_interrupt_accepted(accepted_deadline_unix_ms);+ self.save_state()?;+ }+ // The first interruption attempt is also best-effort. Its+ // durable retry marker is already saved, and propagating the+ // transport error here would terminate runnerd before it can+ // poll the provider's terminal notification.+ Err(_) => {}+ }+ }+ Ok(())+ }++ fn settle_receipt_limit_interrupt_after_deadline(&mut self) -> Result<(), DurableRunnerError> {+ let interrupt_accepted = self+ .state+ .as_ref()+ .is_some_and(|state| state.receipt_limit_interrupt_accepted);+ let provider_shutdown_failed = self+ .provider+ .as_mut()+ .is_some_and(|provider| provider.shutdown().is_err());+ self.provider = None;+ let identity = self.event_identity()?;+ let state = self+ .state+ .as_mut()+ .expect("Codex state remains available at its receipt-limit retry bound");+ state.settle_active_provider_turn_identity()?;+ let settled = state+ .tool_bridge+ .settle_turn("semantic_tool_turn_receipt_limit")+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to settle semantic tools at the receipt-limit retry bound: {error}"+ ))+ })?;+ for result in settled {+ state.push_terminal_event(semantic_result_event(&identity, &result))?;+ }+ state.active_provider_turn_id = None;+ state.completed_turn_authoritative = false;+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;+ state.lifecycle = "provider_exited".to_owned();+ let terminal_event_type = if interrupt_accepted {+ "turn.interrupted"+ } else {+ "turn.failed"+ };+ state.push_terminal_event(NormalizedProviderEvent {+ event_type: terminal_event_type.to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "provider": "codex",+ "code": if interrupt_accepted {+ "semantic_tool_turn_receipt_limit_interrupt_deadline"+ } else {+ "semantic_tool_turn_receipt_limit_interrupt_unconfirmed"+ },+ "message": if interrupt_accepted {+ "Codex accepted the receipt-limit interruption but did not emit its terminal before the bounded shutdown deadline"+ } else {+ "Codex did not confirm terminal state after the bounded receipt-limit interruption attempts"+ },+ "interruptAccepted": interrupt_accepted,+ "providerTerminalObserved": false,+ "providerShutdownFailed": provider_shutdown_failed,+ }),+ })?;+ let terminal = terminal_events(state, terminal_event_type);+ state.extend_terminal_events(terminal)?;+ self.save_state()+ }++ fn retry_receipt_limit_interrupt(&mut self) -> Result<(), DurableRunnerError> {+ let (should_retry, attempts, persisted_deadline) =+ self.state.as_ref().map_or((false, 0, None), |state| {+ (+ state.receipt_limit_interrupt_pending+ && state.active_provider_turn_id.is_some(),+ state.receipt_limit_interrupt_attempts,+ state.receipt_limit_interrupt_deadline_unix_ms,+ )+ });+ if !should_retry {+ return Ok(());+ }+ let now_unix_ms = current_unix_ms()?;+ let deadline_unix_ms = match persisted_deadline {+ Some(deadline) => deadline,+ None => {+ // Older durable state did not record this additive field. Give+ // an already-pending interruption one complete bounded window+ // after recovery rather than falling back on poll count alone.+ let timeout_ms = if self+ .state+ .as_ref()+ .is_some_and(|state| state.receipt_limit_interrupt_accepted)+ {+ RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS+ } else {+ RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS+ };+ let deadline = receipt_limit_deadline_after(timeout_ms)?;+ self.state+ .as_mut()+ .expect("Codex state remains available while adding its receipt-limit deadline")+ .receipt_limit_interrupt_deadline_unix_ms = Some(deadline);+ self.save_state()?;+ deadline+ }+ };+ // Attempts bound network traffic, while the durable wall-clock deadline+ // gives an accepted asynchronous interruption time to deliver its+ // authoritative terminal. The provider is always polled once more below+ // before an elapsed deadline is converted into the conservative fallback.+ if attempts >= MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS || now_unix_ms >= deadline_unix_ms {+ return Ok(());+ }+ // Polling is the autonomous recovery path until a terminal notification+ // clears the durable marker. RPC acceptance alone does not establish+ // that Codex stopped the turn, so accepted-but-unsettled interruptions+ // remain idempotently retryable across polls and process restarts.+ self.state+ .as_mut()+ .expect("Codex state remains available before receipt-limit retry")+ .record_receipt_limit_interrupt_attempt()?;+ self.save_state()?;+ if self+ .interrupt_turn("semantic_tool_turn_receipt_limit_retry")+ .is_ok()+ && self.state.as_ref().is_some_and(|state| {+ state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some()+ })+ {+ let accepted_deadline_unix_ms =+ receipt_limit_deadline_after(RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS)?;+ self.state+ .as_mut()+ .expect("Codex state remains available after receipt-limit retry")+ .mark_receipt_limit_interrupt_accepted(accepted_deadline_unix_ms);+ self.save_state()?;+ }+ Ok(())+ }++ fn settle_receipt_limit_interrupt_if_deadline_elapsed(+ &mut self,+ ) -> Result<(), DurableRunnerError> {+ let deadline = self.state.as_ref().and_then(|state| {+ (state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some())+ .then_some(state.receipt_limit_interrupt_deadline_unix_ms)+ .flatten()+ });+ let Some(deadline) = deadline else {+ return Ok(());+ };+ if current_unix_ms()? >= deadline {+ self.settle_receipt_limit_interrupt_after_deadline()?;+ }+ Ok(())+ }++ fn handle_tool_call(+ &mut self,+ call_id: String,+ operation_id: String,+ input: Value,+ ) -> Result<(), DurableRunnerError> {+ let identity = self.event_identity()?;+ let admission = self+ .state+ .as_mut()+ .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?+ .admit_tool_call(&call_id, &operation_id, &input);+ match admission {+ Ok(ToolCallAdmission::CompletedReplay(result)) => {+ self.provider+ .as_mut()+ .expect("provider remains present while handling its tool call")+ .deliver_tool_result(&result)+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to replay a durable semantic tool result: {error}"+ ))+ })?;+ return Ok(());+ }+ Ok(ToolCallAdmission::PendingReplay) => return Ok(()),+ Err(error) => {+ if error.is_active_turn_receipt_limit() {+ return self.stop_turn_at_tool_receipt_limit(call_id, operation_id);+ }+ return self.reject_tool_call(call_id, operation_id, error.to_string());+ }+ Ok(ToolCallAdmission::Pending(call)) => {+ self.state+ .as_mut()+ .expect("Codex state remains available while accepting a tool call")+ .push_event(semantic_input_event(&identity, &call))?;+ self.save_state()+ }+ }+ }++ fn deliver_semantic_result(+ &mut self,+ payload: &Value,+ ) -> Result<CommandExecution, DurableRunnerError> {+ let result: ToolResult = serde_json::from_value(payload.clone()).map_err(|error| {+ DurableRunnerError::invalid(format!("semantic tool result is invalid: {error}"))+ })?;+ let identity = self.event_identity()?;+ let was_completed = self+ .state+ .as_ref()+ .is_some_and(|state| state.tool_bridge.has_completed_call(&result.call_id));+ let mut next_state = self+ .state+ .clone()+ .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?;+ next_state+ .tool_bridge+ .apply_result(result.clone())+ .map_err(|error| {+ DurableRunnerError::invalid(format!("semantic tool result was rejected: {error}"))+ })?;+ if was_completed {+ return Ok(CommandExecution::result(json!({+ "status": "duplicate",+ "callId": result.call_id,+ })));+ }+ next_state.push_event(semantic_result_event(&identity, &result))?;+ self.persist_state(&next_state)?;+ self.state = Some(next_state);+ self.ensure_provider()?+ .deliver_tool_result(&result)+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to return semantic tool result to Codex: {error}"+ ))+ })?;+ Ok(CommandExecution::result(json!({+ "status": "delivered",+ "callId": result.call_id,+ })))+ }+fn close_session(&mut self) -> Result<CommandExecution, DurableRunnerError> {if let Some(provider) = self.provider.as_mut() {provider.shutdown().map_err(|error| {@@ -830,6 +2148,11 @@ impl CodexCommandExecutor {.ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?;state.active_provider_turn_id = None;state.ambiguous_turn_start_pending = false;+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;state.lifecycle = "closed".to_owned();let thread_id = state.thread_id.clone();self.save_state()?;@@ -865,15 +2188,24 @@ impl CodexCommandExecutor {// an ambiguous-start failure cannot degrade into an empty successful// poll on the same executor.self.restore_provider_if_needed()?;- if self- .state- .as_ref()- .is_some_and(|state| !state.pending_events.is_empty())+ // Receipt-limit interruption is autonomous recovery. It must advance+ // even while older durable events await acknowledgement, otherwise a+ // slow or disconnected controller can keep an exhausted provider turn+ // alive forever. Terminal settlement uses the reserved event capacity.+ self.retry_receipt_limit_interrupt()?;+ let receipt_limit_terminal_poll = self.state.as_ref().is_some_and(|state| {+ state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some()+ });+ if !receipt_limit_terminal_poll+ && self+ .state+ .as_ref()+ .is_some_and(|state| !state.pending_events.is_empty()){return Ok(());}if self.provider.is_none() {- return Ok(());+ return self.settle_receipt_limit_interrupt_if_deadline_elapsed();}for _ in 0..MAX_EVENTS_PER_POLL {let event = self@@ -889,11 +2221,9 @@ impl CodexCommandExecutor {CodexProviderEvent::ToolCall {call_id,operation_id,- ..+ input,} => {- return Err(DurableRunnerError::invalid(format!(- "Codex emitted semantic tool call {call_id} for {operation_id} before the durable tool bridge was attached"- )));+ self.handle_tool_call(call_id, operation_id, input)?;}CodexProviderEvent::Notification { method, params } => {let active_provider_turn_id = if method == "turn/started" {@@ -904,14 +2234,17 @@ impl CodexCommandExecutor {} else {None};- let completed_turn_authority = if method == "turn/completed" {- self.provider- .as_ref()- .and_then(CodexProvider::completed_turn_authority)- .map(|(generation, turn_id)| (generation, turn_id.to_owned()))- } else {- None- };+ let normalized_terminal_type =+ normalized_codex_terminal_event_type(&method, ¶ms);+ let completed_turn_authority =+ if normalized_terminal_type == Some("turn.completed") {+ self.provider+ .as_ref()+ .and_then(CodexProvider::completed_turn_authority)+ .map(|(generation, turn_id)| (generation, turn_id.to_owned()))+ } else {+ None+ };let normalized = normalize_codex_notification(&method, ¶ms);let terminal_event_type = normalized.iter()@@ -926,6 +2259,7 @@ impl CodexCommandExecutor {| "turn.interrupted")});+ let identity = self.event_identity.clone();let state = self.state.as_mut()@@ -948,23 +2282,63 @@ impl CodexCommandExecutor {})?;state.reconcile_active_provider_turn(Some(provider_turn_id));}- if method == "turn/completed" {- let (process_generation, provider_turn_id) = completed_turn_authority- .ok_or_else(|| {+ if terminal_event_type.is_some() {+ state.settle_active_provider_turn_identity()?;+ let settled = state+ .tool_bridge+ .settle_turn("provider_turn_terminated")+ .map_err(|error| {+ DurableRunnerError::invalid(format!(+ "failed to settle semantic tools at turn termination: {error}"+ ))+ })?;+ if !settled.is_empty() {+ let identity = identity.as_ref().ok_or_else(|| {DurableRunnerError::invalid(- "Codex completion omitted process and turn authority",+ "Codex semantic tool events require the durable runner identity",)})?;+ for result in settled {+ state.push_terminal_event(semantic_result_event(+ identity, &result,+ ))?;+ }+ }state.active_provider_turn_id = None;- state.completed_turn_authoritative = true;- state.completed_turn_process_generation = Some(process_generation);- state.completed_provider_turn_id = Some(provider_turn_id);+ if terminal_event_type.as_deref() == Some("turn.completed") {+ let (process_generation, provider_turn_id) = completed_turn_authority+ .ok_or_else(|| {+ DurableRunnerError::invalid(+ "Codex completion omitted process and turn authority",+ )+ })?;+ state.completed_turn_authoritative = true;+ state.completed_turn_process_generation = Some(process_generation);+ state.completed_provider_turn_id = Some(provider_turn_id);+ } else {+ state.completed_turn_authoritative = false;+ state.completed_turn_process_generation = None;+ state.completed_provider_turn_id = None;+ }+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;state.ambiguous_turn_start_pending = false;state.lifecycle = "session_open".to_owned();}- state.extend_events(normalized)?;+ if terminal_event_type.is_some() {+ state.extend_terminal_events(normalized)?;+ } else if receipt_limit_terminal_poll {+ for event in normalized {+ state.push_receipt_limit_cleanup_event(event)?;+ }+ } else {+ state.extend_events(normalized)?;+ }if let Some(event_type) = terminal_event_type {- state.extend_events(terminal_events(state, &event_type))?;+ state.extend_terminal_events(terminal_events(state, &event_type))?;}self.save_state()?;}@@ -977,29 +2351,35 @@ impl CodexCommandExecutor {.or_else(|| question_set.pointer("/questions/0/prompt")).and_then(Value::as_str).unwrap_or("Codex needs your input");- self.state+ let state = self+ .state.as_mut()- .expect("Codex state remains available while polling")- .push_event(NormalizedProviderEvent {- event_type: "runtime_request.created".to_owned(),- priority: EventPriority::P0,- payload: json!({- "request": {- "schema": "paperclip.runtime_request.v2",- "requestKind": "runtime",- "requestId": request_id,- "type": "input",- "status": "pending",- "prompt": prompt,- "input": question_set,- "origin": {- "adapter": "codex-app-server",- "provider": "codex",- "method": "item/tool/requestUserInput",- },+ .expect("Codex state remains available while polling");+ let event = NormalizedProviderEvent {+ event_type: "runtime_request.created".to_owned(),+ priority: EventPriority::P0,+ payload: json!({+ "request": {+ "schema": "paperclip.runtime_request.v2",+ "requestKind": "runtime",+ "requestId": request_id,+ "type": "input",+ "status": "pending",+ "prompt": prompt,+ "input": question_set,+ "origin": {+ "adapter": "codex-app-server",+ "provider": "codex",+ "method": "item/tool/requestUserInput",},- }),- })?;+ },+ }),+ };+ if receipt_limit_terminal_poll {+ state.push_receipt_limit_cleanup_event(event)?;+ } else {+ state.push_event(event)?;+ }self.save_state()?;}CodexProviderEvent::Exited {@@ -1022,7 +2402,7 @@ impl CodexCommandExecutor {// whether this separate session exit belongs to that// completion or is a later idle-provider failure.state.lifecycle = "provider_exited".to_owned();- state.push_event(NormalizedProviderEvent {+ state.push_terminal_event(NormalizedProviderEvent {// A completed turn remains authoritative, while the// reusable provider session independently becomes// unavailable. Avoid emitting session.failed for@@ -1053,7 +2433,11 @@ impl CodexCommandExecutor {}}}- Ok(())+ // Check the durable deadline only after a complete provider poll. A+ // terminal that arrived after interruption acceptance but before this+ // observation remains authoritative even when several fast controller+ // polls have already exhausted the interruption-attempt budget.+ self.settle_receipt_limit_interrupt_if_deadline_elapsed()}}@@ -1066,6 +2450,7 @@ impl CommandExecutor for CodexCommandExecutor {if self.state.is_none() && command.payload.get("provider").is_some() {self.prepare(&command.payload)?;}+ self.verify_attached_tools(&command.payload)?;let mut execution = self.open_session()?;execution.events.push(("run.attached".to_owned(),@@ -1081,6 +2466,7 @@ impl CommandExecutor for CodexCommandExecutor {self.interrupt_turn(&command.command_type)}"request.resolve" => self.resolve_request(&command.payload),+ "semantic_tool.result" => self.deliver_semantic_result(&command.payload),"session.snapshot" => self.snapshot(),"session.close" | "session.destroy" => self.close_session(),"runner.drain" | "runner.suspend" | "runner.shutdown" => {@@ -1119,6 +2505,7 @@ impl CommandExecutor for CodexCommandExecutor {));}next_state.pending_events.drain(..count);+ next_state.refill_pending_events();self.persist_state(&next_state)?;self.state = Some(next_state);Ok(())@@ -1160,6 +2547,7 @@ mod tests {approval_policy: "never".to_owned(),},completion_contract: None,+ tool_bridge: ProviderToolBridge::default(),thread_id: Some("thread-1".to_owned()),provider_session_id: None,active_provider_turn_id: None,@@ -1168,8 +2556,16 @@ mod tests {provider_process_generation: 0,completed_turn_process_generation: None,completed_provider_turn_id: None,+ settled_provider_turn_ids: std::collections::BTreeSet::new(),+ settled_provider_turn_filter: DurableReplayFilter::default(),+ receipt_limit_diagnostic_emitted: false,+ receipt_limit_interrupt_pending: false,+ receipt_limit_interrupt_accepted: false,+ receipt_limit_interrupt_attempts: 0,+ receipt_limit_interrupt_deadline_unix_ms: None,last_agent_message: None,pending_events: VecDeque::new(),+ queued_events: VecDeque::new(),next_provider_event_seq: initial_provider_event_seq(),};assert!(state.validate().is_err());@@ -1194,6 +2590,7 @@ mod tests {approval_policy: "never".to_owned(),},None,+ ProviderToolBridge::default(),);state.thread_id = Some("thread-1".to_owned());state.lifecycle = "session_open".to_owned();@@ -1236,6 +2633,7 @@ mod tests {revision: "1".to_owned(),criterion_ids: vec!["objective".to_owned()],}),+ ProviderToolBridge::default(),);state.last_agent_message = Some("Finished the requested work.".to_owned());let events = terminal_events(&state, "turn.completed");@@ -1244,4 +2642,752 @@ mod tests {assert_eq!(events[1].event_type, "run.terminal");assert_eq!(events[1].payload["runTerminalState"], "succeeded");}++ #[test]+ fn semantic_input_digest_covers_the_transmitted_redacted_value() {+ let identity = ProviderEventIdentity {+ run_id: "run-1".to_owned(),+ normalized_session_id: "session-1".to_owned(),+ turn_id: "turn-1".to_owned(),+ item_id: "item-1".to_owned(),+ };+ let call = PendingToolCall {+ call_id: "call-1".to_owned(),+ operation_id: "get_task_context".to_owned(),+ input: json!({"password": "do-not-persist", "safe": true}),+ };+ let event = semantic_input_event(&identity, &call);+ let transmitted = &event.payload["semantic_tool"]["input"];+ assert_eq!(transmitted["password"], "[REDACTED]");+ assert_eq!(+ event.payload["semantic_tool"]["content"]["digest"],+ semantic_value_digest(transmitted)+ );+ }++ #[test]+ fn receipt_limit_diagnostic_is_durable_and_turn_idempotent() {+ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ ProviderToolBridge::default(),+ );+ state.thread_id = Some("thread-1".to_owned());+ state.active_provider_turn_id = Some("turn-1".to_owned());+ state.lifecycle = "turn_active".to_owned();++ assert!(state+ .begin_receipt_limit_stop("call-first".to_owned(), "tool.first".to_owned(), 10_000)+ .unwrap());+ state.mark_receipt_limit_interrupt_accepted(50_000);+ assert!(state+ .begin_receipt_limit_stop("call-second".to_owned(), "tool.second".to_owned(), 20_000)+ .unwrap());+ assert_eq!(state.pending_events.len(), 1);+ assert_eq!(state.pending_events[0].payload["callId"], "call-first");+ assert_eq!(state.receipt_limit_interrupt_deadline_unix_ms, Some(50_000));++ let mut recovered: CodexProviderState =+ serde_json::from_slice(&serde_json::to_vec(&state).unwrap()).unwrap();+ recovered.validate().unwrap();+ assert!(recovered.receipt_limit_interrupt_accepted);+ assert_eq!(+ recovered.receipt_limit_interrupt_deadline_unix_ms,+ Some(50_000)+ );+ assert!(recovered+ .begin_receipt_limit_stop(+ "call-after-restart".to_owned(),+ "tool.third".to_owned(),+ 30_000,+ )+ .unwrap());+ assert_eq!(recovered.pending_events.len(), 1);+ // Only a terminal notification clears the retry marker. Accepting an+ // interrupt request does not prove that the provider stopped.+ assert!(recovered+ .begin_receipt_limit_stop(+ "call-after-success".to_owned(),+ "tool.fourth".to_owned(),+ 40_000,+ )+ .unwrap());+ assert_eq!(recovered.pending_events.len(), 1);+ }++ #[test]+ fn settled_receipt_limit_interrupt_cannot_be_marked_accepted() {+ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ ProviderToolBridge::default(),+ );+ state.thread_id = Some("thread-1".to_owned());+ state.active_provider_turn_id = Some("turn-1".to_owned());+ state.lifecycle = "turn_active".to_owned();+ state+ .begin_receipt_limit_stop("call-1".to_owned(), "tool.one".to_owned(), 10_000)+ .unwrap();+ state.record_receipt_limit_interrupt_attempt().unwrap();++ // Recovery observed that the turn ended before the interrupt RPC was+ // issued and cleared the receipt-limit interruption state.+ state.settle_active_provider_turn_identity().unwrap();+ state.active_provider_turn_id = None;+ state.lifecycle = "session_open".to_owned();+ state.receipt_limit_diagnostic_emitted = false;+ state.receipt_limit_interrupt_pending = false;+ state.receipt_limit_interrupt_accepted = false;+ state.receipt_limit_interrupt_attempts = 0;+ state.receipt_limit_interrupt_deadline_unix_ms = None;++ state.mark_receipt_limit_interrupt_accepted(50_000);++ assert!(!state.receipt_limit_interrupt_accepted);+ assert!(state.receipt_limit_interrupt_deadline_unix_ms.is_none());+ state.validate().unwrap();+ }++ #[test]+ fn regular_backlog_preserves_receipt_limit_and_terminal_settlement_capacity() {+ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ ProviderToolBridge::default(),+ );+ let identity = ProviderEventIdentity {+ run_id: "run-1".to_owned(),+ normalized_session_id: "session-1".to_owned(),+ turn_id: "turn-1".to_owned(),+ item_id: "item-1".to_owned(),+ };+ let ordinary_event = || NormalizedProviderEvent {+ event_type: "harness.diagnostic".to_owned(),+ priority: EventPriority::P1,+ payload: json!({"code": "ordinary_backlog"}),+ };+ for _ in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) {+ state.push_event(ordinary_event()).unwrap();+ }+ assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL);+ assert_eq!(+ state.queued_events.len(),+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS+ );+ assert!(state.push_event(ordinary_event()).is_err());++ for _ in 0..MAX_EVENTS_PER_POLL {+ state+ .push_receipt_limit_cleanup_event(ordinary_event())+ .unwrap();+ }+ let cleanup_boundary = state.queued_events.len();+ state+ .push_receipt_limit_cleanup_event(ordinary_event())+ .expect("cleanup overflow is dropped while preserving terminal capacity");+ assert_eq!(state.queued_events.len(), cleanup_boundary);++ for index in 0..MAX_PENDING_CALLS {+ state+ .push_terminal_event(semantic_result_event(+ &identity,+ &ToolResult {+ call_id: format!("call-{index}"),+ operation_id: "get_task_context".to_owned(),+ result: json!({"error": {"code": "provider_turn_terminated"}}),+ is_error: true,+ },+ ))+ .unwrap();+ }+ for event_type in [+ "harness.diagnostic",+ "turn.completed",+ "run.result.proposed",+ "run.terminal",+ ] {+ state+ .push_terminal_event(NormalizedProviderEvent {+ event_type: event_type.to_owned(),+ priority: EventPriority::P0,+ payload: json!({"terminal": true}),+ })+ .unwrap();+ }++ assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL);+ assert_eq!(state.queued_events.len(), MAX_QUEUED_PROVIDER_EVENTS);+ state.validate().unwrap();+ assert!(state.push_terminal_event(ordinary_event()).is_err());+ assert!(serde_json::to_vec(&state).unwrap().len() as u64 <= MAX_PROVIDER_STATE_BYTES);+ }++ #[test]+ fn completed_replays_are_read_only_at_the_regular_event_boundary() {+ let operation = crate::provider_bridge::AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation],+ })+ .unwrap();+ bridge+ .begin_call(+ "call-replayed".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .unwrap();+ let replayed_result = ToolResult {+ call_id: "call-replayed".to_owned(),+ operation_id: "get_task_context".to_owned(),+ result: json!({"ok": true}),+ is_error: false,+ };+ bridge.apply_result(replayed_result.clone()).unwrap();++ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ bridge,+ );+ let identity = ProviderEventIdentity {+ run_id: "run-1".to_owned(),+ normalized_session_id: "session-1".to_owned(),+ turn_id: "turn-1".to_owned(),+ item_id: "item-1".to_owned(),+ };++ // Keep the pending window occupied, then model the input and result+ // receipts retained by the maximum 4,096 completed calls. Only three+ // regular queued-event slots remain at this boundary.+ for index in 0..MAX_EVENTS_PER_POLL {+ state+ .push_event(NormalizedProviderEvent {+ event_type: "provider.notice.recorded".to_owned(),+ priority: EventPriority::P1,+ payload: json!({"index": index}),+ })+ .unwrap();+ }+ for index in 0..MAX_PENDING_CALLS {+ let call = PendingToolCall {+ call_id: format!("call-{index}"),+ operation_id: "get_task_context".to_owned(),+ input: json!({}),+ };+ state+ .push_event(semantic_input_event(&identity, &call))+ .unwrap();+ state+ .push_event(semantic_result_event(+ &identity,+ &ToolResult {+ call_id: call.call_id,+ operation_id: call.operation_id,+ result: json!({"ok": true}),+ is_error: false,+ },+ ))+ .unwrap();+ }+ assert_eq!(+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS - state.queued_events.len(),+ 3+ );+ let pending_len = state.pending_events.len();+ let queued_len = state.queued_events.len();+ let next_sequence = state.next_provider_event_seq;++ for _ in 0..4 {+ assert_eq!(+ state+ .admit_tool_call("call-replayed", "get_task_context", &json!({}))+ .unwrap(),+ ToolCallAdmission::CompletedReplay(replayed_result.clone())+ );+ }+ assert_eq!(state.pending_events.len(), pending_len);+ assert_eq!(state.queued_events.len(), queued_len);+ assert_eq!(state.next_provider_event_seq, next_sequence);+ state.validate().unwrap();+ }++ #[test]+ fn pending_replays_are_read_only_at_the_regular_event_boundary() {+ let operation = crate::provider_bridge::AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation],+ })+ .unwrap();+ bridge+ .begin_call(+ "call-pending".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .unwrap();++ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ bridge,+ );+ for index in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) {+ state+ .push_event(NormalizedProviderEvent {+ event_type: "provider.notice.recorded".to_owned(),+ priority: EventPriority::P1,+ payload: json!({"index": index}),+ })+ .unwrap();+ }+ assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL);+ assert_eq!(+ state.queued_events.len(),+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS+ );+ let next_sequence = state.next_provider_event_seq;++ for _ in 0..4 {+ assert_eq!(+ state+ .admit_tool_call("call-pending", "get_task_context", &json!({}))+ .unwrap(),+ ToolCallAdmission::PendingReplay+ );+ }+ assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL);+ assert_eq!(+ state.queued_events.len(),+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS+ );+ assert_eq!(state.next_provider_event_seq, next_sequence);+ state.validate().unwrap();+ }++ #[test]+ fn exact_regular_backlog_capacity_rejects_turn_admission() {+ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ ProviderToolBridge::default(),+ );+ state.lifecycle = "prepared".to_owned();+ for _ in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) {+ state+ .push_event(NormalizedProviderEvent {+ event_type: "harness.diagnostic".to_owned(),+ priority: EventPriority::P1,+ payload: json!({"code": "admission_boundary"}),+ })+ .unwrap();+ }+ assert_eq!(+ state.queued_events.len(),+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS,+ );+ let mut executor = CodexCommandExecutor::new(PathBuf::from("unused-test-state"));+ executor.state = Some(state);+ executor.restore_checked = true;++ let error = executor+ .start_turn(&json!({"text": "must not reach the provider"}))+ .expect_err("the exact regular backlog limit must reject admission");+ assert!(error+ .to_string()+ .contains("until terminal events are acknowledged"));+ assert!(executor.provider.is_none());+ }++ #[test]+ fn transient_receipt_limit_clears_before_a_later_turn() {+ let operation = crate::provider_bridge::AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation],+ })+ .unwrap();+ let mut encoded = serde_json::to_value(&bridge).unwrap();+ encoded["durableRunReceiptLimitReached"] = Value::Bool(true);+ let mut bridge: ProviderToolBridge = serde_json::from_value(encoded).unwrap();+ bridge.attach_existing_run().unwrap();++ assert!(bridge.durable_run_receipt_limit_reached());+ bridge.prepare_turn().unwrap();+ assert!(!bridge.durable_run_receipt_limit_reached());+ }++ #[test]+ fn restore_reattaches_the_durable_tool_result_byte_counter() {+ let directory = std::env::temp_dir().join(format!(+ "paperclip-provider-tool-byte-restore-{}",+ std::process::id()+ ));+ let _ = fs::remove_dir_all(&directory);+ fs::create_dir_all(&directory).unwrap();+ let operation = crate::provider_bridge::AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation],+ })+ .unwrap();+ bridge+ .begin_call(+ "call-1".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .unwrap();+ bridge+ .apply_result(ToolResult {+ call_id: "call-1".to_owned(),+ operation_id: "get_task_context".to_owned(),+ result: json!({"ok": true}),+ is_error: false,+ })+ .unwrap();+ bridge.settle_turn("provider_turn_terminated").unwrap();+ assert!(bridge.retained_result_bytes_for_test() > 0);++ let state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ bridge,+ );+ let writer = CodexCommandExecutor::new(&directory);+ writer.persist_state(&state).unwrap();++ let mut recovered = CodexCommandExecutor::new(&directory);+ recovered.restore().unwrap();+ assert!(+ recovered+ .state+ .as_ref()+ .unwrap()+ .tool_bridge+ .retained_result_bytes_for_test()+ > 0+ );+ fs::remove_dir_all(directory).unwrap();+ }++ #[test]+ fn durable_provider_turn_ledger_backfills_legacy_completion_authority() {+ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ ProviderToolBridge::default(),+ );+ state.completed_turn_authoritative = true;+ state.completed_turn_process_generation = Some(1);+ state.completed_provider_turn_id = Some("provider-turn-legacy".to_owned());+ state.provider_process_generation = 1;++ let (recovered, recovered_filter) = state.recovered_settled_provider_turn_ids().unwrap();++ assert!(settled_provider_turn_contains(+ &recovered,+ &recovered_filter,+ "provider-turn-legacy"+ ));+ }++ #[test]+ fn durable_provider_turn_ledger_never_evicts_within_an_epoch() {+ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ ProviderToolBridge::default(),+ );+ state.thread_id = Some("thread-1".to_owned());+ state.lifecycle = "turn_active".to_owned();+ for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS - 1 {+ state+ .settled_provider_turn_ids+ .insert(format!("provider-turn-{index:04}"));+ }+ state.active_provider_turn_id = Some("provider-turn-final".to_owned());++ state.settle_active_provider_turn_identity().unwrap();+ state.active_provider_turn_id = None;+ state.lifecycle = "session_open".to_owned();++ assert_eq!(+ state.settled_provider_turn_ids.len(),+ MAX_SETTLED_PROVIDER_TURN_IDS+ );+ assert!(state.settled_provider_turn_filter.is_empty());+ assert!(state+ .settled_provider_turn_ids+ .contains("provider-turn-final"));+ assert!(state+ .settled_provider_turn_ids+ .contains("provider-turn-0000"));++ state.lifecycle = "turn_active".to_owned();+ state.active_provider_turn_id = Some("provider-turn-overflow".to_owned());+ assert!(state.settle_active_provider_turn_identity().is_err());+ assert!(!state+ .settled_provider_turn_ids+ .contains("provider-turn-overflow"));+ state.active_provider_turn_id = None;+ state.lifecycle = "session_open".to_owned();+ state.validate().unwrap();+ let recovered: CodexProviderState =+ serde_json::from_str(&serde_json::to_string(&state).unwrap()).unwrap();+ assert_eq!(+ recovered.settled_provider_turn_ids,+ state.settled_provider_turn_ids+ );+ assert_eq!(+ recovered.settled_provider_turn_filter,+ state.settled_provider_turn_filter+ );+ recovered.validate().unwrap();+ }++ #[test]+ fn receipt_limit_deadline_settlement_preserves_unacknowledged_events() {+ let directory = std::env::temp_dir().join(format!(+ "paperclip-provider-receipt-deadline-{}",+ std::process::id()+ ));+ let _ = fs::remove_dir_all(&directory);+ fs::create_dir_all(&directory).unwrap();+ let mut state = CodexProviderState::new(+ CodexProviderConfig {+ provider: "codex".to_owned(),+ driver: "codex_app_server".to_owned(),+ provider_version: "test".to_owned(),+ command: PathBuf::from("codex"),+ args: vec!["app-server".to_owned()],+ cwd: std::env::current_dir()+ .unwrap()+ .to_string_lossy()+ .into_owned(),+ model: None,+ provider_session_id: None,+ instructions: String::new(),+ approval_policy: "never".to_owned(),+ },+ None,+ ProviderToolBridge::default(),+ );+ state.thread_id = Some("thread-1".to_owned());+ state.active_provider_turn_id = Some("turn-1".to_owned());+ state.lifecycle = "turn_active".to_owned();+ state.receipt_limit_diagnostic_emitted = true;+ state.receipt_limit_interrupt_pending = true;+ state.receipt_limit_interrupt_attempts = MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS;+ state.receipt_limit_interrupt_deadline_unix_ms = Some(1);+ state+ .push_event(NormalizedProviderEvent {+ event_type: "provider.notice.recorded".to_owned(),+ priority: EventPriority::P1,+ payload: json!({"message": "awaiting acknowledgement"}),+ })+ .unwrap();+ let mut executor = CodexCommandExecutor::new(&directory);+ executor.state = Some(state);+ executor.event_identity = Some(ProviderEventIdentity {+ run_id: "run-1".to_owned(),+ normalized_session_id: "session-1".to_owned(),+ turn_id: "turn-1".to_owned(),+ item_id: "item-1".to_owned(),+ });+ executor.restore_checked = true;++ // Exercise receipt-limit recovery directly. `poll_provider` also+ // restores a missing provider by design, while this unit test+ // intentionally injects state without constructing a provider.+ executor.retry_receipt_limit_interrupt().unwrap();+ executor+ .settle_receipt_limit_interrupt_if_deadline_elapsed()+ .unwrap();++ let state = executor.state.as_ref().unwrap();+ assert_eq!(state.lifecycle, "provider_exited");+ assert!(state.active_provider_turn_id.is_none());+ assert!(!state.receipt_limit_interrupt_pending);+ assert!(state.pending_events.iter().any(|event| {+ event.event_type == "provider.notice.recorded"+ && event.payload == json!({"message": "awaiting acknowledgement"})+ }));+ assert!(state+ .pending_events+ .iter()+ .any(|event| event.event_type == "turn.failed"));+ fs::remove_dir_all(directory).unwrap();+ }}diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rsindex f5ef365baf..62213d3378 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs@@ -1,9 +1,9 @@-use std::collections::{BTreeMap, BTreeSet};+use std::collections::{BTreeMap, BTreeSet, VecDeque};use std::error::Error;use std::fmt::{self, Display, Formatter};use serde::de::{self, MapAccess, Visitor};-use serde::{Deserialize, Deserializer, Serialize};+use serde::{Deserialize, Deserializer, Serialize, Serializer};use serde_json::Value;use sha2::{Digest, Sha256};@@ -12,18 +12,32 @@ pub const TOOL_CALL_SCHEMA: &str = "paperclip.prp.semantic_tool.v1";pub const TOOL_RESULT_COMMAND: &str = "semantic_tool.result";const MAX_AUTHORIZED_TOOLS: usize = 256;const MAX_DESCRIPTION_BYTES: usize = 16 * 1024;-const MAX_SCHEMA_BYTES: usize = 1024 * 1024;-const MAX_TOOL_SET_BYTES: usize = 4 * 1024 * 1024;-const MAX_TOOL_VALUE_BYTES: usize = 1024 * 1024;-// Settled results are authoritative replay receipts and live for the durable-// run. Bound their complete encoded map, while reserving enough room for every-// active call to later produce a maximum-sized result. The 1 KiB allowance-// covers the map key, bounded call/operation identities, JSON field names, and-// escaping around a 1 MiB result value.+const MAX_SCHEMA_BYTES: usize = 512 * 1024;+// Leave room for the authenticated PRP command or event envelope inside the+// default 1 MiB transport frame.+const MAX_TOOL_SET_BYTES: usize = 768 * 1024;+const MAX_TOOL_VALUE_BYTES: usize = 768 * 1024;+const MAX_ACCEPTED_TOOL_VALUE_BYTES: usize = 4 * 1024 * 1024;+const MAX_RETAINED_TOOL_VALUE_BYTES: usize = 8 * 1024 * 1024;+// Exact settled receipts are authoritative for the durable run. Bound their+// complete encoded representation independently of the raw-value budget so a+// long sequence of tiny results cannot accumulate unbounded identity and JSON+// envelope overhead.const MAX_SETTLED_RESULT_BYTES: usize = 8 * 1024 * 1024;-const MAX_SETTLED_RESULT_ENTRY_BYTES: usize = MAX_TOOL_VALUE_BYTES + 1024;-const MAX_RETAINED_CALLS: usize = 4_096;+pub(crate) const MAX_PENDING_CALLS: usize = 4_096;+// Active-turn inputs and results are separately bounded. Settled identities+// remain exact for a complete provider-process epoch so a delayed request+// cannot replay an old call ID after crossing a turn boundary. At the bound,+// the backend must reap the idle process before it rotates this ledger.+const MAX_DURABLE_CALL_RECEIPTS: usize = 4_096;const MAX_SETTLED_CALL_IDS: usize = 65_536;+// Retain the legacy serialized filter shape for recovery compatibility. New+// state never inserts probabilistic identities. A recovered non-empty filter+// makes the durable run fail closed until attach_run because its identities+// cannot safely be distinguished from fresh calls.+const REPLAY_FILTER_WORDS: usize = 32_768;+const ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE: &str =+ "durable provider tool receipt limit reached for the active turn";#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]#[serde(rename_all = "camelCase")]@@ -62,26 +76,145 @@ pub struct ToolResult {pub is_error: bool,}+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]+#[serde(rename_all = "camelCase")]+struct CompletedToolCall {+ call: PendingToolCall,+ result: ToolResult,+}++#[derive(Clone, Debug, Default, PartialEq)]+struct SettledCallIds {+ order: VecDeque<String>,+ members: BTreeSet<String>,+}++// Preserve the existing on-disk array shape so durable state written before+// this bounded ordering was introduced remains readable. New state records IDs+// from oldest to newest, which makes pruning deterministic across recovery.+impl Serialize for SettledCallIds {+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>+ where+ S: Serializer,+ {+ self.order.serialize(serializer)+ }+}++impl<'de> Deserialize<'de> for SettledCallIds {+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>+ where+ D: Deserializer<'de>,+ {+ let values = VecDeque::<String>::deserialize(deserializer)?;+ let mut settled = Self::default();+ for value in values {+ if settled.members.insert(value.clone()) {+ settled.order.push_back(value);+ }+ }+ Ok(settled)+ }+}++impl SettledCallIds {+ fn clear(&mut self) {+ self.order.clear();+ self.members.clear();+ }++ fn contains(&self, call_id: &str) -> bool {+ self.members.contains(call_id)+ }++ fn is_empty(&self) -> bool {+ self.order.is_empty()+ }++ fn len(&self) -> usize {+ self.order.len()+ }++ fn iter(&self) -> impl Iterator<Item = &String> {+ self.order.iter()+ }++ fn extend_recent(+ &mut self,+ call_ids: impl IntoIterator<Item = String>,+ limit: usize,+ ) -> Vec<String> {+ for call_id in call_ids {+ if self.members.insert(call_id.clone()) {+ self.order.push_back(call_id);+ }+ }+ let mut evicted = Vec::new();+ while self.order.len() > limit {+ if let Some(call_id) = self.order.pop_front() {+ self.members.remove(&call_id);+ evicted.push(call_id);+ }+ }+ evicted+ }+}++#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]+pub(crate) struct DurableReplayFilter {+ words: Vec<u64>,+}++impl DurableReplayFilter {+ pub(crate) fn is_empty(&self) -> bool {+ self.words.is_empty()+ }++ pub(crate) fn validate(&self) -> Result<(), ProviderBridgeError> {+ if self.words.is_empty() || self.words.len() == REPLAY_FILTER_WORDS {+ Ok(())+ } else {+ Err(ProviderBridgeError::invalid(+ "durable replay filter has an invalid size",+ ))+ }+ }+}+#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]#[serde(rename_all = "camelCase")]pub struct ProviderToolBridge {authorized: BTreeMap<String, AuthorizedTool>,+ #[serde(default)]+ catalog_operations: Vec<AuthorizedTool>,catalog_digest: Option<String>,pending: BTreeMap<String, PendingToolCall>,#[serde(deserialize_with = "deserialize_retained_results")]- completed: BTreeMap<String, ToolResult>,+ completed: BTreeMap<String, CompletedToolCall>,+ // Keep authoritative result bodies through the provider's replay window;+ // prepare_turn releases them after settlement.#[serde(default, deserialize_with = "deserialize_retained_results")]- settled_results: BTreeMap<String, ToolResult>,+ settled_results: BTreeMap<String, CompletedToolCall>,// Derived from completed + settled results. It is intentionally omitted// from durable JSON and recomputed by attach_existing_run so old state and// tampered counters cannot bypass the byte envelope.#[serde(skip)]retained_result_bytes: usize,- // Compatibility tombstones for state written before settled results were- // retained. They still fail closed on call-id reuse, but cannot replay a- // value that the older state format discarded.+ // Exact replay tombstones span every provider turn in the durable run.#[serde(default)]- settled_call_ids: BTreeSet<String>,+ settled_call_ids: SettledCallIds,+ #[serde(default)]+ settled_call_filter: DurableReplayFilter,+ // Resource exhaustion stops the active turn. prepare_turn releases bulky+ // result payloads and recomputes this marker, but replay ambiguity remains+ // fail-closed until attach_run or a verified idle provider-process restart+ // establishes fresh replay authority.+ #[serde(+ default,+ alias = "settledHistoryResetPending",+ skip_serializing_if = "is_false"+ )]+ durable_run_receipt_limit_reached: bool,}#[derive(Clone, Debug, PartialEq, Eq)]@@ -91,6 +224,14 @@ impl ProviderBridgeError {fn invalid(message: impl Into<String>) -> Self {Self(message.into())}++ fn active_turn_receipt_limit() -> Self {+ Self::invalid(ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE)+ }++ pub fn is_active_turn_receipt_limit(&self) -> bool {+ self.0 == ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE+ }}impl Display for ProviderBridgeError {@@ -101,6 +242,10 @@ impl Display for ProviderBridgeError {impl Error for ProviderBridgeError {}+fn is_false(value: &bool) -> bool {+ !*value+}+impl ProviderToolBridge {pub fn prepare(&mut self, tool_set: AuthorizedToolSet) -> Result<(), ProviderBridgeError> {self.prepare_internal(tool_set, false)@@ -117,6 +262,8 @@ impl ProviderToolBridge {self.settled_results.clear();self.retained_result_bytes = 0;self.settled_call_ids.clear();+ self.settled_call_filter = DurableReplayFilter::default();+ self.durable_run_receipt_limit_reached = false;Ok(())}@@ -125,92 +272,9 @@ impl ProviderToolBridge {// preserve them so an interrupted dispatcher can resume or replay the// authoritative result. `attach_run` remains the boundary that rejects// carrying pending calls into a different run.- if self- .authorized- .iter()- .any(|(operation_id, tool)| operation_id != &tool.operation_id)- {- return Err(ProviderBridgeError::invalid(- "recovered authorized tool identities are inconsistent",- ));- }- let Some(catalog_digest) = self.catalog_digest.clone() else {- if self.authorized.is_empty()- && self.pending.is_empty()- && self.completed.is_empty()- && self.settled_results.is_empty()- && self.settled_call_ids.is_empty()- {- self.retained_result_bytes = 0;- return Ok(());- }- return Err(ProviderBridgeError::invalid(- "recovered authorized tools omit the catalog digest",- ));- };- let recovered_tool_set = AuthorizedToolSet {- schema: TOOL_SET_SCHEMA.to_owned(),- schema_version: 1,- catalog_digest,- operations: self.authorized.values().cloned().collect(),- };- validate_authorized_tool_set(&recovered_tool_set).map_err(|error| {- ProviderBridgeError::invalid(format!(- "recovered authorized tool catalog is invalid: {error}"- ))- })?;- if self- .settled_call_ids- .len()- .checked_add(self.settled_results.len())- .and_then(|total| total.checked_add(self.pending.len()))- .and_then(|total| total.checked_add(self.completed.len()))- .is_none_or(|total| total > MAX_SETTLED_CALL_IDS)- || self.pending.len().saturating_add(self.completed.len()) > MAX_RETAINED_CALLS- || self- .settled_call_ids- .iter()- .any(|call_id| !is_stable_call_id(call_id))- || self.settled_results.iter().any(|(call_id, result)| {- !is_stable_call_id(call_id)- || call_id != &result.call_id- || self.settled_call_ids.contains(call_id)- || validate_retained_result(result).is_err()- || validate_tool_result_contract(&self.authorized, result).is_err()- })- || self.settled_call_ids.iter().any(|call_id| {- self.pending.contains_key(call_id) || self.completed.contains_key(call_id)- })- || self.settled_results.keys().any(|call_id| {- self.pending.contains_key(call_id) || self.completed.contains_key(call_id)- })- || self.pending.iter().any(|(call_id, call)| {- !is_stable_call_id(call_id)- || call_id != &call.call_id- || self.completed.contains_key(call_id)- || self.settled_call_ids.contains(call_id)- || self.settled_results.contains_key(call_id)- || validate_pending_tool_call(&self.authorized, call).is_err()- })- || self.completed.iter().any(|(call_id, result)| {- !is_stable_call_id(call_id)- || call_id != &result.call_id- || self.pending.contains_key(call_id)- || validate_retained_result(result).is_err()- || validate_tool_result_contract(&self.authorized, result).is_err()- })- {- return Err(ProviderBridgeError::invalid(- "recovered provider tool call state is invalid",- ));- }+ self.validate_recovered()?;self.retained_result_bytes =retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?;- self.ensure_settled_result_capacity(0).map_err(|_| {- ProviderBridgeError::invalid(- "recovered provider tool results exceed the durable byte limit",- )- })?;Ok(())}@@ -235,6 +299,7 @@ impl ProviderToolBridge {}}self.catalog_digest = Some(tool_set.catalog_digest);+ self.catalog_operations = tool_set.operations.clone();self.authorized = tool_set.operations.into_iter()@@ -244,7 +309,245 @@ impl ProviderToolBridge {}pub fn authorized_tools(&self) -> impl Iterator<Item = &AuthorizedTool> {- self.authorized.values()+ self.catalog_operations.iter()+ }++ pub fn validate_recovered(&self) -> Result<(), ProviderBridgeError> {+ let Some(catalog_digest) = self.catalog_digest.clone() else {+ return if self.authorized.is_empty()+ && self.catalog_operations.is_empty()+ && self.pending.is_empty()+ && self.completed.is_empty()+ && self.settled_results.is_empty()+ && self.settled_call_ids.is_empty()+ && self.settled_call_filter.is_empty()+ && !self.durable_run_receipt_limit_reached+ {+ Ok(())+ } else {+ Err(ProviderBridgeError::invalid(+ "recovered provider tool bridge omitted its catalog identity",+ ))+ };+ };+ let mut expected = ProviderToolBridge::default();+ expected.prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest,+ operations: self.catalog_operations.clone(),+ })?;+ if self.authorized != expected.authorized {+ return Err(ProviderBridgeError::invalid(+ "recovered provider tool bridge changed its authorized catalog",+ ));+ }+ let retained_receipts = self.pending.len().saturating_add(self.completed.len());+ if self.pending.len() > MAX_PENDING_CALLS || retained_receipts > MAX_DURABLE_CALL_RECEIPTS {+ return Err(ProviderBridgeError::invalid(+ "recovered provider tool bridge exceeds its call limit",+ ));+ }+ self.validate_retained_value_bytes()?;+ self.settled_call_filter.validate()?;+ if self.settled_call_ids.len() > MAX_SETTLED_CALL_IDS {+ return Err(ProviderBridgeError::invalid(+ "recovered provider tool bridge exceeds its settled call identity limit",+ ));+ }+ if !self.settled_identity_capacity_allows(0) {+ return Err(ProviderBridgeError::invalid(+ "recovered provider tool bridge cannot settle its active call identities",+ ));+ }+ for call_id in self.settled_call_ids.iter() {+ validate_stable_id(call_id, "settled tool call id")?;+ if self.pending.contains_key(call_id) || self.completed.contains_key(call_id) {+ return Err(ProviderBridgeError::invalid(+ "recovered settled provider tool call identity is inconsistent",+ ));+ }+ }+ let mut pending_validator = expected.clone();+ for (call_id, call) in &self.pending {+ if call_id != &call.call_id+ || self.completed.contains_key(call_id)+ || self.settled_results.contains_key(call_id)+ || self.has_settled_call_id(call_id)+ {+ return Err(ProviderBridgeError::invalid(+ "recovered provider tool call identity is inconsistent",+ ));+ }+ pending_validator.begin_call(+ call.call_id.clone(),+ call.operation_id.clone(),+ call.input.clone(),+ )?;+ }+ for (call_id, completed) in &self.completed {+ if call_id != &completed.call.call_id+ || call_id != &completed.result.call_id+ || completed.call.operation_id != completed.result.operation_id+ || self.settled_results.contains_key(call_id)+ || self.has_settled_call_id(call_id)+ {+ return Err(ProviderBridgeError::invalid(+ "recovered completed tool call identity is inconsistent",+ ));+ }+ let mut completed_validator = expected.clone();+ completed_validator.begin_call(+ completed.call.call_id.clone(),+ completed.call.operation_id.clone(),+ completed.call.input.clone(),+ )?;+ completed_validator.apply_result(completed.result.clone())?;+ }+ for (call_id, settled) in &self.settled_results {+ if call_id != &settled.call.call_id+ || call_id != &settled.result.call_id+ || settled.call.operation_id != settled.result.operation_id+ || !self.settled_call_ids.contains(call_id)+ || self.pending.contains_key(call_id)+ || self.completed.contains_key(call_id)+ {+ return Err(ProviderBridgeError::invalid(+ "recovered settled tool result identity is inconsistent",+ ));+ }+ let mut settled_validator = expected.clone();+ settled_validator.begin_call(+ settled.call.call_id.clone(),+ settled.call.operation_id.clone(),+ settled.call.input.clone(),+ )?;+ settled_validator.apply_result(settled.result.clone())?;+ }+ let retained_bytes =+ retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?;+ ensure_settled_result_capacity(retained_bytes, self.pending.values()).map_err(|_| {+ ProviderBridgeError::invalid(+ "recovered provider tool results exceed the durable byte limit",+ )+ })?;+ Ok(())+ }++ pub fn verify_tool_set(&self, tool_set: &AuthorizedToolSet) -> Result<(), ProviderBridgeError> {+ let mut expected = ProviderToolBridge::default();+ expected.prepare(tool_set.clone())?;+ if self.catalog_digest != expected.catalog_digest+ || self.catalog_operations != expected.catalog_operations+ || self.authorized != expected.authorized+ {+ return Err(ProviderBridgeError::invalid(+ "authorized tool set changed across a durable session",+ ));+ }+ Ok(())+ }++ pub fn has_catalog(&self) -> bool {+ self.catalog_digest.is_some()+ }++ pub fn durable_run_receipt_limit_reached(&self) -> bool {+ self.durable_run_receipt_limit_reached+ }++ fn has_settled_call_id(&self, call_id: &str) -> bool {+ self.settled_call_ids.contains(call_id)+ }++ pub(crate) fn replay_history_blocks_admission(&self) -> bool {+ self.settled_call_ids.len() >= MAX_SETTLED_CALL_IDS || !self.settled_call_filter.is_empty()+ }++ pub(crate) fn has_active_receipts(&self) -> bool {+ !self.pending.is_empty() || !self.completed.is_empty()+ }++ /// Forget an exact replay epoch only after its provider process has been+ /// reaped and an idle replacement generation has been established. The+ /// process boundary is what prevents an old provider request from being+ /// admitted after its tombstone is released; callers must not use this as+ /// ordinary capacity eviction.+ pub(crate) fn rollover_replay_epoch_after_provider_restart(+ &mut self,+ ) -> Result<(), ProviderBridgeError> {+ if !self.replay_history_blocks_admission() {+ return Err(ProviderBridgeError::invalid(+ "cannot rotate provider tool replay authority before its exact capacity boundary",+ ));+ }+ if !self.pending.is_empty()+ || !self.completed.is_empty()+ || !self.settled_results.is_empty()+ || self.retained_result_bytes != 0+ {+ return Err(ProviderBridgeError::invalid(+ "cannot rotate provider tool replay authority while receipts are retained",+ ));+ }+ self.settled_call_ids.clear();+ self.settled_call_filter = DurableReplayFilter::default();+ self.durable_run_receipt_limit_reached = false;+ Ok(())+ }++ fn settled_identity_capacity_allows(&self, additional_calls: usize) -> bool {+ self.settled_call_ids+ .len()+ .checked_add(self.total_call_receipts())+ .and_then(|total| total.checked_add(additional_calls))+ .is_some_and(|total| total <= MAX_SETTLED_CALL_IDS)+ }++ pub fn prepare_turn(&mut self) -> Result<(), ProviderBridgeError> {+ if !self.pending.is_empty() || !self.completed.is_empty() {+ return Err(ProviderBridgeError::invalid(+ "cannot prepare the next provider turn while tool calls are active",+ ));+ }+ // The authoritative result bodies are needed only while the provider+ // can replay the just-settled turn. Release that bulky data at the+ // verified turn boundary, but retain every call-ID tombstone and any+ // recovered legacy filter until the owning provider process is reaped.+ self.settled_results.clear();+ self.retained_result_bytes = 0;+ self.durable_run_receipt_limit_reached = self.replay_history_blocks_admission();+ Ok(())+ }++ #[cfg(test)]+ pub(crate) fn retained_result_bytes_for_test(&self) -> usize {+ self.retained_result_bytes+ }++ pub fn replay_result(+ &self,+ call_id: &str,+ operation_id: &str,+ input: &Value,+ ) -> Result<Option<ToolResult>, ProviderBridgeError> {+ if let Some(completed) = self+ .completed+ .get(call_id)+ .or_else(|| self.settled_results.get(call_id))+ {+ if completed.call.operation_id != operation_id || &completed.call.input != input {+ return Err(ProviderBridgeError::invalid(+ "provider replayed a completed tool call with different input",+ ));+ }+ return Ok(Some(completed.result.clone()));+ }+ Ok(None)+ }++ pub fn has_completed_call(&self, call_id: &str) -> bool {+ self.completed.contains_key(call_id) || self.has_settled_call_id(call_id)}pub fn begin_call(@@ -253,6 +556,24 @@ impl ProviderToolBridge {operation_id: String,input: Value,) -> Result<PendingToolCall, ProviderBridgeError> {+ validate_stable_id(&call_id, "tool call id")?;+ validate_operation_id(&operation_id)?;+ let authorized = self.authorized.get(&operation_id).ok_or_else(|| {+ ProviderBridgeError::invalid(format!(+ "provider requested unauthorized tool {operation_id}"+ ))+ })?;+ let validator = jsonschema::validator_for(&authorized.input_schema).map_err(|_| {+ ProviderBridgeError::invalid(format!(+ "tool {operation_id} has an invalid durable input JSON Schema"+ ))+ })?;+ if !validator.is_valid(&input) {+ return Err(ProviderBridgeError::invalid(format!(+ "provider arguments for {operation_id} failed JSON Schema validation"+ )));+ }+ bounded_json(&input, MAX_TOOL_VALUE_BYTES, "provider tool input")?;let call = PendingToolCall {call_id: call_id.clone(),operation_id,@@ -270,55 +591,93 @@ impl ProviderToolBridge {}if self.completed.contains_key(&call_id)|| self.settled_results.contains_key(&call_id)- || self.settled_call_ids.contains(&call_id)+ || self.has_settled_call_id(&call_id){return Err(ProviderBridgeError::invalid("provider reused a completed tool call id",));}- if self.pending.len().saturating_add(self.completed.len()) >= MAX_RETAINED_CALLS {+ // A durable receipt ledger may be exactly full or legacy state may+ // contain only a probabilistic summary of evicted identities. Neither+ // state can admit more work safely in this provider-process epoch. The+ // backend reaps an idle process before establishing a fresh epoch.+ if self.replay_history_blocks_admission() {+ self.durable_run_receipt_limit_reached = true;+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }+ if self.durable_run_receipt_limit_reached {+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }+ // Pending and completed calls become exact settled identities when the+ // turn terminates. Reserve that identity capacity before dispatch so+ // every admitted call can be settled without stranding active state.+ if !self.settled_identity_capacity_allows(1) {+ self.durable_run_receipt_limit_reached = true;+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }+ if self.pending.len() >= MAX_PENDING_CALLS {return Err(ProviderBridgeError::invalid(- "provider tool receipt limit reached for the active turn",+ "concurrent provider tool call limit reached",));}- // Reserve durable identity space before accepting work. Settlement can- // then never fail merely because earlier turns filled the ledger and- // leave completed receipts stranded in the active-turn budget.- if self- .settled_call_ids- .len()- .saturating_add(self.settled_results.len())- .saturating_add(self.pending.len())- .saturating_add(self.completed.len())- >= MAX_SETTLED_CALL_IDS+ if self.total_call_receipts() >= MAX_DURABLE_CALL_RECEIPTS {+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }+ let input_bytes = json_size(&call.input, "provider tool input")?;+ let pending_bytes = self.pending_value_bytes()?;+ if pending_bytes+ .checked_add(input_bytes)+ .is_none_or(|total| total > MAX_ACCEPTED_TOOL_VALUE_BYTES){return Err(ProviderBridgeError::invalid(- "durable provider tool call identity limit reached",+ "retained provider tool values exceed the 4 MiB acceptance limit",));}- // Reserve the worst-case encoded result before accepting the call.- // This makes apply_result and settlement infallible with respect to- // durable result capacity: accepted work can always retain its exact- // authoritative replay value.- self.ensure_settled_result_capacity(1)?;+ let result_reserve = self+ .pending+ .len()+ .saturating_add(1)+ .checked_mul(MAX_TOOL_VALUE_BYTES)+ .ok_or_else(|| ProviderBridgeError::invalid("provider tool result reserve overflow"))?;+ let retained_value_bytes = self.retained_value_bytes()?;+ if retained_value_bytes+ .checked_add(input_bytes)+ .and_then(|total| total.checked_add(MAX_TOOL_VALUE_BYTES))+ .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES)+ {+ self.durable_run_receipt_limit_reached = true;+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }+ if retained_value_bytes+ .checked_add(input_bytes)+ .and_then(|total| total.checked_add(result_reserve))+ .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES)+ {+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }+ // Reserve the complete encoded receipt, including this exact input and+ // a maximum-sized result, before accepting work. An admitted call can+ // therefore always retain its authoritative result at settlement.+ if ensure_settled_result_capacity(self.retained_result_bytes, std::iter::once(&call))+ .is_err()+ {+ self.durable_run_receipt_limit_reached = true;+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }+ if self.ensure_settled_result_capacity(Some(&call)).is_err() {+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }self.pending.insert(call_id, call.clone());Ok(call)}pub fn apply_result(&mut self, result: ToolResult) -> Result<Value, ProviderBridgeError> {- if result.call_id.is_empty()- || result.call_id.len() > 160- || result.call_id.chars().any(char::is_control)- {- return Err(ProviderBridgeError::invalid(- "tool result call id is invalid",- ));- }+ validate_stable_id(&result.call_id, "tool result call id")?;validate_operation_id(&result.operation_id)?;bounded_json(&result.result, MAX_TOOL_VALUE_BYTES, "provider tool result")?;if let Some(existing) = self.completed.get(&result.call_id) {- return if existing == &result {- Ok(existing.result.clone())+ return if existing.result == result {+ Ok(existing.result.result.clone())} else {Err(ProviderBridgeError::invalid("conflicting duplicate tool result",@@ -326,20 +685,20 @@ impl ProviderToolBridge {};}if let Some(existing) = self.settled_results.get(&result.call_id) {- return if existing == &result {- Ok(existing.result.clone())+ return if existing.result == result {+ Ok(existing.result.result.clone())} else {Err(ProviderBridgeError::invalid("conflicting duplicate settled tool result",))};}- if self.settled_call_ids.contains(&result.call_id) {+ if self.has_settled_call_id(&result.call_id) {return Err(ProviderBridgeError::invalid("legacy settled tool result cannot be replayed",));}- let pending = self.pending.get(&result.call_id).ok_or_else(|| {+ let pending = self.pending.get(&result.call_id).cloned().ok_or_else(|| {ProviderBridgeError::invalid("tool result does not match a pending provider call")})?;if pending.operation_id != result.operation_id {@@ -347,89 +706,218 @@ impl ProviderToolBridge {"tool result operation does not match its call",));}- validate_tool_result_contract(&self.authorized, &result)?;- let result_bytes = retained_result_entry_bytes(&result.call_id, &result)?;- let next_retained_bytes = self- .retained_result_bytes+ let authorized = self.authorized.get(&result.operation_id).ok_or_else(|| {+ ProviderBridgeError::invalid("tool result operation is no longer authorized")+ })?;+ let validator = jsonschema::validator_for(&authorized.response_schema).map_err(|_| {+ ProviderBridgeError::invalid(format!(+ "tool {} has an invalid durable response JSON Schema",+ result.operation_id+ ))+ })?;+ let response = semantic_response_value(&result)?;+ if !result.is_error {+ // Paperclip semantic dispatchers return an authoritative envelope;+ // provider contracts describe the operation-specific value inside+ // `result`. Direct values remain valid for compatibility with v1+ // peers that do not wrap their semantic result.+ if let Some(response) = response {+ if !validator.is_valid(response) {+ return Err(ProviderBridgeError::invalid(format!(+ "tool result for {} failed JSON Schema validation",+ result.operation_id+ )));+ }+ }+ }+ let result_bytes = json_size(&result.result, "provider tool result")?;+ let retained_bytes = self.retained_value_bytes()?;+ if retained_bytes.checked_add(result_bytes)- .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?;- let remaining_pending_reserve = self- .pending- .len()- .saturating_sub(1)- .checked_mul(MAX_SETTLED_RESULT_ENTRY_BYTES)- .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?;- if next_retained_bytes- .checked_add(remaining_pending_reserve)- .is_none_or(|bytes| bytes > MAX_SETTLED_RESULT_BYTES)+ .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES){return Err(ProviderBridgeError::invalid(- "durable provider tool result byte limit reached",+ "retained provider tool values exceed the 8 MiB aggregate limit",));}+ let completed = CompletedToolCall {+ call: pending,+ result: result.clone(),+ };+ let receipt_bytes = retained_result_entry_bytes(&result.call_id, &completed)?;+ let next_retained_bytes = self+ .retained_result_bytes+ .checked_add(receipt_bytes)+ .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?;+ ensure_settled_result_capacity(+ next_retained_bytes,+ self.pending+ .iter()+ .filter(|(call_id, _)| *call_id != &result.call_id)+ .map(|(_, call)| call),+ )?;self.pending.remove(&result.call_id);self.retained_result_bytes = next_retained_bytes;- self.completed- .insert(result.call_id.clone(), result.clone());+ self.completed.insert(result.call_id.clone(), completed);Ok(result.result)}- pub fn settle_turn(&mut self) -> Result<(), ProviderBridgeError> {- if !self.pending.is_empty() {- return Err(ProviderBridgeError::invalid(- "cannot settle provider tool receipts while calls are pending",- ));- }- self.retained_result_bytes =- retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?;- self.ensure_settled_result_capacity(0)?;- // The identity capacity was reserved in `begin_call`, so moving the- // authoritative receipts cannot fail a valid admitted turn. The check- // above rejects only recovered state that bypassed attach validation.- self.settled_results.append(&mut self.completed);- Ok(())- }-pub fn pending_calls(&self) -> impl Iterator<Item = &PendingToolCall> {self.pending.values()}- fn ensure_settled_result_capacity(- &self,- additional_pending: usize,- ) -> Result<(), ProviderBridgeError> {- let pending_count = self+ pub fn cancel_pending_calls(+ &mut self,+ code: &str,+ ) -> Result<Vec<ToolResult>, ProviderBridgeError> {+ let mut next = self.clone();+ let results = next.cancel_pending_calls_internal(code)?;+ *self = next;+ Ok(results)+ }++ pub fn settle_turn(&mut self, code: &str) -> Result<Vec<ToolResult>, ProviderBridgeError> {+ validate_stable_id(code, "tool cancellation code")?;+ let results = self.pending- .len()- .checked_add(additional_pending)- .ok_or_else(|| {- ProviderBridgeError::invalid("durable provider result count overflow")- })?;- let pending_reserve = pending_count- .checked_mul(MAX_SETTLED_RESULT_ENTRY_BYTES)- .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?;+ .values()+ .map(|call| cancelled_tool_result(call, code))+ .collect::<Vec<_>>();+ let mut settled_entries = self.completed.clone();+ for (call_id, call) in &self.pending {+ settled_entries.insert(+ call_id.clone(),+ CompletedToolCall {+ call: call.clone(),+ result: cancelled_tool_result(call, code),+ },+ );+ }+ let next_retained_bytes =+ retained_result_bytes(self.settled_results.iter().chain(settled_entries.iter()))?;+ ensure_settled_result_capacity(next_retained_bytes, std::iter::empty())?;++ // Byte capacity was reserved at admission. Keep every identity exact+ // for the lifetime of this provider-process epoch; prepare_turn+ // releases only the bulky result bodies.+ let new_identity_count = settled_entries+ .keys()+ .filter(|call_id| !self.settled_call_ids.contains(call_id))+ .count();if self- .retained_result_bytes- .checked_add(pending_reserve)- .is_none_or(|bytes| bytes > MAX_SETTLED_RESULT_BYTES)+ .settled_call_ids+ .len()+ .checked_add(new_identity_count)+ .is_none_or(|total| total > MAX_SETTLED_CALL_IDS){+ return Err(ProviderBridgeError::active_turn_receipt_limit());+ }+ let evicted = self+ .settled_call_ids+ .extend_recent(settled_entries.keys().cloned(), MAX_SETTLED_CALL_IDS);+ debug_assert!(evicted.is_empty());+ self.pending.clear();+ self.completed.clear();+ self.settled_results.append(&mut settled_entries);+ self.retained_result_bytes = next_retained_bytes;+ Ok(results)+ }++ fn cancel_pending_calls_internal(+ &mut self,+ code: &str,+ ) -> Result<Vec<ToolResult>, ProviderBridgeError> {+ validate_stable_id(code, "tool cancellation code")?;+ let pending = self.pending.values().cloned().collect::<Vec<_>>();+ let mut results = Vec::with_capacity(pending.len());+ for call in pending {+ let result = cancelled_tool_result(&call, code);+ self.apply_result(result.clone())?;+ results.push(result);+ }+ Ok(results)+ }++ fn retained_value_bytes(&self) -> Result<usize, ProviderBridgeError> {+ let pending = self+ .pending+ .values()+ .map(|call| json_size(&call.input, "retained provider tool input"));+ let completed = self.completed.values().flat_map(|entry| {+ [+ json_size(&entry.call.input, "retained provider tool input"),+ json_size(&entry.result.result, "retained provider tool result"),+ ]+ });+ let settled = self.settled_results.values().flat_map(|entry| {+ [+ json_size(&entry.call.input, "retained provider tool input"),+ json_size(&entry.result.result, "retained provider tool result"),+ ]+ });+ pending+ .chain(completed)+ .chain(settled)+ .try_fold(0usize, |total, bytes| {+ total.checked_add(bytes?).ok_or_else(|| {+ ProviderBridgeError::invalid("retained provider tool values overflow")+ })+ })+ }++ fn pending_value_bytes(&self) -> Result<usize, ProviderBridgeError> {+ self.pending.values().try_fold(0usize, |total, call| {+ total+ .checked_add(json_size(&call.input, "retained provider tool input")?)+ .ok_or_else(|| {+ ProviderBridgeError::invalid("retained provider tool values overflow")+ })+ })+ }++ fn total_call_receipts(&self) -> usize {+ self.pending.len().saturating_add(self.completed.len())+ }++ fn validate_retained_value_bytes(&self) -> Result<(), ProviderBridgeError> {+ if self.retained_value_bytes()? > MAX_RETAINED_TOOL_VALUE_BYTES {return Err(ProviderBridgeError::invalid(- "durable provider tool result byte limit reached",+ "retained provider tool values exceed the 8 MiB aggregate limit",));}Ok(())}++ fn ensure_settled_result_capacity(+ &self,+ additional_pending: Option<&PendingToolCall>,+ ) -> Result<(), ProviderBridgeError> {+ ensure_settled_result_capacity(+ self.retained_result_bytes,+ self.pending.values().chain(additional_pending.into_iter()),+ )+ }}-fn validate_retained_result(result: &ToolResult) -> Result<(), ProviderBridgeError> {- if !is_stable_call_id(&result.call_id) {+fn validate_retained_result(entry: &CompletedToolCall) -> Result<(), ProviderBridgeError> {+ validate_stable_id(&entry.call.call_id, "retained tool call id")?;+ validate_stable_id(&entry.result.call_id, "retained tool result call id")?;+ validate_operation_id(&entry.call.operation_id)?;+ validate_operation_id(&entry.result.operation_id)?;+ if entry.call.call_id != entry.result.call_id+ || entry.call.operation_id != entry.result.operation_id+ {return Err(ProviderBridgeError::invalid(- "retained tool result call id is invalid",+ "retained provider tool receipt identity is inconsistent",));}- validate_operation_id(&result.operation_id)?;bounded_json(- &result.result,+ &entry.call.input,+ MAX_TOOL_VALUE_BYTES,+ "retained provider tool input",+ )?;+ bounded_json(+ &entry.result.result,MAX_TOOL_VALUE_BYTES,"retained provider tool result",)@@ -439,9 +927,7 @@ fn validate_pending_tool_call(authorized: &BTreeMap<String, AuthorizedTool>,call: &PendingToolCall,) -> Result<(), ProviderBridgeError> {- if !is_stable_call_id(&call.call_id) {- return Err(ProviderBridgeError::invalid("tool call id is invalid"));- }+ validate_stable_id(&call.call_id, "tool call id")?;validate_operation_id(&call.operation_id)?;let tool = authorized.get(&call.operation_id).ok_or_else(|| {ProviderBridgeError::invalid(format!(@@ -571,16 +1057,16 @@ fn validate_authorized_tool_set(tool_set: &AuthorizedToolSet) -> Result<(), Provfn retained_result_entry_bytes(call_id: &str,- result: &ToolResult,+ result: &CompletedToolCall,) -> Result<usize, ProviderBridgeError> {// A two-item tuple has the same delimiter cost as a one-entry JSON map.// Summing tuples therefore equals one entry exactly and conservatively// overcounts a multi-entry map by one byte per additional receipt.- encoded_json_bytes(&(call_id, result), "retained provider tool result")+ json_size(&(call_id, result), "retained provider tool result")}fn retained_result_bytes<'a>(- results: impl IntoIterator<Item = (&'a String, &'a ToolResult)>,+ results: impl IntoIterator<Item = (&'a String, &'a CompletedToolCall)>,) -> Result<usize, ProviderBridgeError> {results.into_iter()@@ -593,16 +1079,53 @@ fn retained_result_bytes<'a>(})}+fn pending_result_reserve_bytes(call: &PendingToolCall) -> Result<usize, ProviderBridgeError> {+ let placeholder = CompletedToolCall {+ call: call.clone(),+ result: ToolResult {+ call_id: call.call_id.clone(),+ operation_id: call.operation_id.clone(),+ result: Value::Null,+ is_error: false,+ },+ };+ retained_result_entry_bytes(&call.call_id, &placeholder)?+ .checked_sub(json_size(&Value::Null, "provider tool result reserve")?)+ .and_then(|bytes| bytes.checked_add(MAX_TOOL_VALUE_BYTES))+ .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))+}++fn ensure_settled_result_capacity<'a>(+ retained_bytes: usize,+ pending: impl IntoIterator<Item = &'a PendingToolCall>,+) -> Result<(), ProviderBridgeError> {+ let reserved_bytes = pending+ .into_iter()+ .try_fold(retained_bytes, |total, call| {+ total+ .checked_add(pending_result_reserve_bytes(call)?)+ .ok_or_else(|| {+ ProviderBridgeError::invalid("durable provider result size overflow")+ })+ })?;+ if reserved_bytes > MAX_SETTLED_RESULT_BYTES {+ return Err(ProviderBridgeError::invalid(+ "durable provider tool result byte limit reached",+ ));+ }+ Ok(())+}+fn deserialize_retained_results<'de, D>(deserializer: D,-) -> Result<BTreeMap<String, ToolResult>, D::Error>+) -> Result<BTreeMap<String, CompletedToolCall>, D::Error>whereD: Deserializer<'de>,{struct RetainedResultsVisitor;impl<'de> Visitor<'de> for RetainedResultsVisitor {- type Value = BTreeMap<String, ToolResult>;+ type Value = BTreeMap<String, CompletedToolCall>;fn expecting(&self, formatter: &mut Formatter<'_>) -> fmt::Result {formatter.write_str("a bounded map of retained provider tool results")@@ -622,14 +1145,14 @@ where}let mut results = BTreeMap::new();let mut retained_bytes = 0usize;- while let Some((call_id, result)) = map.next_entry::<String, ToolResult>()? {+ while let Some((call_id, result)) = map.next_entry::<String, CompletedToolCall>()? {if results.len() >= MAX_SETTLED_CALL_IDS {return Err(de::Error::custom("retained provider tool result count exceeds the durable limit",));}validate_retained_result(&result).map_err(de::Error::custom)?;- if call_id != result.call_id {+ if call_id != result.call.call_id || call_id != result.result.call_id {return Err(de::Error::custom("retained provider tool result identity is inconsistent",));@@ -658,8 +1181,19 @@ wheredeserializer.deserialize_map(RetainedResultsVisitor)}-fn is_stable_call_id(value: &str) -> bool {- !value.is_empty() && value.len() <= 160 && !value.chars().any(char::is_control)+fn cancelled_tool_result(call: &PendingToolCall, code: &str) -> ToolResult {+ ToolResult {+ call_id: call.call_id.clone(),+ operation_id: call.operation_id.clone(),+ result: serde_json::json!({+ "error": {+ "code": code,+ "message": "The provider turn stopped before this semantic tool completed",+ "retryable": false,+ },+ }),+ is_error: true,+ }}pub fn authorized_tool_catalog_digest(@@ -678,11 +1212,16 @@ pub fn authorized_tool_catalog_digest(Ok(format!("sha256:{digest:x}"))}+pub fn semantic_value_digest(value: &Value) -> String {+ let digest = Sha256::digest(canonical_json(value).as_bytes());+ format!("sha256:{digest:x}")+}+fn canonical_json(value: &Value) -> String {match value {Value::Null => "null".to_owned(),Value::Bool(value) => value.to_string(),- Value::Number(value) => value.to_string(),+ Value::Number(value) => canonical_json_number(value),Value::String(value) => {serde_json::to_string(value).expect("serializing an in-memory JSON string cannot fail")}@@ -696,7 +1235,7 @@ fn canonical_json(value: &Value) -> String {),Value::Object(object) => {let mut entries = object.iter().collect::<Vec<_>>();- entries.sort_by_key(|(key, _)| *key);+ entries.sort_by(|(left, _), (right, _)| left.encode_utf16().cmp(right.encode_utf16()));format!("{{{}}}",entries@@ -714,6 +1253,82 @@ fn canonical_json(value: &Value) -> String {}}+fn canonical_json_number(value: &serde_json::Number) -> String {+ if value.is_i64() || value.is_u64() {+ return value.to_string();+ }+ let Some(float) = value.as_f64() else {+ return value.to_string();+ };+ if float == 0.0 {+ return "0".to_owned();+ }++ // serde_json preserves valid lexical spellings such as `1.0`, whereas+ // JSON.stringify canonicalizes JavaScript numbers. Normalize the shortest+ // serde representation to the ECMAScript decimal/exponent thresholds so+ // the server and runner hash the same schema value.+ let encoded = value.to_string().to_ascii_lowercase();+ let (negative, unsigned) = encoded+ .strip_prefix('-')+ .map_or((false, encoded.as_str()), |rest| (true, rest));+ let (coefficient, explicit_exponent) = unsigned+ .split_once('e')+ .map_or((unsigned, 0_i32), |(coefficient, exponent)| {+ (coefficient, exponent.parse::<i32>().unwrap_or(0))+ });+ let fraction_digits = coefficient+ .split_once('.')+ .map_or(0_i32, |(_, fraction)| fraction.len() as i32);+ let mut digits = coefficient+ .bytes()+ .filter(|byte| *byte != b'.')+ .map(char::from)+ .collect::<String>();+ let mut decimal_position = digits.len() as i32 + explicit_exponent - fraction_digits;++ let leading_zeros = digits.bytes().take_while(|byte| *byte == b'0').count();+ digits.drain(..leading_zeros);+ decimal_position -= leading_zeros as i32;+ while digits.ends_with('0') {+ digits.pop();+ }+ if digits.is_empty() {+ return "0".to_owned();+ }++ let body = if (1e-6..1e21).contains(&float.abs()) {+ if decimal_position <= 0 {+ format!("0.{}{}", "0".repeat((-decimal_position) as usize), digits)+ } else if decimal_position >= digits.len() as i32 {+ format!(+ "{}{}",+ digits,+ "0".repeat((decimal_position - digits.len() as i32) as usize)+ )+ } else {+ let split = decimal_position as usize;+ format!("{}.{}", &digits[..split], &digits[split..])+ }+ } else {+ let exponent = decimal_position - 1;+ let coefficient = if digits.len() == 1 {+ digits+ } else {+ format!("{}.{}", &digits[..1], &digits[1..])+ };+ format!(+ "{coefficient}e{}{exponent}",+ if exponent >= 0 { "+" } else { "" }+ )+ };+ if negative {+ format!("-{body}")+ } else {+ body+ }+}+fn semantic_response_value(result: &ToolResult) -> Result<Option<&Value>, ProviderBridgeError> {let Some(envelope) = result.result.as_object() else {return Ok(Some(&result.result));@@ -732,9 +1347,11 @@ fn semantic_response_value(result: &ToolResult) -> Result<Option<&Value>, Provid));}if ok {- envelope.get("result").map(Some).ok_or_else(|| {- ProviderBridgeError::invalid("successful semantic result omitted result")- })+ envelope+ .get("result")+ .or_else(|| envelope.get("value"))+ .map(Some)+ .ok_or_else(|| ProviderBridgeError::invalid("successful semantic result omitted value"))} else if envelope.get("denial").is_some() || envelope.get("error").is_some() {Ok(None)} else {@@ -745,6 +1362,10 @@ fn semantic_response_value(result: &ToolResult) -> Result<Option<&Value>, Provid}fn validate_operation_id(value: &str) -> Result<(), ProviderBridgeError> {+ validate_stable_id(value, "tool operation id")+}++fn validate_stable_id(value: &str, label: &str) -> Result<(), ProviderBridgeError> {let mut chars = value.chars();let first = chars.next()@@ -755,7 +1376,7 @@ fn validate_operation_id(value: &str) -> Result<(), ProviderBridgeError> {if first && rest && value.len() <= 160 {Ok(())} else {- Err(ProviderBridgeError::invalid("tool operation id is invalid"))+ Err(ProviderBridgeError::invalid(format!("{label} is invalid")))}}@@ -774,7 +1395,7 @@ fn bounded_json(max_bytes: usize,label: &str,) -> Result<(), ProviderBridgeError> {- let bytes = encoded_json_bytes(value, label)?;+ let bytes = json_size(value, label)?;if bytes > max_bytes {return Err(ProviderBridgeError::invalid(format!("{label} exceeds the {max_bytes} byte limit"@@ -783,7 +1404,7 @@ fn bounded_json(Ok(())}-fn encoded_json_bytes(value: &impl Serialize, label: &str) -> Result<usize, ProviderBridgeError> {+fn json_size(value: &impl Serialize, label: &str) -> Result<usize, ProviderBridgeError> {serde_json::to_vec(value).map(|bytes| bytes.len()).map_err(|_| ProviderBridgeError::invalid(format!("{label} is not serializable")))@@ -792,24 +1413,524 @@ fn encoded_json_bytes(value: &impl Serialize, label: &str) -> Result<usize, Prov#[cfg(test)]mod tests {use super::*;+ use serde_json::json;#[test]fn worst_case_result_identity_overhead_fits_the_admission_reserve() {- let call_id = "\\".repeat(160);- let result = ToolResult {+ let call_id = "x".repeat(160);+ let call = PendingToolCall {call_id: call_id.clone(),operation_id: "x".repeat(160),- result: Value::String("x".repeat(MAX_TOOL_VALUE_BYTES - 2)),- is_error: false,+ input: Value::String("x".repeat(MAX_TOOL_VALUE_BYTES - 2)),+ };+ let completed = CompletedToolCall {+ call: call.clone(),+ result: ToolResult {+ call_id: call_id.clone(),+ operation_id: call.operation_id.clone(),+ result: Value::String("x".repeat(MAX_TOOL_VALUE_BYTES - 2)),+ is_error: false,+ },};assert_eq!(- encoded_json_bytes(&result.result, "test result").unwrap(),+ json_size(&completed.result.result, "test result").unwrap(),MAX_TOOL_VALUE_BYTES);assert!(- retained_result_entry_bytes(&call_id, &result).unwrap()- <= MAX_SETTLED_RESULT_ENTRY_BYTES+ retained_result_entry_bytes(&call_id, &completed).unwrap()+ <= pending_result_reserve_bytes(&call).unwrap()+ );+ }++ #[test]+ fn exact_completed_results_are_bounded_without_changing_replay() {+ let operation = AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation.clone()],+ })+ .unwrap();+ bridge+ .begin_call(+ "call-0".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect("begin the first exact call");+ bridge+ .apply_result(ToolResult {+ call_id: "call-0".to_owned(),+ operation_id: "get_task_context".to_owned(),+ result: json!({"ok": true}),+ is_error: false,+ })+ .expect("complete the first exact call");+ assert_eq!(+ bridge+ .replay_result("call-0", "get_task_context", &json!({}))+ .unwrap()+ .unwrap()+ .result,+ json!({"ok": true})+ );+ for index in 1..MAX_DURABLE_CALL_RECEIPTS {+ let call_id = format!("call-{index}");+ bridge.completed.insert(+ call_id.clone(),+ CompletedToolCall {+ call: PendingToolCall {+ call_id: call_id.clone(),+ operation_id: "get_task_context".to_owned(),+ input: json!({}),+ },+ result: ToolResult {+ call_id,+ operation_id: "get_task_context".to_owned(),+ result: json!({"ok": true}),+ is_error: false,+ },+ },+ );+ }+ let error = bridge+ .begin_call(+ "call-new".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("the exact receipt limit must stop active-turn growth");+ assert!(error.is_active_turn_receipt_limit());++ assert_eq!(+ bridge+ .replay_result("call-0", "get_task_context", &json!({}))+ .unwrap()+ .unwrap()+ .result,+ json!({"ok": true})+ );+ assert!(bridge+ .apply_result(ToolResult {+ call_id: "call-0".to_owned(),+ operation_id: "get_task_context".to_owned(),+ result: json!({"ok": false}),+ is_error: false,+ })+ .is_err());+ bridge.validate_retained_value_bytes().unwrap();+ assert!(serde_json::to_vec_pretty(&bridge).unwrap().len() < 4 * 1024 * 1024);+ let recovered: ProviderToolBridge =+ serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();+ recovered.validate_recovered().unwrap();+ assert_eq!(+ recovered+ .replay_result("call-0", "get_task_context", &json!({}))+ .unwrap()+ .unwrap()+ .result,+ json!({"ok": true})+ );++ bridge.settle_turn("provider_turn_terminated").unwrap();+ assert!(bridge+ .begin_call(+ "call-0".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .is_err());+ bridge.prepare_turn().unwrap();+ assert!(bridge+ .replay_result("call-0", "get_task_context", &json!({}))+ .unwrap()+ .is_none());+ assert!(bridge+ .begin_call(+ "call-0".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .is_err());+ bridge+ .begin_call(+ "next-turn-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect("a new identity remains admissible after bulky results are released");++ let recovered: ProviderToolBridge =+ serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();+ recovered.validate_recovered().unwrap();+ assert!(recovered.has_completed_call("call-0"));+ }++ #[test]+ fn admission_reserves_exact_identity_capacity_for_settlement() {+ let operation = AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation.clone()],+ })+ .unwrap();+ for index in 0..MAX_SETTLED_CALL_IDS - 1 {+ let call_id = format!("settled-call-{index}");+ bridge.settled_call_ids.order.push_back(call_id.clone());+ bridge.settled_call_ids.members.insert(call_id);+ }++ bridge+ .begin_call(+ "final-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect("the final exact identity slot remains usable");+ let mut pending_at_capacity = bridge.clone();+ let error = pending_at_capacity+ .begin_call(+ "rejected-while-pending".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("a pending call must reserve its eventual settled identity");+ assert!(error.is_active_turn_receipt_limit());+ assert!(pending_at_capacity.durable_run_receipt_limit_reached);+ assert_eq!(pending_at_capacity.pending.len(), 1);+ assert!(pending_at_capacity.pending.contains_key("final-call"));+ assert!(!pending_at_capacity+ .pending+ .contains_key("rejected-while-pending"));+ pending_at_capacity+ .begin_call(+ "final-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect("an idempotent retry remains valid at capacity");+ let cancelled = pending_at_capacity+ .settle_turn("semantic_tool_turn_receipt_limit")+ .unwrap();+ assert_eq!(cancelled.len(), 1);+ assert_eq!(+ pending_at_capacity.settled_call_ids.len(),+ MAX_SETTLED_CALL_IDS);++ bridge+ .apply_result(ToolResult {+ call_id: "final-call".to_owned(),+ operation_id: "get_task_context".to_owned(),+ result: json!({"ok": true}),+ is_error: false,+ })+ .expect("complete the call admitted into the final identity slot");++ let mut overcommitted = bridge.clone();+ overcommitted.pending.insert(+ "unsettleable-call".to_owned(),+ PendingToolCall {+ call_id: "unsettleable-call".to_owned(),+ operation_id: "get_task_context".to_owned(),+ input: json!({}),+ },+ );+ assert_eq!(+ overcommitted.validate_recovered().unwrap_err().to_string(),+ "recovered provider tool bridge cannot settle its active call identities"+ );+ let overcommitted_snapshot = overcommitted.clone();+ let error = overcommitted+ .settle_turn("semantic_tool_turn_receipt_limit")+ .expect_err("defensive settlement must reject an overcommitted recovered state");+ assert!(error.is_active_turn_receipt_limit());+ assert_eq!(overcommitted, overcommitted_snapshot);++ let error = bridge+ .begin_call(+ "rejected-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("admission must reserve space for every active identity");+ assert!(error.is_active_turn_receipt_limit());+ assert!(bridge.durable_run_receipt_limit_reached);+ assert!(!bridge.pending.contains_key("rejected-call"));+ assert!(bridge.completed.contains_key("final-call"));++ assert!(bridge+ .settle_turn("semantic_tool_turn_receipt_limit")+ .unwrap()+ .is_empty());+ assert!(bridge.pending.is_empty());+ assert!(bridge.completed.is_empty());+ assert_eq!(bridge.settled_call_ids.len(), MAX_SETTLED_CALL_IDS);+ assert!(bridge.settled_results.contains_key("final-call"));++ bridge.prepare_turn().unwrap();+ assert_eq!(bridge.settled_call_ids.len(), MAX_SETTLED_CALL_IDS);+ assert!(bridge.settled_results.is_empty());+ assert!(bridge.durable_run_receipt_limit_reached);+ let error = bridge+ .begin_call(+ "next-turn-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("a turn boundary must not reopen a saturated durable run");+ assert!(error.is_active_turn_receipt_limit());++ let encoded = serde_json::to_string(&bridge).unwrap();+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();+ recovered.attach_existing_run().unwrap();+ assert_eq!(recovered.settled_call_ids.len(), MAX_SETTLED_CALL_IDS);+ assert!(recovered.durable_run_receipt_limit_reached);+ let error = recovered+ .begin_call(+ "recovered-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("recovery must preserve saturated replay authority");+ assert!(error.is_active_turn_receipt_limit());++ recovered+ .attach_run(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation.clone()],+ })+ .unwrap();+ assert!(recovered.settled_call_ids.is_empty());+ assert!(!recovered.durable_run_receipt_limit_reached);+ recovered+ .begin_call(+ "settled-call-0".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect("a new durable run receives fresh replay authority");+ }++ #[test]+ fn legacy_replay_filter_quarantines_the_durable_run() {+ let operation = AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation.clone()],+ })+ .unwrap();+ bridge.settled_call_filter.words = vec![0; REPLAY_FILTER_WORDS];+ bridge.settled_call_filter.words[0] = 1;++ let error = bridge+ .begin_call(+ "legacy-evicted-or-fresh".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("a legacy probabilistic epoch cannot dispatch more work");+ assert!(error.is_active_turn_receipt_limit());+ assert!(bridge.pending.is_empty());+ assert!(bridge.durable_run_receipt_limit_reached);++ bridge.prepare_turn().unwrap();+ assert!(!bridge.settled_call_filter.is_empty());+ assert!(bridge.durable_run_receipt_limit_reached);+ let error = bridge+ .begin_call(+ "legacy-evicted-or-fresh".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("a turn boundary cannot make legacy ambiguity safe");+ assert!(error.is_active_turn_receipt_limit());++ let encoded = serde_json::to_string(&bridge).unwrap();+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();+ recovered.attach_existing_run().unwrap();+ assert!(!recovered.settled_call_filter.is_empty());+ assert!(recovered.durable_run_receipt_limit_reached);++ recovered+ .attach_run(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation],+ })+ .unwrap();+ assert!(recovered.settled_call_filter.is_empty());+ assert!(!recovered.durable_run_receipt_limit_reached);+ recovered+ .begin_call(+ "legacy-evicted-or-fresh".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect("a new durable run starts a fresh exact receipt ledger");+ }++ #[test]+ fn legacy_full_exact_receipt_epoch_is_quarantined_before_dispatch() {+ let operation = AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation.clone()],+ })+ .unwrap();+ for index in 0..MAX_SETTLED_CALL_IDS {+ let call_id = format!("legacy-call-{index}");+ bridge.settled_call_ids.order.push_back(call_id.clone());+ bridge.settled_call_ids.members.insert(call_id);+ }++ let error = bridge+ .begin_call(+ "fresh-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("a full legacy exact epoch cannot dispatch more work");+ assert!(error.is_active_turn_receipt_limit());+ assert!(bridge.pending.is_empty());++ bridge.prepare_turn().unwrap();+ let error = bridge+ .begin_call(+ "fresh-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect_err("a turn boundary must preserve the full durable ledger");+ assert!(error.is_active_turn_receipt_limit());++ bridge+ .attach_run(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation],+ })+ .unwrap();+ bridge+ .begin_call(+ "fresh-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect("a new durable run resets the full legacy ledger");+ }++ #[test]+ fn provider_restart_rotates_only_an_idle_replay_epoch() {+ let operation = AuthorizedTool {+ operation_id: "get_task_context".to_owned(),+ version: 1,+ description: "Read the active task context.".to_owned(),+ input_schema: json!({"type": "object"}),+ response_schema: json!({"type": "object"}),+ };+ let mut bridge = ProviderToolBridge::default();+ bridge+ .prepare(AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))+ .unwrap(),+ operations: vec![operation],+ })+ .unwrap();+ bridge+ .begin_call(+ "old-epoch-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .unwrap();+ bridge+ .apply_result(ToolResult {+ call_id: "old-epoch-call".to_owned(),+ operation_id: "get_task_context".to_owned(),+ result: json!({"ok": true}),+ is_error: false,+ })+ .unwrap();+ bridge.settle_turn("provider_turn_terminated").unwrap();+ bridge.settled_call_filter.words = vec![0; REPLAY_FILTER_WORDS];+ bridge.settled_call_filter.words[0] = 1;++ let error = bridge+ .rollover_replay_epoch_after_provider_restart()+ .expect_err("retained replay results keep the current epoch authoritative");+ assert!(error.to_string().contains("receipts are retained"));+ assert!(bridge+ .begin_call(+ "old-epoch-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .is_err());++ bridge.prepare_turn().unwrap();+ bridge+ .rollover_replay_epoch_after_provider_restart()+ .expect("an idle replacement process establishes fresh replay authority");+ bridge+ .begin_call(+ "old-epoch-call".to_owned(),+ "get_task_context".to_owned(),+ json!({}),+ )+ .expect("the fresh process generation remains live after bounded rotation");}}diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rsindex 634f8ddda6..801763b8bc 100644--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs@@ -12,6 +12,29 @@ pub struct NormalizedProviderEvent {pub payload: Value,}+pub(crate) fn normalized_codex_terminal_event_type(+ method: &str,+ params: &Value,+) -> Option<&'static str> {+ let status = match method {+ "turn/failed" => "failed",+ "turn/cancelled" => "cancelled",+ "turn/interrupted" => "interrupted",+ "turn/completed" => string(+ params+ .pointer("/turn/status")+ .or_else(|| params.get("status")),+ ),+ _ => return None,+ };+ Some(match status {+ "failed" | "error" => "turn.failed",+ "cancelled" | "canceled" => "turn.cancelled",+ "interrupted" | "aborted" => "turn.interrupted",+ _ => "turn.completed",+ })+}+fn bounded_text(value: &str, max_chars: usize) -> String {redact_text(value).chars().take(max_chars).collect()}@@ -118,18 +141,19 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali"providerTurnId": params.pointer("/turn/id").or_else(|| params.get("turnId")).and_then(Value::as_str),}),),- "turn/completed" => {- let status = string(- params- .pointer("/turn/status")- .or_else(|| params.get("status")),- );- let event_type = match status {- "failed" | "error" => "turn.failed",- "cancelled" | "canceled" => "turn.cancelled",- "interrupted" | "aborted" => "turn.interrupted",- _ => "turn.completed",+ "turn/completed" | "turn/failed" | "turn/cancelled" | "turn/interrupted" => {+ let status = match method {+ "turn/failed" => "failed",+ "turn/cancelled" => "cancelled",+ "turn/interrupted" => "interrupted",+ _ => string(+ params+ .pointer("/turn/status")+ .or_else(|| params.get("status")),+ ),};+ let event_type = normalized_codex_terminal_event_type(method, params)+ .expect("matched Codex terminal method has a normalized terminal type");push(&mut events,event_type,@@ -366,6 +390,16 @@ mod tests {);assert_eq!(terminal[0].event_type, "turn.failed");assert_eq!(terminal[0].priority, EventPriority::P0);+ for (method, expected) in [+ ("turn/failed", "turn.failed"),+ ("turn/cancelled", "turn.cancelled"),+ ("turn/interrupted", "turn.interrupted"),+ ] {+ let terminal =+ normalize_codex_notification(method, &json!({"turnId": "provider-turn"}));+ assert_eq!(terminal[0].event_type, expected);+ assert_eq!(terminal[0].priority, EventPriority::P0);+ }let usage = normalize_codex_notification("thread/tokenUsage/updated",diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rsindex 08c3380360..4e8050f1b1 100644--- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs@@ -5,9 +5,14 @@ use std::sync::atomic::{AtomicU64, Ordering};use paperclip_runner_core::codex_provider::{CodexProvider, CodexProviderConfig, CodexProviderEvent,};-use paperclip_runner_core::durable::{Command, CommandExecutor, DurableRunnerError, PolledEvent};+use paperclip_runner_core::durable::{+ Command, CommandExecutor, DurableRunnerConfig, DurableRunnerError, PolledEvent,+};use paperclip_runner_core::provider_backend::CodexCommandExecutor;-use paperclip_runner_core::provider_bridge::{AuthorizedTool, ToolResult};+use paperclip_runner_core::provider_bridge::{+ authorized_tool_catalog_digest, AuthorizedTool, AuthorizedToolSet, ProviderToolBridge,+ ToolResult, TOOL_SET_SCHEMA,+};use paperclip_runner_core::provider_events::normalize_codex_notification;use serde_json::{json, Value};@@ -62,6 +67,36 @@ fn task_context_tool() -> AuthorizedTool {}}+fn task_context_tool_set() -> AuthorizedToolSet {+ let operations = vec![task_context_tool()];+ AuthorizedToolSet {+ schema: TOOL_SET_SCHEMA.to_owned(),+ schema_version: 1,+ catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(),+ operations,+ }+}++fn durable_config(directory: &Path) -> DurableRunnerConfig {+ DurableRunnerConfig {+ connect_url: "ws://127.0.0.1:3000/runner".to_owned(),+ state_dir: directory.to_path_buf(),+ runner_instance_id: "runner-1".to_owned(),+ environment_lease_id: "lease-1".to_owned(),+ run_id: "run-1".to_owned(),+ normalized_session_id: "session-1".to_owned(),+ turn_id: "turn-1".to_owned(),+ item_id: "item-1".to_owned(),+ runner_version: "test-1".to_owned(),+ runner_digest: format!("sha256:{}", "a".repeat(64)),+ max_outbox_bytes: 16 * 1024 * 1024,+ p0_reserve_bytes: 1024 * 1024,+ max_frame_bytes: 1024 * 1024,+ reconnect_delay: std::time::Duration::from_millis(1),+ max_runtime: std::time::Duration::from_secs(5),+ }+}+fn command(id: &str, sequence: u64, command_type: &str, payload: Value) -> Command {Command {schema: "paperclip.prp.command.v1".to_owned(),@@ -83,6 +118,14 @@ fn call_count(directory: &Path, method: &str) -> usize {.count()}+fn recorded_tool_responses(directory: &Path) -> Vec<String> {+ fs::read_to_string(directory.join("calls.log"))+ .unwrap_or_default()+ .lines()+ .filter_map(|line| line.strip_prefix("tool-response:").map(str::to_owned))+ .collect()+}+fn poll_and_ack(executor: &mut CodexCommandExecutor,) -> Result<Vec<PolledEvent>, DurableRunnerError> {@@ -91,6 +134,132 @@ fn poll_and_ack(Ok(events)}+fn wait_for_notification(provider: &mut CodexProvider, expected_method: &str) -> Value {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while std::time::Instant::now() < deadline {+ match provider.poll().expect("poll provider notification") {+ Some(CodexProviderEvent::Notification { method, params }) => {+ if method == expected_method {+ return params;+ }+ }+ Some(_) => {}+ None => std::thread::sleep(std::time::Duration::from_millis(1)),+ }+ }+ panic!("did not observe Codex {expected_method} notification before the deadline");+}++fn wait_for_provider_error(provider: &mut CodexProvider) -> String {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while std::time::Instant::now() < deadline {+ match provider.poll() {+ Err(error) => return error.to_string(),+ Ok(Some(_)) => {}+ Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)),+ }+ }+ panic!("did not observe the expected Codex provider error before the deadline");+}++fn wait_for_executor_event(+ executor: &mut CodexCommandExecutor,+ expected_event_type: &str,+) -> PolledEvent {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while std::time::Instant::now() < deadline {+ let events = poll_and_ack(executor).expect("poll Codex executor event");+ if let Some(event) = events+ .into_iter()+ .find(|event| event.event_type == expected_event_type)+ {+ return event;+ }+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ panic!("did not observe Codex {expected_event_type} event before the deadline");+}++fn wait_for_provider_exit(provider: &mut CodexProvider) {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while std::time::Instant::now() < deadline {+ if matches!(+ provider.poll().expect("poll terminated provider"),+ Some(CodexProviderEvent::Exited { .. })+ ) {+ return;+ }+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ panic!("the provider accepted a reused turn identity but remained live");+}++fn wait_for_reused_identity_reap(provider: &mut CodexProvider) {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while std::time::Instant::now() < deadline {+ match provider.poll() {+ Err(error) => assert!(+ error.to_string().contains("reused a settled"),+ "unexpected error while reaping reused provider identity: {error}"+ ),+ Ok(Some(CodexProviderEvent::Exited {+ completed_turn_authoritative,+ ..+ })) => {+ assert!(+ !completed_turn_authoritative,+ "accepted identity reuse must revoke prior completion authority"+ );+ return;+ }+ Ok(Some(_)) => {}+ Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)),+ }+ }+ panic!("the provider accepted a reused turn identity but was not reaped");+}++fn wait_for_fake_provider_idle(directory: &Path) {+ let state_path = directory.join("fake-state.json");+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while std::time::Instant::now() < deadline {+ let active_turn_id = fs::read(&state_path)+ .ok()+ .and_then(|bytes| serde_json::from_slice::<Value>(&bytes).ok())+ .and_then(|state| state.get("activeTurnId").cloned());+ if active_turn_id == Some(Value::Null) {+ return;+ }+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ panic!("the fake provider did not persist its idle turn state before the deadline");+}++fn saturate_provider_tool_receipts(directory: &Path) {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(task_context_tool_set()).unwrap();+ for index in 0..4_096 {+ let call_id = format!("retained-call-{index}");+ bridge+ .begin_call(call_id.clone(), "get_task_context".into(), json!({}))+ .unwrap();+ bridge+ .apply_result(ToolResult {+ call_id,+ operation_id: "get_task_context".into(),+ result: json!({"ok": true}),+ is_error: false,+ })+ .unwrap();+ }+ let state_path = directory.join("codex-provider-state.json");+ let mut persisted: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read provider state")).unwrap();+ persisted["toolBridge"] = serde_json::to_value(bridge).unwrap();+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())+ .expect("write saturated provider state");+}+#[test]fn codex_transport_buffers_notifications_while_waiting_for_responses() {let directory = temporary_directory("buffering");@@ -505,6 +674,285 @@ fn clean_provider_exit_does_not_refail_a_completed_turn() {fs::remove_dir_all(directory).expect("remove Codex integration-test directory");}+#[test]+fn durable_backend_reaps_before_rotating_full_provider_identity_epochs() {+ let directory = temporary_directory("provider-identity-epoch-rollover");+ let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]);+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ drop(first);++ let state_path = directory.join("codex-provider-state.json");+ let mut persisted: Value = serde_json::from_slice(+ &fs::read(&state_path).expect("read provider state before epoch rollover"),+ )+ .expect("parse provider state before epoch rollover");+ let prior_generation = persisted["providerProcessGeneration"]+ .as_u64()+ .expect("provider generation is persisted");+ persisted["settledProviderTurnIds"] = Value::Array(+ (0..4_096)+ .map(|index| Value::String(format!("provider-turn-{index}")))+ .collect(),+ );+ persisted["completedTurnAuthoritative"] = Value::Bool(true);+ persisted["completedTurnProcessGeneration"] = json!(prior_generation);+ persisted["completedProviderTurnId"] = json!("provider-turn-4095");+ persisted["toolBridge"]["settledCallIds"] = Value::Array(+ (0..65_536)+ .map(|index| Value::String(format!("semantic-call-{index}")))+ .collect(),+ );+ persisted["toolBridge"]["durableRunReceiptLimitReached"] = Value::Bool(true);+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())+ .expect("write full provider identity epochs");++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ recovered+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Start after a verified provider epoch rollover."}),+ ))+ .expect("roll over the idle provider generation and start fresh work");++ let rolled: Value = serde_json::from_slice(+ &fs::read(&state_path).expect("read provider state after epoch rollover"),+ )+ .expect("parse provider state after epoch rollover");+ assert!(rolled["providerProcessGeneration"].as_u64().unwrap() > prior_generation);+ assert_eq!(+ rolled["settledProviderTurnIds"],+ json!(["provider-turn-4095"])+ );+ assert_eq!(rolled["settledProviderTurnFilter"], json!({"words": []}));+ assert_eq!(rolled["toolBridge"]["settledCallIds"], json!([]));+ assert_eq!(+ rolled["toolBridge"]["settledCallFilter"],+ json!({"words": []})+ );+ assert!(rolled["toolBridge"]["durableRunReceiptLimitReached"].is_null());++ let mut event_types = Vec::new();+ let semantic_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while std::time::Instant::now() < semantic_deadline {+ event_types.extend(+ poll_and_ack(&mut recovered)+ .expect("poll semantic work in the fresh provider epoch")+ .into_iter()+ .map(|event| event.event_type),+ );+ if event_types+ .iter()+ .any(|event| event == "semantic_tool.input")+ {+ break;+ }+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ assert!(event_types+ .iter()+ .any(|event| event == "semantic_tool.input"));++ recovered.shutdown().expect("stop rolled provider process");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_closes_when_identity_rollover_resumes_unowned_work() {+ let directory = temporary_directory("provider-identity-rollover-unowned-work");+ let config = provider_config(&directory, &["--resume-unowned-turn-when-marked"]);+ let mut first = CodexCommandExecutor::new(&directory);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ drop(first);++ let state_path = directory.join("codex-provider-state.json");+ let mut persisted: Value = serde_json::from_slice(+ &fs::read(&state_path).expect("read provider state before unsafe rollover"),+ )+ .expect("parse provider state before unsafe rollover");+ persisted["settledProviderTurnIds"] = Value::Array(+ (0..4_096)+ .map(|index| Value::String(format!("provider-turn-{index}")))+ .collect(),+ );+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())+ .expect("write full provider identity epoch");++ let mut recovered = CodexCommandExecutor::new(&directory);+ poll_and_ack(&mut recovered).expect("attach an idle provider before rollover");+ wait_for_fake_provider_idle(&directory);+ let attached: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read attached provider state"))+ .expect("parse attached provider state");+ let attached_generation = attached["providerProcessGeneration"]+ .as_u64()+ .expect("attached provider generation is persisted");++ // Race the provider's idle snapshot with work Paperclip never dispatched.+ // The replacement process observes this turn during thread/resume and must+ // close the durable run instead of leaving a quarantined session open.+ fs::write(directory.join("resume-unowned-turn"), b"armed")+ .expect("arm unowned provider work before rollover");+ let resumes_before_rollover = call_count(&directory, "thread/resume");+ let error = recovered+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Never overlap the unowned provider turn."}),+ ))+ .expect_err("identity rollover fails closed after resuming unowned work");+ assert!(error+ .to_string()+ .contains("identity epoch rollover failed closed after an invalid accepted identity"));+ assert_eq!(+ call_count(&directory, "thread/resume"),+ resumes_before_rollover + 1,+ );++ let closed: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read fail-closed provider state"))+ .expect("parse fail-closed provider state");+ assert_eq!(closed["lifecycle"], "closed");+ assert!(closed["activeProviderTurnId"].is_null());+ assert_eq!(closed["ambiguousTurnStartPending"], false);+ assert_eq!(closed["completedTurnAuthoritative"], false);+ assert!(closed["providerProcessGeneration"].as_u64().unwrap() > attached_generation);++ let events = poll_and_ack(&mut recovered).expect("read fail-closed rollover diagnostic");+ assert!(events.iter().any(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "provider_turn_identity_invalid"+ && event.payload["paperclipAccepted"] == false+ && event.payload["providerAccepted"] == true+ }));++ let resumes_before_retry = call_count(&directory, "thread/resume");+ assert!(recovered+ .execute(&command(+ "turn-retry",+ 4,+ "turn.start",+ json!({"text": "Do not resume the quarantined provider."}),+ ))+ .unwrap_err()+ .to_string()+ .contains("provider session is closed"));+ assert_eq!(+ call_count(&directory, "thread/resume"),+ resumes_before_retry,+ "closed rollover state must never resume the unowned provider turn",+ );++ recovered.shutdown().expect("close fail-closed executor");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn direct_provider_reaps_before_rotating_a_full_turn_identity_epoch() {+ let directory = temporary_directory("direct-turn-identity-epoch-rollover");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--resume-unowned-turn-when-marked",+ ],+ );+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)+ .expect("start Codex provider");+ let initial_process_id = provider.process_id();++ // Keep every identity exact until the process boundary makes it safe to+ // forget them. The next turn must transparently resume in a new process+ // generation instead of permanently rejecting this session.+ for index in 0..4_096 {+ provider+ .start_turn(&format!("Complete provider turn {index}."), &config.cwd)+ .expect("start provider turn before identity rollover");+ wait_for_notification(&mut provider, "turn/completed");+ }++ provider+ .start_turn(+ "Continue after the exact identity epoch fills.",+ &config.cwd,+ )+ .expect("roll over the provider process and start fresh work");+ assert_ne!(provider.process_id(), initial_process_id);+ assert_eq!(call_count(&directory, "thread/resume"), 1);+ wait_for_notification(&mut provider, "turn/completed");+ let rolled_process_id = provider.process_id();++ // Fill the next process epoch and force its resume probe to observe an+ // active turn. Rollover must retain that work instead of dispatching a+ // concurrent replacement after forgetting the old exact identities. The+ // prior generation's authoritative completion remains as one tombstone in+ // this epoch, so 4,094 additional completions fill the remaining slots.+ for index in 1..4_095 {+ provider+ .start_turn(+ &format!("Complete rolled provider turn {index}."),+ &config.cwd,+ )+ .expect("start provider turn in the rolled identity epoch");+ wait_for_notification(&mut provider, "turn/completed");+ }+ assert_eq!(provider.process_id(), rolled_process_id);+ // The terminal notification is flushed before the fake provider persists+ // its idle state. Wait for that write, then arm a one-shot resume race so+ // only the replacement generation reports unowned active work.+ wait_for_fake_provider_idle(&directory);+ fs::write(directory.join("resume-unowned-turn"), b"armed")+ .expect("arm unowned work for the replacement provider resume");+ let error = provider+ .start_turn(+ "Do not overlap the turn recovered during epoch rollover.",+ &config.cwd,+ )+ .expect_err("a resumed active turn is reaped before replacement work");+ assert!(error+ .to_string()+ .contains("resumed unowned active work; the provider was terminated"));+ assert_eq!(provider.active_provider_turn_id(), None);+ assert!(provider+ .start_turn(+ "Never admit replacement work after quarantine.",+ &config.cwd,+ )+ .unwrap_err()+ .to_string()+ .contains("quarantined after unsafe recovered work"));+ wait_for_provider_exit(&mut provider);++ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}+#[test]fn post_completion_observation_does_not_hide_same_or_resumed_process_failure() {let directory = temporary_directory("completion-then-nonzero-exit");@@ -650,7 +1098,11 @@ fn rejected_replacement_turn_start_preserves_result_and_exit_authority() {.start_turn("Reject replacement work.", &config.cwd).expect_err("the replacement turn/start returns a definite rejection");let mut buffered_notification_seen = false;- let rejected_start_exit = (0..64).find_map(|_| {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ let rejected_start_exit = loop {+ if std::time::Instant::now() >= deadline {+ break None;+ }match provider.poll().expect("poll exit after rejected replacement start")@@ -661,21 +1113,23 @@ fn rejected_replacement_turn_start_preserves_result_and_exit_authority() {== Some("buffered before replacement rejection") =>{buffered_notification_seen = true;- None}Some(CodexProviderEvent::Exited {success,completed_turn_authoritative,completion_reconciles_exit,..- }) => Some((- success,- completed_turn_authoritative,- completion_reconciles_exit,- )),- _ => None,+ }) => {+ break Some((+ success,+ completed_turn_authoritative,+ completion_reconciles_exit,+ ));+ }+ _ => {}}- });+ std::thread::sleep(std::time::Duration::from_millis(1));+ };assert!(buffered_notification_seen);assert_eq!(rejected_start_exit, Some((false, true, false)));@@ -727,7 +1181,11 @@ fn rejected_replacement_turn_start_does_not_hide_contradictory_turn_evidence() {"unexpected duplicate-start error: {duplicate_error}");let mut contradictory_turn_seen = false;- let rejected_start_exit = (0..64).find_map(|_| {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ let rejected_start_exit = loop {+ if std::time::Instant::now() >= deadline {+ break None;+ }match provider.poll().expect("poll exit after contradictory replacement rejection")@@ -738,21 +1196,23 @@ fn rejected_replacement_turn_start_does_not_hide_contradictory_turn_evidence() {== Some("provider-turn-contradiction") =>{contradictory_turn_seen = true;- None}Some(CodexProviderEvent::Exited {success,completed_turn_authoritative,completion_reconciles_exit,..- }) => Some((- success,- completed_turn_authoritative,- completion_reconciles_exit,- )),- _ => None,+ }) => {+ break Some((+ success,+ completed_turn_authoritative,+ completion_reconciles_exit,+ ));+ }+ _ => {}}- });+ std::thread::sleep(std::time::Duration::from_millis(1));+ };assert!(contradictory_turn_seen);assert_eq!(provider.active_provider_turn_id(),@@ -771,7 +1231,6 @@ fn ambiguous_or_dead_replacement_start_preserves_result_not_exit_authority() {"--fail-after-accepting-second-turn-before-response",),("malformed-error", "--malformed-error-second-turn-start"),- ("missing-turn-id", "--missing-id-second-turn-start"),] {let directory = temporary_directory(label);let config = provider_config(&directory, &[switch]);@@ -822,6 +1281,63 @@ fn ambiguous_or_dead_replacement_start_preserves_result_not_exit_authority() {}}+#[test]+fn durable_backend_closes_after_accepted_turn_omits_its_identity() {+ let directory = temporary_directory("accepted-turn-missing-identity");+ let config = provider_config(&directory, &["--missing-id-live-turn-start"]);+ let runner_config = durable_config(&directory);+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ executor+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare Codex provider");+ executor+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");++ let error = executor+ .execute(&command(+ "turn-without-id",+ 3,+ "turn.start",+ json!({"text": "Accept work but omit its durable identity."}),+ ))+ .expect_err("accepted work without a turn identity must fail closed");+ assert!(error.to_string().contains("omitted turn.id"));++ let persisted: Value = serde_json::from_slice(+ &fs::read(directory.join("codex-provider-state.json"))+ .expect("read fail-closed provider state"),+ )+ .expect("parse fail-closed provider state");+ assert_eq!(persisted["lifecycle"], "closed");+ assert!(persisted["activeProviderTurnId"].is_null());++ drop(executor);+ let resumes_before_closed_restore = call_count(&directory, "thread/resume");+ let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let events = closed+ .poll_events()+ .expect("closed invalid-identity state remains readable without resuming Codex");+ assert!(events.iter().any(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "provider_turn_identity_invalid"+ && event.payload["providerTurnId"].is_null()+ }));+ assert_eq!(+ call_count(&directory, "thread/resume"),+ resumes_before_closed_restore,+ "recovery must not resume provider work accepted without a durable identity"+ );++ closed.shutdown().expect("close fail-closed executor");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}+#[test]fn ambiguous_replacement_turn_adopts_one_later_completion_identity() {for (label, switch, omit_started) in [@@ -940,7 +1456,7 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() {let config = provider_config(&directory,&[- "--missing-id-second-turn-start",+ "--malformed-error-second-turn-start","--conflicting-ambiguous-second-turn",],);@@ -948,41 +1464,23 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() {provider.start_turn("Complete the first turn.", &config.cwd).expect("start first provider turn");- let first_completed = (0..32).any(|_| {- matches!(- provider.poll().expect("poll first turn"),- Some(CodexProviderEvent::Notification { method, .. })- if method == "turn/completed"- )- });- assert!(- first_completed,- "observe the authoritative first completion"- );+ wait_for_notification(&mut provider, "turn/completed");provider.start_turn("Accept replacement work ambiguously.", &config.cwd)- .expect_err("the replacement response omits its turn identity");- let replacement_started = provider- .poll()- .expect("poll replacement start")- .expect("replacement start is available");- assert!(matches!(- replacement_started,- CodexProviderEvent::Notification { method, params }- if method == "turn/started"- && params.pointer("/turn/id").and_then(Value::as_str)- == Some("provider-turn-2")- ));+ .expect_err("the replacement response is transport-ambiguous");+ let replacement_started = wait_for_notification(&mut provider, "turn/started");+ assert_eq!(+ replacement_started+ .pointer("/turn/id")+ .and_then(Value::as_str),+ Some("provider-turn-2")+ );assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));- let conflicting_completion = provider- .poll()- .expect_err("a second replacement identity must fail closed");+ let conflicting_completion = wait_for_provider_error(&mut provider);assert!(- conflicting_completion- .to_string()- .contains("another active turn"),+ conflicting_completion.contains("another active turn"),"unexpected conflicting-identity error: {conflicting_completion}");assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));@@ -990,6 +1488,53 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() {fs::remove_dir_all(directory).expect("remove Codex integration-test directory");}+#[test]+fn ambiguous_replacement_turn_rejects_an_older_settled_identity() {+ let directory = temporary_directory("ambiguous-older-settled-turn-identity");+ let config = provider_config(&directory, &["--ambiguous-older-reused-turn"]);+ let mut provider = CodexProvider::start(&config, None).expect("start Codex provider");+ for message in ["Complete turn one.", "Complete turn two."] {+ provider+ .start_turn(message, &config.cwd)+ .expect("start completed provider turn");+ wait_for_notification(&mut provider, "turn/completed");+ }+ provider+ .start_turn("Ambiguously reuse the first turn identity.", &config.cwd)+ .expect_err("the replacement response is transport-ambiguous");+ let error = wait_for_provider_error(&mut provider);+ assert!(+ error.contains("reused a settled provider turn identity"),+ "unexpected older-identity error: {error}"+ );+ assert_eq!(provider.active_provider_turn_id(), None);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ let completed_turn_authoritative = (0..)+ .take_while(|_| std::time::Instant::now() < deadline)+ .find_map(|_| {+ match provider+ .poll()+ .expect("poll provider after rejected identity")+ {+ Some(CodexProviderEvent::Exited {+ completed_turn_authoritative,+ ..+ }) => Some(completed_turn_authoritative),+ Some(_) => None,+ None => {+ std::thread::sleep(std::time::Duration::from_millis(1));+ None+ }+ }+ });+ assert!(+ completed_turn_authoritative == Some(false),+ "accepted work with an older settled identity must terminate the provider and revoke completion authority"+ );++ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}+#[test]fn clean_exit_after_ambiguous_replacement_start_fails_the_durable_session() {let directory = temporary_directory("durable-clean-exit-after-ambiguous-turn-start");@@ -1161,7 +1706,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() {let config = provider_config(&directory,&[- "--missing-id-second-turn-start",+ "--malformed-error-second-turn-start","--hold-ambiguous-second-turn-after-item",],);@@ -1185,15 +1730,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() {json!({"text": "Complete the first turn."}),)).expect("start first provider turn");- for _ in 0..32 {- if poll_and_ack(&mut executor)- .expect("poll first turn")- .iter()- .any(|event| event.event_type == "turn.completed")- {- break;- }- }+ wait_for_executor_event(&mut executor, "turn.completed");executor.execute(&command(@@ -1202,7 +1739,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() {"turn.start",json!({"text": "Emit replacement output before terminal authority."}),))- .expect_err("replacement response omits its identity");+ .expect_err("replacement response is transport-ambiguous");assert!(poll_and_ack(&mut executor).expect("defer identity-less replacement output")@@ -1252,7 +1789,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() {let config = provider_config(&directory,&[- "--missing-id-second-turn-start",+ "--malformed-error-second-turn-start","--complete-ambiguous-second-turn-before-response","--omit-ambiguous-turn-started",],@@ -1277,15 +1814,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() {json!({"text": "Complete the first turn."}),)).expect("start first provider turn");- for _ in 0..32 {- if poll_and_ack(&mut executor)- .expect("poll first turn")- .iter()- .any(|event| event.event_type == "turn.completed")- {- break;- }- }+ wait_for_executor_event(&mut executor, "turn.completed");executor.execute(&command(@@ -1294,7 +1823,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() {"turn.start",json!({"text": "Complete replacement work before returning an invalid response."}),))- .expect_err("replacement response omits its identity");+ .expect_err("replacement response is transport-ambiguous");executor.shutdown().expect("stop first provider process");drop(executor);@@ -1335,7 +1864,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() {let config = provider_config(&directory,&[- "--missing-id-second-turn-start",+ "--malformed-error-second-turn-start","--complete-ambiguous-second-turn",],);@@ -1360,19 +1889,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() {)).expect("start first provider turn");- let mut first_events = Vec::new();- for _ in 0..32 {- first_events.extend(- poll_and_ack(&mut executor)- .expect("poll first turn")- .into_iter()- .map(|event| event.event_type),- );- if first_events.iter().any(|event| event == "turn.completed") {- break;- }- }- assert!(first_events.iter().any(|event| event == "turn.completed"));+ wait_for_executor_event(&mut executor, "turn.completed");executor.execute(&command(@@ -1381,10 +1898,11 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() {"turn.start",json!({"text": "Complete replacement work after the malformed response."}),))- .expect_err("accepted replacement response omits its turn identity");+ .expect_err("accepted replacement response is transport-ambiguous");let mut replacement_events = Vec::new();- for _ in 0..64 {+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while std::time::Instant::now() < deadline {replacement_events.extend(poll_and_ack(&mut executor).expect("poll accepted replacement evidence")@@ -1397,6 +1915,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() {{break;}+ std::thread::sleep(std::time::Duration::from_millis(1));}assert!(replacement_events.iter()@@ -1479,81 +1998,128 @@ fn codex_rejects_a_tool_call_that_was_not_advertised() {}#[test]-fn codex_resume_advertises_the_same_authorized_tools() {- let directory = temporary_directory("dynamic-tool-resume");- let config = provider_config(&directory, &["--require-dynamic-tool"]);- let mut provider =- CodexProvider::start_with_tools(&config, [task_context_tool()], Some("codex-thread-1"))- .expect("resume Codex with the run-scoped tool set");- assert_eq!(provider.thread_id(), "codex-thread-1");- provider.shutdown().expect("stop provider");+fn codex_rejects_delayed_calls_after_every_terminal_notification() {+ let directory = temporary_directory("delayed-tool-after-failure");+ let config = provider_config(&directory, &["--delayed-tool-after-failed-turn"]);+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)+ .expect("start Codex with an authorized tool");+ provider+ .start_turn("Fail before invoking a tool.", &config.cwd)+ .expect("start provider turn");++ wait_for_notification(&mut provider, "turn/failed");+ assert_eq!(provider.active_provider_turn_id(), None);+ let warning = wait_for_notification(&mut provider, "warning");+ assert!(warning["message"]+ .as_str()+ .is_some_and(|message| message.contains("after the Codex turn terminated")));++ let _ = provider.shutdown();fs::remove_dir_all(directory).expect("remove Codex integration-test directory");}#[test]-fn durable_backend_resumes_the_active_thread_without_restarting_the_turn() {- let directory = temporary_directory("resume");- let config = provider_config(&directory, &["--hold-turn"]);- let mut first = CodexCommandExecutor::new(&directory);- first- .execute(&command(- "prepare",- 1,- "run.prepare",- json!({"provider": config}),- ))- .expect("prepare Codex provider");- first- .execute(&command("open", 2, "session.open", json!({})))- .expect("open Codex session");- first- .execute(&command(- "turn",- 3,- "turn.start",- json!({"text": "Hold this turn for recovery."}),- ))- .expect("start held provider turn");- assert_eq!(call_count(&directory, "turn/start"), 1);- first.shutdown().expect("stop first provider process");- drop(first);+fn codex_rejects_a_delayed_prior_turn_call_while_the_next_turn_is_active() {+ let directory = temporary_directory("delayed-tool-after-next-turn-start");+ let config = provider_config(&directory, &["--delayed-tool-after-next-turn-start"]);+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)+ .expect("start Codex with an authorized tool");+ provider+ .start_turn("Complete the first turn.", &config.cwd)+ .expect("start first provider turn");+ wait_for_notification(&mut provider, "turn/completed");- let mut recovered = CodexCommandExecutor::new(&directory);- let snapshot = recovered- .execute(&command("snapshot", 4, "session.snapshot", json!({})))- .expect("restore provider session");- assert_eq!(snapshot.result["status"], "turn_active");- assert_eq!(snapshot.result["activeProviderTurnId"], "provider-turn-1");- assert_eq!(call_count(&directory, "turn/start"), 1);- assert_eq!(call_count(&directory, "thread/resume"), 1);- assert_eq!(call_count(&directory, "thread/read"), 1);+ provider+ .start_turn("Keep the second turn active.", &config.cwd)+ .expect("start second provider turn");+ assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));+ let warning = wait_for_notification(&mut provider, "warning");+ assert!(warning["message"]+ .as_str()+ .is_some_and(|message| message.contains("after the Codex turn terminated")));+ assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));- recovered- .execute(&command("interrupt", 5, "turn.interrupt", json!({})))- .expect("interrupt recovered provider turn");- let mut terminal_seen = false;- for _ in 0..16 {- let events = poll_and_ack(&mut recovered).expect("poll interrupted turn");- terminal_seen |= events- .iter()- .any(|event| event.event_type == "turn.interrupted");- if terminal_seen {- break;- }+ let _ = provider.shutdown();+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn codex_rejects_a_two_turn_old_call_while_fresh_work_is_active() {+ let directory = temporary_directory("delayed-tool-after-third-turn-start");+ let config = provider_config(&directory, &["--delayed-tool-after-third-turn-start"]);+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)+ .expect("start Codex with an authorized tool");++ for message in ["Complete turn one.", "Complete turn two."] {+ provider+ .start_turn(message, &config.cwd)+ .expect("start completed provider turn");+ wait_for_notification(&mut provider, "turn/completed");}- assert!(terminal_seen);- recovered- .shutdown()- .expect("stop recovered provider process");+ provider+ .start_turn("Keep turn three active.", &config.cwd)+ .expect("start third provider turn");+ assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-3"));++ let warning = wait_for_notification(&mut provider, "warning");+ assert_eq!(warning["providerMethod"], "item/tool/call");+ assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-3"));++ let _ = provider.shutdown();fs::remove_dir_all(directory).expect("remove Codex integration-test directory");}#[test]-fn provider_exit_preserves_and_reconciles_the_active_turn() {- let directory = temporary_directory("exit-active-turn");- let config = provider_config(&directory, &["--exit-after-turn-start"]);- let mut executor = CodexCommandExecutor::new(&directory);- executor+fn codex_fails_closed_when_a_provider_reuses_a_settled_turn_id() {+ let directory = temporary_directory("tool-after-reused-turn-start");+ let config = provider_config(&directory, &["--tool-after-reused-turn-start"]);+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)+ .expect("start Codex with an authorized tool");+ provider+ .start_turn("Complete the first turn.", &config.cwd)+ .expect("start first provider turn");+ wait_for_notification(&mut provider, "turn/completed");++ let error = provider+ .start_turn("Reuse the settled provider identity.", &config.cwd)+ .expect_err("reject a provider turn with a reused identity");+ assert!(error.to_string().contains("reused a settled"));+ assert_eq!(provider.active_provider_turn_id(), None);+ wait_for_reused_identity_reap(&mut provider);++ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn codex_fails_closed_when_a_provider_reuses_an_older_settled_turn_id() {+ let directory = temporary_directory("tool-after-older-reused-turn-start");+ let config = provider_config(&directory, &["--tool-after-older-reused-turn-start"]);+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)+ .expect("start Codex with an authorized tool");+ for message in ["Complete turn one.", "Complete turn two."] {+ provider+ .start_turn(message, &config.cwd)+ .expect("start completed provider turn");+ wait_for_notification(&mut provider, "turn/completed");+ }++ let error = provider+ .start_turn("Reuse the older settled provider identity.", &config.cwd)+ .expect_err("reject a provider turn with an older reused identity");+ assert!(error.to_string().contains("reused a settled"));+ assert_eq!(provider.active_provider_turn_id(), None);+ wait_for_reused_identity_reap(&mut provider);++ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_rejects_an_older_provider_turn_id_after_restart() {+ let directory = temporary_directory("durable-older-reused-turn-after-restart");+ let config = provider_config(&directory, &[]);+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first.execute(&command("prepare",1,@@ -1561,65 +2127,82 @@ fn provider_exit_preserves_and_reconciles_the_active_turn() {json!({"provider": config}),)).expect("prepare Codex provider");- executor+ first.execute(&command("open", 2, "session.open", json!({}))).expect("open Codex session");- executor++ for (sequence, message) in [(3, "Complete turn one."), (4, "Complete turn two.")] {+ first+ .execute(&command(+ &format!("turn-{sequence}"),+ sequence,+ "turn.start",+ json!({"text": message}),+ ))+ .expect("start completed provider turn");+ let mut completed = false;+ for _ in 0..32 {+ completed |= poll_and_ack(&mut first)+ .expect("poll completed provider turn")+ .iter()+ .any(|event| event.event_type == "turn.completed");+ if completed {+ break;+ }+ }+ assert!(completed);+ }+ first.shutdown().expect("stop first provider process");+ drop(first);++ // The fake provider's process-local counter restarts at provider-turn-1.+ // Durable state must still remember that older identity, not only the most+ // recent provider-turn-2 completion authority.+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let error = recovered.execute(&command(- "turn",- 3,+ "turn-reused",+ 5,"turn.start",- json!({"text": "Keep the native turn active while the provider exits."}),+ json!({"text": "Do not accept an older provider turn identity."}),))- .expect("start provider turn");+ .expect_err("reject a provider turn identity retained before restart");+ assert!(error.to_string().contains("reused a settled"));- let mut provider_exit_seen = false;- for _ in 0..32 {- provider_exit_seen |= poll_and_ack(&mut executor)- .expect("poll provider exit")- .iter()- .any(|event| event.event_type == "session.failed");- if provider_exit_seen {- break;- }- }- assert!(provider_exit_seen);let persisted: Value = serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json"))- .expect("read provider state after exit"),+ .expect("read fail-closed provider state"),)- .expect("parse provider state after exit");- assert_eq!(persisted["lifecycle"], "provider_exited");- assert_eq!(persisted["activeProviderTurnId"], "provider-turn-1");+ .expect("parse fail-closed provider state");+ assert_eq!(persisted["lifecycle"], "closed");+ assert!(persisted["activeProviderTurnId"].is_null());- let interrupted = executor- .execute(&command("interrupt", 4, "turn.interrupt", json!({})))- .expect("interrupt reconciled provider turn");- assert_eq!(interrupted.result["status"], "interrupt_requested");- assert_eq!(call_count(&directory, "thread/resume"), 1);- assert_eq!(call_count(&directory, "thread/read"), 1);- assert_eq!(call_count(&directory, "turn/interrupt"), 1);+ drop(recovered);+ let resumes_before_closed_restore = call_count(&directory, "thread/resume");+ let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let events = closed+ .poll_events()+ .expect("closed reused-identity state remains readable without resuming Codex");+ assert!(events.iter().any(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "provider_turn_identity_reused"+ }));+ assert_eq!(+ call_count(&directory, "thread/resume"),+ resumes_before_closed_restore,+ "recovery must not resume provider work accepted under a reused identity"+ );- let mut terminal_seen = false;- for _ in 0..32 {- terminal_seen |= poll_and_ack(&mut executor)- .expect("poll reconciled interruption")- .iter()- .any(|event| event.event_type == "turn.interrupted");- if terminal_seen {- break;- }- }- assert!(terminal_seen);- executor.shutdown().expect("stop resumed provider process");+ closed.shutdown().expect("close fail-closed executor");fs::remove_dir_all(directory).expect("remove Codex integration-test directory");}#[test]-fn unacknowledged_provider_events_survive_executor_restart() {- let directory = temporary_directory("pending-event-recovery");- let config = provider_config(&directory, &["--emit-question"]);- let mut first = CodexCommandExecutor::new(&directory);+fn durable_recovery_closes_a_provider_that_reopens_a_settled_turn() {+ let directory = temporary_directory("durable-settled-turn-active-on-recovery");+ let config = provider_config(&directory, &[]);+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);first.execute(&command("prepare",@@ -1636,23 +2219,1685 @@ fn unacknowledged_provider_events_survive_executor_restart() {"turn",3,"turn.start",- json!({"text": "Emit a durable question."}),+ json!({"text": "Complete the durable turn."}),)).expect("start provider turn");-- let mut retained = None;+ let mut completed = false;for _ in 0..32 {- let events = first.poll_events().expect("poll provider events");- if events+ completed |= poll_and_ack(&mut first)+ .expect("poll completed provider turn").iter()- .any(|event| event.event_type == "runtime_request.created")- {- retained = Some(events);+ .any(|event| event.event_type == "turn.completed");+ if completed {break;}- first- .acknowledge_events(events.len())- .expect("acknowledge events before the question");+ }+ assert!(completed);+ first.shutdown().expect("stop first provider process");+ drop(first);++ // Contradict the durable terminal ledger with a resumed provider snapshot+ // that reports the exact settled identity as active again.+ fs::write(+ directory.join("fake-state.json"),+ serde_json::to_vec_pretty(&json!({+ "threadId": "codex-thread-1",+ "activeTurnId": "provider-turn-1",+ }))+ .unwrap(),+ )+ .expect("write contradictory fake provider state");++ let resumes_before_recovery = call_count(&directory, "thread/resume");+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let events = recovered+ .poll_events()+ .expect("fail-closed recovery remains observable");+ assert!(events.iter().any(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "provider_turn_identity_reused"+ && event.payload["providerTurnId"] == "provider-turn-1"+ }));+ assert_eq!(+ call_count(&directory, "thread/resume"),+ resumes_before_recovery + 1,+ );+ let persisted: Value = serde_json::from_slice(+ &fs::read(directory.join("codex-provider-state.json"))+ .expect("read fail-closed provider state"),+ )+ .expect("parse fail-closed provider state");+ assert_eq!(persisted["lifecycle"], "closed");+ assert!(persisted["activeProviderTurnId"].is_null());+ assert_eq!(persisted["completedTurnAuthoritative"], false);++ drop(recovered);+ let resumes_before_closed_restore = call_count(&directory, "thread/resume");+ let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ closed+ .poll_events()+ .expect("closed recovery state remains readable");+ assert_eq!(+ call_count(&directory, "thread/resume"),+ resumes_before_closed_restore,+ "closed recovery must not resume the contradictory provider turn again",+ );++ closed.shutdown().expect("close fail-closed executor");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn codex_rejects_a_two_turn_old_call_while_idle() {+ let directory = temporary_directory("delayed-tool-after-second-turn-completion");+ let config = provider_config(&directory, &["--delayed-tool-after-second-turn-completion"]);+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)+ .expect("start Codex with an authorized tool");++ for message in ["Complete turn one.", "Complete turn two."] {+ provider+ .start_turn(message, &config.cwd)+ .expect("start completed provider turn");+ wait_for_notification(&mut provider, "turn/completed");+ }+ assert_eq!(provider.active_provider_turn_id(), None);++ let warning = wait_for_notification(&mut provider, "warning");+ assert_eq!(warning["providerMethod"], "item/tool/call");+ assert_eq!(provider.active_provider_turn_id(), None);++ let _ = provider.shutdown();+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_delivers_a_terminal_before_rejecting_a_delayed_tool_call() {+ let directory = temporary_directory("durable-delayed-tool-after-terminal");+ let config = provider_config(&directory, &["--delayed-tool-after-failed-turn"]);+ let runner_config = durable_config(&directory);+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ executor+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ "completionContract": {+ "revision": "sha256:delayed-tool-contract",+ "criterionIds": ["criterion_delayed_tool"]+ },+ }),+ ))+ .unwrap();+ executor+ .execute(&command("open", 2, "session.open", json!({})))+ .unwrap();+ executor+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Fail before invoking a delayed tool."}),+ ))+ .unwrap();++ let mut observed = Vec::new();+ for _ in 0..32 {+ observed.extend(poll_and_ack(&mut executor).expect("poll durable provider events"));+ if observed+ .iter()+ .any(|event| event.event_type == "provider.notice.recorded")+ {+ break;+ }+ }+ let terminal = observed+ .iter()+ .position(|event| event.event_type == "run.terminal")+ .expect("the durable terminal is delivered");+ let rejection = observed+ .iter()+ .position(|event| event.event_type == "provider.notice.recorded")+ .expect("the delayed request is rejected non-fatally");+ assert!(terminal < rejection);++ executor.shutdown().unwrap();+ fs::remove_dir_all(directory).unwrap();+}++#[test]+fn codex_rejects_a_runtime_response_after_its_turn_terminates() {+ let directory = temporary_directory("delayed-question-response");+ let config = provider_config(&directory, &["--question-before-failed-turn"]);+ let mut provider = CodexProvider::start(&config, None).expect("start Codex provider");+ provider+ .start_turn("Ask and then fail.", &config.cwd)+ .expect("start provider turn");++ let mut request_id = None;+ let mut terminal_seen = false;+ for _ in 0..16 {+ match provider.poll().expect("poll question and terminal") {+ Some(CodexProviderEvent::RuntimeRequest {+ request_id: observed,+ ..+ }) => request_id = Some(observed),+ Some(CodexProviderEvent::Notification { method, .. }) if method == "turn/completed" => {+ terminal_seen = true;+ }+ _ => {}+ }+ if request_id.is_some() && terminal_seen {+ break;+ }+ }+ let request_id = request_id.expect("observe the runtime request before termination");+ assert!(terminal_seen);+ let error = provider+ .resolve_runtime_request(+ &request_id,+ &json!({+ "schema": "paperclip.question_response.v1",+ "answers": {"environment": {"selectedOptionIds": ["option-1"]}}+ }),+ )+ .expect_err("terminal requests must not remain resolvable");+ assert!(error.to_string().contains("no pending Codex request"));++ let _ = provider.shutdown();+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn reused_provider_question_ids_get_unique_controller_identities() {+ let directory = temporary_directory("reused-question-id");+ let config = provider_config(&directory, &["--emit-question", "--reuse-question-id"]);+ let mut provider = CodexProvider::start(&config, None).expect("start Codex provider");+ provider+ .start_turn("Ask twice with one provider request id.", &config.cwd)+ .expect("start provider turn");++ let first_request_id = (0..16)+ .find_map(|_| match provider.poll().expect("poll first question") {+ Some(CodexProviderEvent::RuntimeRequest { request_id, .. }) => Some(request_id),+ _ => None,+ })+ .expect("observe first runtime request");+ let response = json!({+ "schema": "paperclip.question_response.v1",+ "answers": {"environment": {"selectedOptionIds": ["option-1"]}}+ });+ provider+ .resolve_runtime_request(&first_request_id, &response)+ .expect("resolve first runtime request");+ let second_request_id = (0..16)+ .find_map(|_| match provider.poll().expect("poll second question") {+ Some(CodexProviderEvent::RuntimeRequest { request_id, .. }) => Some(request_id),+ _ => None,+ })+ .expect("observe second runtime request");++ assert_ne!(first_request_id, second_request_id);+ let stale = provider+ .resolve_runtime_request(&first_request_id, &response)+ .expect_err("the first controller identity cannot resolve the second question");+ assert!(stale.to_string().contains("no pending Codex request"));+ provider+ .resolve_runtime_request(&second_request_id, &response)+ .expect("resolve second runtime request");++ let _ = provider.shutdown();+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn codex_resume_advertises_the_same_authorized_tools() {+ let directory = temporary_directory("dynamic-tool-resume");+ let config = provider_config(&directory, &["--require-dynamic-tool"]);+ let mut provider =+ CodexProvider::start_with_tools(&config, [task_context_tool()], Some("codex-thread-1"))+ .expect("resume Codex with the run-scoped tool set");+ assert_eq!(provider.thread_id(), "codex-thread-1");+ provider.shutdown().expect("stop provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_routes_a_semantic_tool_result_back_to_codex() {+ let directory = temporary_directory("durable-dynamic-tool");+ let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]);+ let runner_config = durable_config(&directory);+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ executor+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare the durable Codex tool set");+ executor+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open the Codex session");+ executor+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Inspect the durable fake task."}),+ ))+ .expect("start the Codex turn");++ let mut semantic_input = None;+ for _ in 0..32 {+ let events = poll_and_ack(&mut executor).expect("poll semantic input");+ semantic_input = events+ .iter()+ .find(|event| event.event_type == "semantic_tool.input")+ .cloned()+ .or(semantic_input);+ if semantic_input.is_some() {+ break;+ }+ }+ let semantic_input = semantic_input.expect("durable semantic input is emitted");+ assert_eq!(+ semantic_input.payload["semantic_tool"]["correlation"]["runId"],+ "run-1"+ );+ assert_eq!(+ semantic_input.payload["semantic_tool"]["operationId"],+ "get_task_context"+ );++ let delivered = executor+ .execute(&command(+ "tool-result",+ 4,+ "semantic_tool.result",+ json!({+ "callId": "semantic-call-1",+ "operationId": "get_task_context",+ "result": {"ok": true, "task": {"id": "task-1"}},+ "isError": false,+ }),+ ))+ .expect("deliver the durable semantic result");+ assert_eq!(delivered.result["status"], "delivered");++ let mut result_seen = false;+ let mut terminal_seen = false;+ for _ in 0..32 {+ let events = poll_and_ack(&mut executor).expect("poll result and completion");+ result_seen |= events+ .iter()+ .any(|event| event.event_type == "semantic_tool.result");+ terminal_seen |= events+ .iter()+ .any(|event| event.event_type == "turn.completed");+ if result_seen && terminal_seen {+ break;+ }+ }+ assert!(result_seen);+ assert!(terminal_seen);+ executor.shutdown().expect("stop provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_replays_completed_results_without_mutating_the_event_queue() {+ let directory = temporary_directory("durable-completed-tool-replay");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--emit-tool-call",+ "--hold-turn",+ "--replay-completed-tool-call-count",+ "4",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ executor+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare the durable Codex tool set");+ executor+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open the Codex session");+ executor+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Replay the completed fake tool call."}),+ ))+ .expect("start the Codex turn");++ let mut input_seen = false;+ for _ in 0..32 {+ input_seen |= poll_and_ack(&mut executor)+ .expect("poll semantic input")+ .iter()+ .any(|event| event.event_type == "semantic_tool.input");+ if input_seen {+ break;+ }+ }+ assert!(input_seen, "durable semantic input is emitted");++ executor+ .execute(&command(+ "tool-result",+ 4,+ "semantic_tool.result",+ json!({+ "callId": "semantic-call-1",+ "operationId": "get_task_context",+ "result": {"ok": true, "task": {"id": "task-1"}},+ "isError": false,+ }),+ ))+ .expect("deliver the original durable semantic result");+ let result_events = poll_and_ack(&mut executor).expect("acknowledge semantic result");+ assert_eq!(+ result_events+ .iter()+ .filter(|event| event.event_type == "semantic_tool.result")+ .count(),+ 1+ );++ let state_path = directory.join("codex-provider-state.json");+ let state_before_replays = fs::read(&state_path).expect("read state before exact replays");+ let expected_response = r#"{"ok":true,"task":{"id":"task-1"}}"#;+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ while recorded_tool_responses(&directory).len() < 5 && std::time::Instant::now() < deadline {+ let events = poll_and_ack(&mut executor).expect("service completed tool replay");+ assert!(+ events.iter().all(|event| {+ event.event_type != "semantic_tool.input"+ && event.event_type != "semantic_tool.reconciled"+ && event.event_type != "semantic_tool.result"+ }),+ "an exact completed replay must not add semantic reconciliation events"+ );+ std::thread::sleep(std::time::Duration::from_millis(1));+ }++ let responses = recorded_tool_responses(&directory);+ assert_eq!(+ responses.len(),+ 5,+ "the original result and four replays return"+ );+ assert!(responses+ .iter()+ .all(|response| response == expected_response));+ assert_eq!(+ fs::read(&state_path).expect("read state after exact replays"),+ state_before_replays,+ "exact completed replays must not consume durable event capacity"+ );++ executor.shutdown().expect("stop provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_replays_pending_tool_calls_without_mutating_the_event_queue() {+ let directory = temporary_directory("durable-tool-recovery");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--emit-tool-call",+ "--emit-tool-call-on-resume",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare the recoverable tool bridge");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open the first provider");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Hold the tool call across recovery."}),+ ))+ .expect("start the first turn");+ let mut input_seen = false;+ for _ in 0..32 {+ input_seen |= poll_and_ack(&mut first)+ .expect("poll first tool input")+ .iter()+ .any(|event| event.event_type == "semantic_tool.input");+ if input_seen {+ break;+ }+ }+ assert!(input_seen);+ drop(first);++ let state_path = directory.join("codex-provider-state.json");+ let mut recovered = None;+ for replay in 0..4 {+ let before: Value = serde_json::from_slice(+ &fs::read(&state_path).expect("read state before provider recovery"),+ )+ .expect("parse state before provider recovery");+ let resume_count = call_count(&directory, "thread/resume");+ let mut next = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ for _ in 0..32 {+ let events = poll_and_ack(&mut next).expect("poll exact pending replay");+ assert!(events.iter().all(|event| {+ event.event_type != "semantic_tool.input"+ && event.event_type != "semantic_tool.reconciled"+ && event.event_type != "semantic_tool.result"+ }));+ if call_count(&directory, "thread/resume") > resume_count {+ break;+ }+ }+ assert!(+ call_count(&directory, "thread/resume") > resume_count,+ "recovery {replay} resumes the active provider turn"+ );+ for _ in 0..4 {+ let events = poll_and_ack(&mut next).expect("poll exact pending replay");+ assert!(events.iter().all(|event| {+ event.event_type != "semantic_tool.input"+ && event.event_type != "semantic_tool.reconciled"+ && event.event_type != "semantic_tool.result"+ }));+ }+ let after: Value = serde_json::from_slice(+ &fs::read(&state_path).expect("read state after pending replay"),+ )+ .expect("parse state after pending replay");+ assert_eq!(+ after["pendingEvents"], before["pendingEvents"],+ "exact pending replay {replay} must not append pending events"+ );+ assert_eq!(+ after["queuedEvents"], before["queuedEvents"],+ "exact pending replay {replay} must not append queued events"+ );+ assert_eq!(+ after["nextProviderEventSeq"], before["nextProviderEventSeq"],+ "exact pending replay {replay} must not consume durable event capacity"+ );+ recovered = Some(next);+ if replay < 3 {+ drop(recovered.take());+ }+ }+ let mut recovered = recovered.expect("retain the final recovered provider");+ recovered+ .execute(&command(+ "tool-result",+ 4,+ "semantic_tool.result",+ json!({+ "callId": "semantic-call-1",+ "operationId": "get_task_context",+ "result": {"ok": true, "task": {"id": "task-1"}},+ "isError": false,+ }),+ ))+ .expect("complete the replayed tool call");+ let events = poll_and_ack(&mut recovered).expect("poll the completed pending call");+ assert_eq!(+ events+ .iter()+ .filter(|event| event.event_type == "semantic_tool.result")+ .count(),+ 1+ );+ recovered.shutdown().expect("stop recovered provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_settles_pending_tools_when_recovery_finds_the_turn_ended() {+ let directory = temporary_directory("durable-tool-ended-offline");+ let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]);+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ "completionContract": {+ "revision": "sha256:offline-recovery-contract",+ "criterionIds": ["criterion_offline_recovery"]+ },+ }),+ ))+ .expect("prepare the recoverable tool bridge");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open the first provider");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "End while the runner is offline."}),+ ))+ .expect("start the first turn");+ let mut input_seen = false;+ for _ in 0..32 {+ input_seen |= poll_and_ack(&mut first)+ .expect("poll first tool input")+ .iter()+ .any(|event| event.event_type == "semantic_tool.input");+ if input_seen {+ break;+ }+ }+ assert!(input_seen);+ drop(first);++ fs::write(+ directory.join("fake-state.json"),+ serde_json::to_vec_pretty(&json!({+ "threadId": "codex-thread-1",+ "activeTurnId": null,+ }))+ .unwrap(),+ )+ .expect("record that the provider turn ended while offline");++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ recovered+ .execute(&command("snapshot", 4, "session.snapshot", json!({})))+ .expect("restore the provider session");+ let mut observed = Vec::new();+ for _ in 0..32 {+ observed.extend(+ poll_and_ack(&mut recovered)+ .expect("poll recovered settlement")+ .into_iter()+ .map(|event| event.event_type),+ );+ if observed.iter().any(|event| event == "run.terminal") {+ break;+ }+ }+ let semantic_result = observed+ .iter()+ .position(|event| event == "semantic_tool.result")+ .expect("recovery settles the pending semantic tool");+ let reconciled = observed+ .iter()+ .position(|event| event == "session.reconciled")+ .expect("recovery emits a reconciliation event");+ let terminal = observed+ .iter()+ .position(|event| event == "run.terminal")+ .expect("offline turn recovery terminates the run");+ assert!(semantic_result < reconciled);+ assert!(reconciled < terminal);+ assert!(recovered+ .execute(&command(+ "late-result",+ 5,+ "semantic_tool.result",+ json!({+ "callId": "semantic-call-1",+ "operationId": "get_task_context",+ "result": {"ok": true},+ "isError": false,+ }),+ ))+ .is_err());++ recovered.shutdown().expect("stop recovered provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_rejects_tool_catalog_drift_during_attach() {+ let directory = temporary_directory("durable-tool-attach-drift");+ let config = provider_config(&directory, &[]);+ let runner_config = durable_config(&directory);+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ executor+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare the durable tool catalog");++ let mut changed = task_context_tool_set();+ changed.operations[0].description = "Changed after recovery.".to_owned();+ changed.catalog_digest = authorized_tool_catalog_digest(&changed.operations).unwrap();+ let error = executor+ .execute(&command(+ "attach",+ 2,+ "run.attach",+ json!({"authorizedTools": changed}),+ ))+ .expect_err("attach must reject tool catalog drift");+ assert!(error.to_string().contains("tool contract changed"));++ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn durable_backend_settles_tools_before_a_natural_terminal_event() {+ let directory = temporary_directory("durable-tool-terminal");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--emit-tool-call",+ "--finish-turn-with-pending-tool",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ executor+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .unwrap();+ executor+ .execute(&command("open", 2, "session.open", json!({})))+ .unwrap();+ executor+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Terminate with a pending tool."}),+ ))+ .unwrap();++ let mut observed = Vec::new();+ for _ in 0..32 {+ let events = poll_and_ack(&mut executor).unwrap();+ observed.extend(events.into_iter().map(|event| event.event_type));+ if observed.iter().any(|event| event == "turn.completed") {+ break;+ }+ }+ let semantic_result = observed+ .iter()+ .position(|event| event == "semantic_tool.result")+ .expect("terminal settlement emits a failed semantic result");+ let terminal = observed+ .iter()+ .position(|event| event == "turn.completed")+ .expect("provider terminal event is emitted");+ assert!(semantic_result < terminal);+ assert!(executor+ .execute(&command(+ "late-result",+ 4,+ "semantic_tool.result",+ json!({+ "callId": "semantic-call-1",+ "operationId": "get_task_context",+ "result": {"ok": true},+ "isError": false,+ }),+ ))+ .is_err());++ executor.shutdown().unwrap();+ fs::remove_dir_all(directory).unwrap();+}++#[test]+fn durable_backend_resumes_the_active_thread_without_restarting_the_turn() {+ let directory = temporary_directory("resume");+ let config = provider_config(&directory, &["--hold-turn"]);+ let mut first = CodexCommandExecutor::new(&directory);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Hold this turn for recovery."}),+ ))+ .expect("start held provider turn");+ assert_eq!(call_count(&directory, "turn/start"), 1);+ first.shutdown().expect("stop first provider process");+ drop(first);++ let mut recovered = CodexCommandExecutor::new(&directory);+ let snapshot = recovered+ .execute(&command("snapshot", 4, "session.snapshot", json!({})))+ .expect("restore provider session");+ assert_eq!(snapshot.result["status"], "turn_active");+ assert_eq!(snapshot.result["activeProviderTurnId"], "provider-turn-1");+ assert_eq!(call_count(&directory, "turn/start"), 1);+ assert_eq!(call_count(&directory, "thread/resume"), 1);+ assert_eq!(call_count(&directory, "thread/read"), 1);++ recovered+ .execute(&command("interrupt", 5, "turn.interrupt", json!({})))+ .expect("interrupt recovered provider turn");+ let mut terminal_seen = false;+ for _ in 0..16 {+ let events = poll_and_ack(&mut recovered).expect("poll interrupted turn");+ terminal_seen |= events+ .iter()+ .any(|event| event.event_type == "turn.interrupted");+ if terminal_seen {+ break;+ }+ }+ assert!(terminal_seen);+ recovered+ .shutdown()+ .expect("stop recovered provider process");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn legacy_full_active_epoch_is_closed_on_recovery() {+ let directory = temporary_directory("legacy-full-active-recovery");+ let config = provider_config(&directory, &["--hold-turn"]);+ let mut first = CodexCommandExecutor::new(&directory);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Hold the legacy full-epoch turn."}),+ ))+ .expect("start held provider turn");+ first.shutdown().expect("stop first provider process");+ drop(first);++ let state_path = directory.join("codex-provider-state.json");+ let mut persisted: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read active provider state"))+ .expect("parse active provider state");+ let prior_generation = persisted["providerProcessGeneration"]+ .as_u64()+ .expect("provider generation is persisted");+ persisted["settledProviderTurnIds"] = Value::Array(+ (0..4_096)+ .map(|index| Value::String(format!("legacy-provider-turn-{index}")))+ .collect(),+ );+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())+ .expect("write legacy full active state");++ let mut recovered = CodexCommandExecutor::new(&directory);+ let events = recovered+ .poll_events()+ .expect("legacy full-epoch recovery remains observable");+ assert!(events.iter().any(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "legacy_provider_turn_epoch_ambiguous"+ }));+ let closed: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read closed legacy full state"))+ .expect("parse closed legacy full state");+ assert_eq!(closed["lifecycle"], "closed");+ assert!(closed["activeProviderTurnId"].is_null());+ assert_eq!(closed["ambiguousTurnStartPending"], false);+ assert_eq!(closed["completedTurnAuthoritative"], false);+ assert!(closed["settledProviderTurnFilter"]["words"]+ .as_array()+ .expect("legacy filter shape remains valid")+ .is_empty());+ assert_eq!(+ closed["settledProviderTurnIds"].as_array().unwrap().len(),+ 4_096+ );+ assert!(closed["providerProcessGeneration"].as_u64().unwrap() > prior_generation);+ assert!(recovered+ .execute(&command(+ "replacement-turn",+ 4,+ "turn.start",+ json!({"text": "Do not reopen closed legacy work."}),+ ))+ .expect_err("closed legacy full-epoch state rejects replacement work")+ .to_string()+ .contains("closed"));+ assert_eq!(call_count(&directory, "turn/start"), 1);++ recovered.shutdown().expect("close recovered executor");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn legacy_filtered_ambiguous_epoch_is_closed_on_recovery() {+ let directory = temporary_directory("legacy-filtered-ambiguous-recovery");+ let config = provider_config(&directory, &[]);+ let mut first = CodexCommandExecutor::new(&directory);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first.shutdown().expect("stop first provider process");+ drop(first);++ let state_path = directory.join("codex-provider-state.json");+ let mut persisted: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read idle provider state"))+ .expect("parse idle provider state");+ let mut legacy_words = vec![0_u64; 32_768];+ legacy_words[0] = 1;+ persisted["settledProviderTurnFilter"] = json!({"words": legacy_words});+ persisted["ambiguousTurnStartPending"] = Value::Bool(true);+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())+ .expect("write legacy-filtered ambiguous state");++ let mut recovered = CodexCommandExecutor::new(&directory);+ let events = recovered+ .poll_events()+ .expect("legacy ambiguous recovery remains observable");+ let diagnostic = events+ .iter()+ .find(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "legacy_provider_turn_epoch_ambiguous"+ })+ .expect("legacy ambiguous recovery emits a terminal diagnostic");+ assert_eq!(diagnostic.payload["ambiguousStartPending"], true);+ assert_eq!(diagnostic.payload["providerReportedActive"], false);+ let closed: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read closed legacy ambiguous state"))+ .expect("parse closed legacy ambiguous state");+ assert_eq!(closed["lifecycle"], "closed");+ assert!(closed["activeProviderTurnId"].is_null());+ assert_eq!(closed["ambiguousTurnStartPending"], false);+ assert!(!closed["settledProviderTurnFilter"]["words"]+ .as_array()+ .expect("legacy filter remains durable evidence")+ .is_empty());+ assert_eq!(call_count(&directory, "turn/start"), 0);++ recovered.shutdown().expect("close recovered executor");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn legacy_full_ended_epoch_reconciles_before_idle_rollover() {+ let directory = temporary_directory("legacy-full-ended-recovery");+ let config = provider_config(&directory, &["--hold-turn"]);+ let mut first = CodexCommandExecutor::new(&directory);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Let this legacy turn end while runnerd is away."}),+ ))+ .expect("start held provider turn");+ first.shutdown().expect("stop first provider process");+ drop(first);++ let state_path = directory.join("codex-provider-state.json");+ let mut persisted: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read active provider state"))+ .expect("parse active provider state");+ persisted["settledProviderTurnIds"] = Value::Array(+ (0..4_096)+ .map(|index| Value::String(format!("legacy-provider-turn-{index}")))+ .collect(),+ );+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())+ .expect("write full legacy provider epoch");+ fs::write(+ directory.join("fake-state.json"),+ serde_json::to_vec_pretty(&json!({"threadId": "codex-thread-1"})).unwrap(),+ )+ .expect("mark the provider turn ended while runnerd was away");++ let mut recovered = CodexCommandExecutor::new(&directory);+ let events = recovered+ .poll_events()+ .expect("reconcile an ended turn from a full legacy epoch");+ assert!(events.iter().any(|event| event.event_type == "turn.failed"));+ let reconciled: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read reconciled legacy state"))+ .expect("parse reconciled legacy state");+ assert_eq!(reconciled["lifecycle"], "session_open");+ assert!(reconciled["activeProviderTurnId"].is_null());+ assert_eq!(+ reconciled["settledProviderTurnIds"]+ .as_array()+ .unwrap()+ .len(),+ 4_096+ );++ recovered+ .execute(&command(+ "replacement-turn",+ 4,+ "turn.start",+ json!({"text": "Start only after the idle epoch rolls over."}),+ ))+ .expect("roll over the idle full epoch and start replacement work");+ let rolled: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read rolled legacy state"))+ .expect("parse rolled legacy state");+ assert!(rolled["settledProviderTurnIds"]+ .as_array()+ .expect("rolled exact identities remain an array")+ .is_empty());+ assert!(rolled["settledProviderTurnFilter"]["words"]+ .as_array()+ .expect("rolled filter remains valid")+ .is_empty());+ assert_eq!(call_count(&directory, "turn/start"), 2);++ recovered.shutdown().expect("stop rolled provider process");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn codex_resume_rejects_an_oversized_active_turn_identity() {+ let directory = temporary_directory("resume-oversized-active-turn-id");+ let config = provider_config(&directory, &[]);+ fs::write(+ directory.join("fake-state.json"),+ serde_json::to_vec_pretty(&json!({+ "threadId": "codex-thread-1",+ "activeTurnId": "x".repeat(241),+ }))+ .unwrap(),+ )+ .expect("write fake provider state with an oversized active identity");++ let error = CodexProvider::start(&config, Some("codex-thread-1"))+ .err()+ .expect("reject an oversized recovered provider turn identity");+ assert!(error.to_string().contains("invalid turn.id"));++ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn provider_exit_preserves_and_reconciles_the_active_turn() {+ let directory = temporary_directory("exit-active-turn");+ let config = provider_config(&directory, &["--exit-after-turn-start"]);+ let mut executor = CodexCommandExecutor::new(&directory);+ executor+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare Codex provider");+ executor+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ executor+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Keep the native turn active while the provider exits."}),+ ))+ .expect("start provider turn");++ let mut provider_exit_seen = false;+ for _ in 0..32 {+ provider_exit_seen |= poll_and_ack(&mut executor)+ .expect("poll provider exit")+ .iter()+ .any(|event| event.event_type == "session.failed");+ if provider_exit_seen {+ break;+ }+ }+ assert!(provider_exit_seen);+ let persisted: Value = serde_json::from_slice(+ &fs::read(directory.join("codex-provider-state.json"))+ .expect("read provider state after exit"),+ )+ .expect("parse provider state after exit");+ assert_eq!(persisted["lifecycle"], "provider_exited");+ assert_eq!(persisted["activeProviderTurnId"], "provider-turn-1");++ let interrupted = executor+ .execute(&command("interrupt", 4, "turn.interrupt", json!({})))+ .expect("interrupt reconciled provider turn");+ assert_eq!(interrupted.result["status"], "interrupt_requested");+ assert_eq!(call_count(&directory, "thread/resume"), 1);+ assert_eq!(call_count(&directory, "thread/read"), 1);+ assert_eq!(call_count(&directory, "turn/interrupt"), 1);++ let mut terminal_seen = false;+ for _ in 0..32 {+ terminal_seen |= poll_and_ack(&mut executor)+ .expect("poll reconciled interruption")+ .iter()+ .any(|event| event.event_type == "turn.interrupted");+ if terminal_seen {+ break;+ }+ }+ assert!(terminal_seen);+ executor.shutdown().expect("stop resumed provider process");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn receipt_limit_rejects_the_call_and_keeps_polling_when_interrupt_fails() {+ let directory = temporary_directory("receipt-limit-interrupt-failure");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--hold-turn",+ "--emit-tool-call-on-resume",+ "--fail-first-interrupt",+ "--accept-interrupt-without-terminal-once",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Hold this saturated turn for recovery."}),+ ))+ .expect("start held provider turn");+ drop(first);++ saturate_provider_tool_receipts(&directory);++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let events = poll_and_ack(&mut recovered)+ .expect("a failed first interrupt must not terminate durable provider polling");+ assert!(events.iter().any(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "semantic_tool_turn_receipt_limit"+ }));+ assert_eq!(call_count(&directory, "tool-response:failure"), 1);+ assert_eq!(call_count(&directory, "turn/interrupt"), 1);++ let mut settled = Vec::new();+ for _ in 0..4 {+ settled.extend(+ poll_and_ack(&mut recovered)+ .expect("polling must autonomously retry the durable receipt-limit interrupt"),+ );+ if settled+ .iter()+ .any(|event| event.event_type == "turn.interrupted")+ {+ break;+ }+ }+ assert!(+ settled+ .iter()+ .any(|event| event.event_type == "turn.interrupted"),+ "the retry must settle the receipt-exhausted turn"+ );+ assert_eq!(call_count(&directory, "turn/interrupt"), 3);++ recovered.shutdown().expect("stop recovered provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn receipt_limit_retry_preserves_a_turn_settled_during_provider_recovery() {+ let directory = temporary_directory("receipt-limit-recovered-settlement");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--hold-turn",+ "--emit-tool-call-on-resume",+ "--fail-first-interrupt",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Recover a provider-settled receipt-limited turn."}),+ ))+ .expect("start held provider turn");+ drop(first);++ saturate_provider_tool_receipts(&directory);++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let events = poll_and_ack(&mut recovered)+ .expect("the first failed interruption must leave a durable retry");+ assert!(events.iter().any(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "semantic_tool_turn_receipt_limit"+ }));+ assert_eq!(call_count(&directory, "turn/interrupt"), 1);++ // Lose the live transport while retaining its durable interruption retry,+ // then model the provider settling the turn before runnerd reconnects.+ recovered+ .shutdown()+ .expect("stop the provider before retry recovery");+ let fake_state_path = directory.join("fake-state.json");+ let mut fake_state: Value =+ serde_json::from_slice(&fs::read(&fake_state_path).expect("read fake provider state"))+ .expect("parse fake provider state");+ fake_state["activeTurnId"] = Value::Null;+ fs::write(+ &fake_state_path,+ serde_json::to_vec_pretty(&fake_state).unwrap(),+ )+ .expect("settle the fake provider turn before recovery");++ let events = poll_and_ack(&mut recovered)+ .expect("an already-settled retry must preserve the recovered terminal state");+ assert!(events.iter().any(|event| {+ event.event_type == "session.reconciled" && event.payload["activeProviderTurnId"].is_null()+ }));+ assert_eq!(+ call_count(&directory, "turn/interrupt"),+ 1,+ "an already-settled recovered turn must not receive another interrupt RPC"+ );++ let persisted: Value = serde_json::from_slice(+ &fs::read(directory.join("codex-provider-state.json"))+ .expect("read reconciled provider state"),+ )+ .expect("parse reconciled provider state");+ assert!(persisted["activeProviderTurnId"].is_null());+ assert_eq!(persisted["receiptLimitInterruptPending"], false);+ assert_eq!(persisted["receiptLimitInterruptAccepted"], false);+ assert_eq!(persisted["receiptLimitInterruptAttempts"], 0);+ assert!(persisted["receiptLimitInterruptDeadlineUnixMs"].is_null());++ recovered.shutdown().expect("stop recovered provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn receipt_limit_accepts_a_terminal_after_the_initial_interrupt_deadline() {+ let directory = temporary_directory("receipt-limit-delayed-terminal");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--hold-turn",+ "--emit-tool-call-on-resume",+ "--interrupt-terminal-delay-ms",+ "2100",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Accept a delayed authoritative interruption."}),+ ))+ .expect("start held provider turn");+ drop(first);+ saturate_provider_tool_receipts(&directory);++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let mut emitted = Vec::new();+ for _ in 0..4_096 {+ emitted.extend(+ poll_and_ack(&mut recovered)+ .expect("a delayed accepted interrupt must remain durably pollable"),+ );+ if emitted+ .iter()+ .any(|event| event.event_type == "turn.interrupted")+ {+ break;+ }+ }+ assert!(+ emitted+ .iter()+ .any(|event| event.event_type == "turn.interrupted"),+ "the delayed provider terminal must remain authoritative after two seconds"+ );+ assert!(!emitted.iter().any(|event| {+ event.event_type == "turn.interrupted"+ && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_deadline"+ }));+ assert!(+ !emitted.iter().any(|event| {+ event.event_type == "turn.failed"+ && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_unconfirmed"+ }),+ "fast provider polls must not replace a delayed terminal with fallback failure"+ );++ recovered.shutdown().expect("stop recovered provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn receipt_limit_polls_an_authoritative_terminal_with_unacknowledged_events() {+ let directory = temporary_directory("receipt-limit-terminal-with-unacked-events");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--hold-turn",+ "--emit-tool-call-on-resume",+ "--accept-interrupt-without-terminal-once",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Retain the receipt-limit diagnostic until terminal polling."}),+ ))+ .expect("start held provider turn");+ drop(first);+ saturate_provider_tool_receipts(&directory);++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let pending = recovered+ .poll_events()+ .expect("begin the durable receipt-limit stop");+ assert!(pending.iter().any(|event| {+ event.event_type == "harness.diagnostic"+ && event.payload["code"] == "semantic_tool_turn_receipt_limit"+ }));+ assert!(!pending+ .iter()+ .any(|event| event.event_type == "turn.interrupted"));++ let terminal = recovered+ .poll_events()+ .expect("poll the provider terminal before old events are acknowledged");+ assert!(terminal.iter().any(|event| {+ event.event_type == "turn.interrupted" && event.payload.get("code").is_none()+ }));+ assert!(!terminal.iter().any(|event| {+ event.event_type == "turn.interrupted"+ && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_deadline"+ }));+ recovered+ .acknowledge_events(terminal.len())+ .expect("acknowledge the diagnostic and authoritative terminal together");++ recovered.shutdown().expect("stop recovered provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn receipt_limit_polling_bounds_and_rejects_runtime_request_floods() {+ let directory = temporary_directory("receipt-limit-runtime-request-flood");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--hold-turn",+ "--emit-tool-call-on-resume",+ "--accept-interrupt-without-terminal",+ "--flood-runtime-requests-on-interrupt",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Bound runtime requests while stopping this turn."}),+ ))+ .expect("start held provider turn");+ drop(first);+ saturate_provider_tool_receipts(&directory);++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);+ let mut rejected_at_capacity = false;+ while !rejected_at_capacity && std::time::Instant::now() < deadline {+ let events = recovered+ .poll_events()+ .expect("runtime-request cleanup remains bounded across repeated polls");+ assert!(events.len() <= 128);+ rejected_at_capacity = events.iter().any(|event| {+ event.event_type == "provider.notice.recorded"+ && event.payload["summary"]+ == "rejected a Codex runtime request at the bounded pending-input limit"+ });+ recovered+ .acknowledge_events(events.len())+ .expect("advance the bounded durable event prefix");+ if events.is_empty() {+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ }+ assert!(+ rejected_at_capacity,+ "production records that requests above the pending count/byte envelope were rejected"+ );++ recovered.shutdown().expect("stop recovered provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn receipt_limit_synthesizes_interrupted_after_an_accepted_terminal_deadline() {+ let directory = temporary_directory("receipt-limit-missing-terminal");+ let config = provider_config(+ &directory,+ &[+ "--require-dynamic-tool",+ "--hold-turn",+ "--emit-tool-call-on-resume",+ "--accept-interrupt-without-terminal",+ ],+ );+ let runner_config = durable_config(&directory);+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({+ "provider": config,+ "authorizedTools": task_context_tool_set(),+ }),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Bound a missing interrupt terminal."}),+ ))+ .expect("start held provider turn");+ drop(first);+ saturate_provider_tool_receipts(&directory);++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ let mut emitted = Vec::new();+ for _ in 0..8 {+ emitted.extend(+ poll_and_ack(&mut recovered)+ .expect("receipt-limit fallback must remain durably pollable"),+ );+ if call_count(&directory, "turn/interrupt") == 3 {+ break;+ }+ }+ assert!(!emitted+ .iter()+ .any(|event| event.event_type == "turn.failed"));+ recovered+ .shutdown()+ .expect("pause the provider before expiring its durable deadline");+ drop(recovered);++ let state_path = directory.join("codex-provider-state.json");+ let mut persisted: Value =+ serde_json::from_slice(&fs::read(&state_path).expect("read pending receipt-limit state"))+ .expect("parse pending receipt-limit state");+ persisted["receiptLimitInterruptDeadlineUnixMs"] = json!(1);+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())+ .expect("expire the durable receipt-limit deadline");++ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);+ for _ in 0..4 {+ emitted.extend(+ poll_and_ack(&mut recovered)+ .expect("expired receipt-limit fallback must remain durably pollable"),+ );+ if emitted+ .iter()+ .any(|event| event.event_type == "turn.interrupted")+ {+ break;+ }+ }+ let interrupted = emitted+ .iter()+ .find(|event| event.event_type == "turn.interrupted")+ .expect("the bounded accepted path must synthesize an interrupted turn");+ assert_eq!(+ interrupted.payload["code"],+ "semantic_tool_turn_receipt_limit_interrupt_deadline"+ );+ assert_eq!(interrupted.payload["interruptAccepted"], true);+ assert_eq!(interrupted.payload["providerTerminalObserved"], false);+ assert!(!emitted+ .iter()+ .any(|event| event.event_type == "turn.failed"));+ assert_eq!(call_count(&directory, "turn/interrupt"), 3);++ let persisted: Value = serde_json::from_slice(+ &fs::read(directory.join("codex-provider-state.json"))+ .expect("read bounded receipt-limit state"),+ )+ .expect("parse bounded receipt-limit state");+ assert_eq!(persisted["lifecycle"], "provider_exited");+ assert!(persisted["activeProviderTurnId"].is_null());+ assert_eq!(persisted["receiptLimitInterruptPending"], false);+ assert_eq!(persisted["receiptLimitInterruptAttempts"], 0);+ assert!(persisted["receiptLimitInterruptDeadlineUnixMs"].is_null());++ recovered+ .shutdown()+ .expect("bounded fallback stopped provider");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}++#[test]+fn unacknowledged_provider_events_survive_executor_restart() {+ let directory = temporary_directory("pending-event-recovery");+ let config = provider_config(&directory, &["--emit-question"]);+ let mut first = CodexCommandExecutor::new(&directory);+ first+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare Codex provider");+ first+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open Codex session");+ first+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Emit a durable question."}),+ ))+ .expect("start provider turn");++ let mut retained = None;+ for _ in 0..32 {+ let events = first.poll_events().expect("poll provider events");+ if events+ .iter()+ .any(|event| event.event_type == "runtime_request.created")+ {+ retained = Some(events);+ break;+ }+ first+ .acknowledge_events(events.len())+ .expect("acknowledge events before the question");}let retained = retained.expect("observe a durable runtime request");first.shutdown().expect("stop first provider process");@@ -1672,6 +3917,70 @@ fn unacknowledged_provider_events_survive_executor_restart() {fs::remove_dir_all(directory).expect("remove Codex integration-test directory");}+#[test]+fn durable_backend_rejects_a_runtime_response_after_terminal_settlement() {+ let directory = temporary_directory("durable-delayed-question-response");+ let config = provider_config(&directory, &["--question-before-failed-turn"]);+ let mut executor = CodexCommandExecutor::new(&directory);+ executor+ .execute(&command(+ "prepare",+ 1,+ "run.prepare",+ json!({"provider": config}),+ ))+ .expect("prepare provider");+ executor+ .execute(&command("open", 2, "session.open", json!({})))+ .expect("open provider session");+ executor+ .execute(&command(+ "turn",+ 3,+ "turn.start",+ json!({"text": "Ask and then fail."}),+ ))+ .expect("start provider turn");++ let mut request_id = None;+ let mut terminal_seen = false;+ for _ in 0..16 {+ for event in poll_and_ack(&mut executor).expect("poll question and terminal") {+ if event.event_type == "runtime_request.created" {+ request_id = event+ .payload+ .pointer("/request/requestId")+ .and_then(Value::as_str)+ .map(str::to_owned);+ }+ terminal_seen |= event.event_type == "turn.failed";+ }+ if request_id.is_some() && terminal_seen {+ break;+ }+ }+ let request_id = request_id.expect("observe the durable runtime request id");+ assert!(terminal_seen);+ let error = executor+ .execute(&command(+ "resolve",+ 4,+ "request.resolve",+ json!({+ "requestId": request_id,+ "response": {+ "schema": "paperclip.question_response.v1",+ "answers": {"environment": {"selectedOptionIds": ["option-1"]}}+ }+ }),+ ))+ .expect_err("terminal runtime requests must fail closed");+ assert!(error.to_string().contains("outside an active turn"));++ executor.shutdown().expect("stop provider process");+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");+}+#[test]fn structured_question_round_trips_through_the_normalized_backend() {let directory = temporary_directory("questions");@@ -1700,6 +4009,7 @@ fn structured_question_round_trips_through_the_normalized_backend() {assert_eq!(started.events[0].0, "turn.accepted");let mut question_set = None;+ let mut request_id = None;let mut provider_started_events = 0;for _ in 0..16 {for event in poll_and_ack(&mut executor).expect("poll question") {@@ -1710,6 +4020,11 @@ fn structured_question_round_trips_through_the_normalized_backend() {"paperclip.runtime_request.v2");question_set = event.payload.pointer("/request/input").cloned();+ request_id = event+ .payload+ .pointer("/request/requestId")+ .and_then(Value::as_str)+ .map(str::to_owned);}}if question_set.is_some() {@@ -1717,6 +4032,7 @@ fn structured_question_round_trips_through_the_normalized_backend() {}}let question_set = question_set.expect("normalized question set is emitted");+ let request_id = request_id.expect("normalized request id is emitted");assert_eq!(provider_started_events, 1);assert_eq!(question_set["schema"], "paperclip.question_set.v1");assert_eq!(@@ -1730,7 +4046,7 @@ fn structured_question_round_trips_through_the_normalized_backend() {4,"request.resolve",json!({- "requestId": "runtime-request-1",+ "requestId": request_id,"response": {"schema": "paperclip.question_response.v1","answers": {"environment": {"selectedOptionIds": ["option-1"]}}diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rsindex 17303b3bf9..2504f383ed 100644--- a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs+++ b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs@@ -73,6 +73,13 @@ fn rejects_unknown_tools_and_conflicting_duplicate_results() {};bridge.apply_result(result.clone()).unwrap();bridge.apply_result(result).unwrap();+ assert!(bridge+ .replay_result("call-1", "get_task_context", &json!({}))+ .unwrap()+ .is_some());+ assert!(bridge+ .replay_result("call-1", "get_task_context", &json!({"changed": true}))+ .is_err());assert!(bridge.apply_result(ToolResult {call_id: "call-1".to_owned(),@@ -108,6 +115,29 @@ fn catalog_digest_matches_the_typescript_canonical_json_contract() {);}+#[test]+fn catalog_digest_normalizes_json_numbers_like_javascript() {+ let operation = AuthorizedTool {+ operation_id: "get_task_context".into(),+ version: 1,+ description: "Read the active task context.".into(),+ input_schema: json!({+ "type": "object",+ "properties": {+ "limit": { "type": "number", "default": 1.0 },+ "epsilon": { "type": "number", "default": 1e-6 },+ },+ }),+ response_schema: json!({ "type": "object" }),+ };+ let digest = authorized_tool_catalog_digest(&[operation]).unwrap();++ assert_eq!(+ digest,+ "sha256:1c93693d9b5b48b46c83cd1c11d1ea329774f1b9b0ae741197cb2b8e992c4b8d"+ );+}+#[test]fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() {let mut set = tools("sha256:catalog-a");@@ -120,7 +150,7 @@ fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() {let mut bridge = ProviderToolBridge::default();set.catalog_digest = digest('a');set.catalog_digest = authorized_tool_catalog_digest(&set.operations).unwrap();- bridge.prepare(set).unwrap();+ bridge.prepare(set.clone()).unwrap();bridge.begin_call("call-1".into(), "get_task_context".into(), json!({})).unwrap();@@ -139,6 +169,25 @@ fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() {}).unwrap();assert_eq!(bridge.pending_calls().count(), 0);++ let mut second = ProviderToolBridge::default();+ second.prepare(set).unwrap();+ second+ .begin_call("call-2".into(), "get_task_context".into(), json!({}))+ .unwrap();+ second+ .apply_result(ToolResult {+ call_id: "call-2".into(),+ operation_id: "get_task_context".into(),+ result: json!({+ "ok": true,+ "operationId": "get_task_context",+ "callId": "call-2",+ "value": { "value": "accepted" }+ }),+ is_error: false,+ })+ .unwrap();}#[test]@@ -159,6 +208,24 @@ fn rejects_noncanonical_digests_and_oversized_contract_values() {json!({ "value": "x".repeat(1024 * 1024) }),).is_err());++ let retained = json!({"value": "x".repeat(700 * 1024)});+ for index in 0..5 {+ bridge+ .begin_call(+ format!("call-{index}"),+ "get_task_context".into(),+ retained.clone(),+ )+ .unwrap();+ }+ assert!(bridge+ .begin_call(+ "call-over-aggregate-limit".into(),+ "get_task_context".into(),+ retained,+ )+ .is_err());}#[test]@@ -203,12 +270,214 @@ fn recovery_preserves_completed_call_replay_identities() {let encoded = serde_json::to_string(&bridge).unwrap();let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();+ recovered.validate_recovered().unwrap();recovered.attach_existing_run().unwrap();assert!(recovered.begin_call("call-1".into(), "get_task_context".into(), json!({})).is_err());}+#[test]+fn recovered_bridge_rejects_tampered_authorization_state() {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(tools("computed")).unwrap();+ let mut encoded = serde_json::to_value(&bridge).unwrap();+ encoded["authorized"]["get_task_context"]["description"] = json!("Tampered");+ let recovered: ProviderToolBridge = serde_json::from_value(encoded).unwrap();+ assert!(recovered.validate_recovered().is_err());+}++#[test]+fn cancellation_completes_pending_calls_and_rejects_late_results() {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(tools("computed")).unwrap();+ bridge+ .begin_call("call-1".into(), "get_task_context".into(), json!({}))+ .unwrap();+ let cancelled = bridge+ .cancel_pending_calls("provider_turn_stopped")+ .unwrap();+ assert_eq!(cancelled.len(), 1);+ assert!(cancelled[0].is_error);+ assert_eq!(bridge.pending_calls().count(), 0);+ assert!(bridge+ .apply_result(ToolResult {+ call_id: "call-1".into(),+ operation_id: "get_task_context".into(),+ result: json!({"ok": true}),+ is_error: false,+ })+ .is_err());+}++#[test]+fn turn_settlement_releases_value_capacity_without_reusing_call_ids() {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(tools("computed")).unwrap();+ bridge+ .begin_call("call-1".into(), "get_task_context".into(), json!({}))+ .unwrap();+ bridge+ .apply_result(ToolResult {+ call_id: "call-1".into(),+ operation_id: "get_task_context".into(),+ result: json!({"ok": true}),+ is_error: false,+ })+ .unwrap();++ assert!(bridge+ .settle_turn("provider_turn_terminated")+ .unwrap()+ .is_empty());+ assert!(bridge+ .begin_call("call-1".into(), "get_task_context".into(), json!({}))+ .is_err());+ bridge+ .begin_call("call-2".into(), "get_task_context".into(), json!({}))+ .expect("a new turn can use a fresh call id after releasing exact values");+}++#[test]+fn completed_receipts_are_exact_until_the_controlled_turn_limit() {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(tools("computed")).unwrap();+ for index in 0..4_096 {+ let call_id = format!("call-{index}");+ bridge+ .begin_call(call_id.clone(), "get_task_context".into(), json!({}))+ .expect("a completed call must release concurrent capacity");+ bridge+ .apply_result(ToolResult {+ call_id,+ operation_id: "get_task_context".into(),+ result: json!({"ok": true}),+ is_error: false,+ })+ .unwrap();+ }++ let error = bridge+ .begin_call(+ "call-after-limit".into(),+ "get_task_context".into(),+ json!({}),+ )+ .expect_err("the bounded exact receipt ledger must stop the active turn");+ assert!(error.is_active_turn_receipt_limit());+ assert!(bridge+ .replay_result("call-0", "get_task_context", &json!({}))+ .unwrap()+ .is_some());+ assert!(bridge+ .replay_result("call-4095", "get_task_context", &json!({}))+ .unwrap()+ .is_some());++ let recovered: ProviderToolBridge =+ serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();+ recovered.validate_recovered().unwrap();+}++#[test]+fn turn_settlement_cannot_be_blocked_by_completed_value_pressure() {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(tools("computed")).unwrap();+ let large = json!({"value": "x".repeat(700 * 1024)});+ for index in 0..5 {+ bridge+ .begin_call(+ format!("call-{index}"),+ "get_task_context".into(),+ large.clone(),+ )+ .unwrap();+ }+ for index in 0..4 {+ bridge+ .apply_result(ToolResult {+ call_id: format!("call-{index}"),+ operation_id: "get_task_context".into(),+ result: large.clone(),+ is_error: false,+ })+ .unwrap();+ }++ let settled = bridge.settle_turn("provider_turn_terminated").unwrap();+ assert_eq!(settled.len(), 1);+ assert!(settled[0].is_error);+ assert!(bridge+ .begin_call("call-0".into(), "get_task_context".into(), json!({}))+ .is_err());+ bridge+ .begin_call(+ "call-after-settlement".into(),+ "get_task_context".into(),+ json!({}),+ )+ .expect("settlement releases prior turn value retention");+}++#[test]+fn recovered_turn_preserves_exact_results_at_the_value_boundary() {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(tools("computed")).unwrap();+ let large = json!({"value": "x".repeat(700 * 1024)});+ for index in 0..5 {+ bridge+ .begin_call(+ format!("call-{index}"),+ "get_task_context".into(),+ large.clone(),+ )+ .unwrap();+ }+ for index in 0..5 {+ bridge+ .apply_result(ToolResult {+ call_id: format!("call-{index}"),+ operation_id: "get_task_context".into(),+ result: large.clone(),+ is_error: false,+ })+ .unwrap();+ }++ let error = bridge+ .begin_call("call-next".into(), "get_task_context".into(), large.clone())+ .expect_err("exact replay values must not be discarded for later work");+ assert!(error.is_active_turn_receipt_limit());+ assert_eq!(+ bridge+ .replay_result("call-0", "get_task_context", &large)+ .unwrap()+ .unwrap()+ .result,+ large,+ );+ bridge+ .apply_result(ToolResult {+ call_id: "call-0".into(),+ operation_id: "get_task_context".into(),+ result: large.clone(),+ is_error: false,+ })+ .expect("a matching exact result receipt remains idempotent");+ assert!(bridge+ .apply_result(ToolResult {+ call_id: "call-0".into(),+ operation_id: "get_task_context".into(),+ result: json!({"value": "changed"}),+ is_error: false,+ })+ .is_err());++ let recovered: ProviderToolBridge =+ serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();+ recovered.validate_recovered().unwrap();+}+#[test]fn recovery_preserves_pending_calls_for_the_existing_run() {let mut bridge = ProviderToolBridge::default();@@ -252,7 +521,7 @@ fn recovery_rejects_nonempty_state_without_a_catalog_digest() {let error = recovered.attach_existing_run().expect_err("nonempty recovered state must remain bound to a catalog digest");- assert!(error.to_string().contains("omit the catalog digest"));+ assert!(error.to_string().contains("omitted its catalog identity"));}#[test]@@ -269,8 +538,8 @@ fn recovery_rejects_tampered_authorization_catalog_bindings() {let mut recovered: ProviderToolBridge = serde_json::from_value(changed_contract).unwrap();let error = recovered.attach_existing_run()- .expect_err("recovery must recompute the catalog digest");- assert!(error.to_string().contains("catalog digest"));+ .expect_err("recovery must reconstruct the authorized catalog projection");+ assert!(error.to_string().contains("changed its authorized catalog"));let mut changed_map_key = encoded;let authorized = changed_map_key["authorized"].as_object_mut().unwrap();@@ -280,7 +549,7 @@ fn recovery_rejects_tampered_authorization_catalog_bindings() {let error = recovered.attach_existing_run().expect_err("recovery must bind map keys to declared operation identities");- assert!(error.to_string().contains("identities are inconsistent"));+ assert!(error.to_string().contains("changed its authorized catalog"));}#[test]@@ -326,18 +595,21 @@ fn recovery_rejects_tampered_retained_result_contracts() {let completed = serde_json::to_value(&bridge).unwrap();let mut unauthorized = completed.clone();- unauthorized["completed"]["call-1"]["operationId"] = json!("delete_company");- let mut recovered: ProviderToolBridge = serde_json::from_value(unauthorized).unwrap();- assert!(recovered.attach_existing_run().is_err());+ unauthorized["completed"]["call-1"]["result"]["operationId"] = json!("delete_company");+ let error = serde_json::from_value::<ProviderToolBridge>(unauthorized)+ .expect_err("durable decoding must reject mismatched call and result identities");+ assert!(error+ .to_string()+ .contains("retained provider tool receipt identity is inconsistent"));let mut invalid_output = completed;- invalid_output["completed"]["call-1"]["result"] = json!(["not", "an", "object"]);+ invalid_output["completed"]["call-1"]["result"]["result"] = json!(["not", "an", "object"]);let mut recovered: ProviderToolBridge = serde_json::from_value(invalid_output).unwrap();assert!(recovered.attach_existing_run().is_err());- bridge.settle_turn().unwrap();+ bridge.settle_turn("provider_turn_terminated").unwrap();let mut invalid_settled_output = serde_json::to_value(&bridge).unwrap();- invalid_settled_output["settledResults"]["call-1"]["result"] = json!("invalid");+ invalid_settled_output["settledResults"]["call-1"]["result"]["result"] = json!("invalid");let mut recovered: ProviderToolBridge = serde_json::from_value(invalid_settled_output).unwrap();assert!(recovered.attach_existing_run().is_err());}@@ -389,7 +661,7 @@ fn settles_completed_receipts_before_the_next_turn() {assert!(bridge.begin_call("call-next".into(), "get_task_context".into(), json!({})).is_err());- bridge.settle_turn().unwrap();+ bridge.settle_turn("provider_turn_terminated").unwrap();assert!(bridge.begin_call("call-next".into(), "get_task_context".into(), json!({})).is_ok());@@ -410,7 +682,7 @@ fn settlement_preserves_call_ids_for_the_durable_run() {is_error: false,}).unwrap();- bridge.settle_turn().unwrap();+ bridge.settle_turn("provider_turn_terminated").unwrap();let replay = ToolResult {call_id: "call-1".into(),@@ -456,7 +728,7 @@ fn settlement_preserves_call_ids_for_the_durable_run() {}#[test]-fn reserves_identity_capacity_before_accepting_a_call() {+fn exact_identity_overflow_saturates_the_durable_run() {let mut bridge = ProviderToolBridge::default();bridge.prepare(tools("computed")).unwrap();@@ -472,27 +744,147 @@ fn reserves_identity_capacity_before_accepting_a_call() {bridge.begin_call("last-call".into(), "get_task_context".into(), json!({})).unwrap();- bridge- .apply_result(ToolResult {- call_id: "last-call".into(),- operation_id: "get_task_context".into(),- result: json!({"ok": true}),- is_error: false,- })- .unwrap();- bridge.settle_turn().unwrap();- assert!(bridge+ let overflow = bridge+ .begin_call("overflow".into(), "get_task_context".into(), json!({}))+ .expect_err("the pending call reserves the final exact identity slot");+ assert!(overflow.is_active_turn_receipt_limit());++ let encoded = serde_json::to_string(&bridge).unwrap();+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();+ recovered.attach_existing_run().unwrap();+ assert!(recovered.durable_run_receipt_limit_reached());+ recovered+ .settle_turn("provider_turn_terminated")+ .expect("the controlled turn stop retains replay protection");+ assert!(recovered.durable_run_receipt_limit_reached());+ assert!(recovered.has_completed_call("settled-0"));+ assert!(recovered.has_completed_call("last-call"));+ let stopped_turn_receipt = recovered+ .replay_result("last-call", "get_task_context", &json!({}))+ .unwrap()+ .expect("the call admitted before exhaustion retains an exact terminal receipt");+ assert!(stopped_turn_receipt.is_error);+ assert_eq!(+ stopped_turn_receipt.result["error"]["code"],+ "provider_turn_terminated"+ );+ assert!(recovered+ .begin_call("settled-0".into(), "get_task_context".into(), json!({}))+ .is_err());+ assert!(recovered.begin_call("overflow".into(), "get_task_context".into(), json!({})).is_err());- assert!(bridge.settle_turn().is_ok());+ recovered.prepare_turn().unwrap();+ assert!(recovered+ .replay_result("last-call", "get_task_context", &json!({}))+ .unwrap()+ .is_none());+ assert!(recovered.has_completed_call("last-call"));+ assert!(recovered.has_completed_call("settled-0"));+ assert!(recovered.durable_run_receipt_limit_reached());+ let saturation = recovered+ .begin_call("next-call".into(), "get_task_context".into(), json!({}))+ .expect_err("a turn boundary must not reopen the saturated durable run");+ assert!(saturation.is_active_turn_receipt_limit());++ let encoded = serde_json::to_string(&recovered).unwrap();+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();+ recovered.attach_existing_run().unwrap();+ assert!(recovered.has_completed_call("settled-0"));+ assert!(recovered.has_completed_call("last-call"));+ assert!(recovered.durable_run_receipt_limit_reached());++ recovered.attach_run(tools("computed")).unwrap();+ assert!(!recovered.durable_run_receipt_limit_reached());+ recovered+ .begin_call("overflow".into(), "get_task_context".into(), json!({}))+ .expect("a new durable run receives a fresh tool-call identity ledger");+}++#[test]+fn settled_result_byte_exhaustion_recovers_after_turn_cleanup() {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(tools("computed")).unwrap();+ let large_result = json!({"value": "x".repeat(750 * 1024)});++ for index in 0..10 {+ let call_id = format!("large-settled-{index}");+ bridge+ .begin_call(call_id.clone(), "get_task_context".into(), json!({}))+ .unwrap();+ bridge+ .apply_result(ToolResult {+ call_id,+ operation_id: "get_task_context".into(),+ result: large_result.clone(),+ is_error: false,+ })+ .unwrap();+ }+ bridge.settle_turn("provider_turn_terminated").unwrap();++ let error = bridge+ .begin_call(+ "over-byte-limit".into(),+ "get_task_context".into(),+ json!({}),+ )+ .expect_err("settled byte exhaustion must stop the durable run");+ assert!(error.is_active_turn_receipt_limit());++ let encoded = serde_json::to_string(&bridge).unwrap();+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();+ recovered.attach_existing_run().unwrap();+ assert!(recovered.durable_run_receipt_limit_reached());+ assert_eq!(+ recovered+ .replay_result("large-settled-0", "get_task_context", &json!({}))+ .unwrap()+ .unwrap()+ .result,+ large_result+ );+ assert!(recovered+ .begin_call(+ "over-byte-limit".into(),+ "get_task_context".into(),+ json!({})+ )+ .is_err());+ recovered.prepare_turn().unwrap();+ assert!(recovered+ .replay_result("large-settled-0", "get_task_context", &json!({}))+ .unwrap()+ .is_none());+ assert!(recovered.has_completed_call("large-settled-0"));+ assert!(!recovered.durable_run_receipt_limit_reached());+ recovered+ .begin_call(+ "after-turn-boundary".into(),+ "get_task_context".into(),+ json!({}),+ )+ .expect("releasing bulky results clears transient byte pressure");+ recovered+ .settle_turn("provider_turn_terminated")+ .expect("the admitted next-turn call remains settleable");++ recovered.attach_run(tools("computed")).unwrap();+ recovered+ .begin_call(+ "over-byte-limit".into(),+ "get_task_context".into(),+ json!({}),+ )+ .expect("a new durable run resets the settled result budget");}#[test]fn reserves_settled_result_bytes_before_accepting_a_call() {let mut bridge = ProviderToolBridge::default();bridge.prepare(tools("computed")).unwrap();- let large_result = json!({"value": "x".repeat(900 * 1024)});+ let large_result = json!({"value": "x".repeat(700 * 1024)});let mut completed = 0;for index in 0..20 {@@ -523,7 +915,7 @@ fn reserves_settled_result_bytes_before_accepting_a_call() {).is_err());bridge- .settle_turn()+ .settle_turn("provider_turn_terminated").expect("settlement cannot strand results whose bytes were reserved at admission");assert_eq!(bridge@@ -534,7 +926,7 @@ fn reserves_settled_result_bytes_before_accepting_a_call() {is_error: false,}).unwrap(),- json!({"value": "x".repeat(900 * 1024)})+ json!({"value": "x".repeat(700 * 1024)}));}@@ -544,15 +936,22 @@ fn recovery_rejects_an_oversized_settled_result_envelope() {bridge.prepare(tools("computed")).unwrap();let mut encoded = serde_json::to_value(&bridge).unwrap();let settled = encoded["settledResults"].as_object_mut().unwrap();- for index in 0..10 {+ for index in 0..12 {let call_id = format!("recovered-large-{index}");settled.insert(call_id.clone(),json!({- "callId": call_id,- "operationId": "get_task_context",- "result": {"value": "x".repeat(900 * 1024)},- "isError": false+ "call": {+ "callId": call_id,+ "operationId": "get_task_context",+ "input": {}+ },+ "result": {+ "callId": call_id,+ "operationId": "get_task_context",+ "result": {"value": "x".repeat(700 * 1024)},+ "isError": false+ }}),);}@@ -568,18 +967,28 @@ fn recovery_rejects_state_without_room_for_a_pending_result() {bridge.prepare(tools("computed")).unwrap();let mut encoded = serde_json::to_value(&bridge).unwrap();let settled = encoded["settledResults"].as_object_mut().unwrap();- for index in 0..8 {+ let mut settled_ids = Vec::new();+ for index in 0..11 {let call_id = format!("recovered-large-{index}");+ settled_ids.push(json!(call_id));settled.insert(call_id.clone(),json!({- "callId": call_id,- "operationId": "get_task_context",- "result": {"value": "x".repeat(900 * 1024)},- "isError": false+ "call": {+ "callId": call_id,+ "operationId": "get_task_context",+ "input": {}+ },+ "result": {+ "callId": call_id,+ "operationId": "get_task_context",+ "result": {"value": "x".repeat(700 * 1024)},+ "isError": false+ }}),);}+ encoded["settledCallIds"] = serde_json::Value::Array(settled_ids);encoded["pending"]["pending-call"] = json!({"callId": "pending-call","operationId": "get_task_context",@@ -594,13 +1003,62 @@ fn recovery_rejects_state_without_room_for_a_pending_result() {}#[test]-fn refuses_to_settle_receipts_while_calls_are_pending() {+fn settles_pending_receipts_with_explicit_terminal_results() {let mut bridge = ProviderToolBridge::default();bridge.prepare(tools("computed")).unwrap();bridge.begin_call("call-1".into(), "get_task_context".into(), json!({})).unwrap();- assert!(bridge.settle_turn().is_err());- assert_eq!(bridge.pending_calls().count(), 1);+ let settled = bridge.settle_turn("provider_turn_terminated").unwrap();+ assert_eq!(settled.len(), 1);+ assert_eq!(settled[0].call_id, "call-1");+ assert!(settled[0].is_error);+ assert_eq!(bridge.pending_calls().count(), 0);+}++#[test]+fn full_identity_ledger_fails_closed_across_turn_and_recovery() {+ let mut bridge = ProviderToolBridge::default();+ bridge.prepare(tools("computed")).unwrap();+ let mut encoded = serde_json::to_value(&bridge).unwrap();+ encoded["settledCallIds"] = serde_json::Value::Array(+ (0..65_536)+ .map(|index| json!(format!("settled-{index:05}")))+ .collect(),+ );+ let mut recovered: ProviderToolBridge = serde_json::from_value(encoded).unwrap();+ recovered.validate_recovered().unwrap();+ let saturation = recovered+ .begin_call("current-call".into(), "get_task_context".into(), json!({}))+ .expect_err("a full exact ledger must stop fresh work");+ assert!(saturation.is_active_turn_receipt_limit());+ assert!(recovered.durable_run_receipt_limit_reached());+ assert!(recovered.has_completed_call("settled-65535"));+ assert!(recovered.has_completed_call("settled-00000"));+ assert!(!recovered.has_completed_call("current-call"));+ assert!(recovered+ .begin_call("settled-00000".into(), "get_task_context".into(), json!({}))+ .is_err());+ recovered.prepare_turn().unwrap();+ let fresh = recovered+ .begin_call("fresh-call".into(), "get_task_context".into(), json!({}))+ .expect_err("a turn boundary must preserve durable-run saturation");+ assert!(fresh.is_active_turn_receipt_limit());++ let round_trip = serde_json::to_value(&recovered).unwrap();+ assert_eq!(+ round_trip["settledCallIds"].as_array().unwrap().len(),+ 65_536+ );+ let mut recovered_again: ProviderToolBridge = serde_json::from_value(round_trip).unwrap();+ recovered_again.attach_existing_run().unwrap();+ assert!(recovered_again.durable_run_receipt_limit_reached());+ assert!(recovered_again.has_completed_call("settled-00000"));++ recovered_again.attach_run(tools("computed")).unwrap();+ assert!(!recovered_again.durable_run_receipt_limit_reached());+ recovered_again+ .begin_call("current-call".into(), "get_task_context".into(), json!({}))+ .expect("only a new durable run resets replay authority");}