Skip to content

The open-source app everyone uses to manage agents at work

feat(runner): durably reconcile Codex tools (#12384)

Dotta6c85fa060e96
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
index 1095e28495..81763e1043 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
@@ -120,6 +120,66 @@ fn emit_ambiguous_turn_evidence(
}
}
+fn emit_ambiguous_turn_item() -> io::Result<()> {
+ send(json!({
+ "method": "item/completed",
+ "params": {"item": {
+ "id": "replacement-message-before-terminal",
+ "type": "agentMessage",
+ "status": "completed",
+ "text": "Replacement output before terminal authority."
+ }}
+ }))
+}
+
+fn send_question(state: &FakeState) -> io::Result<()> {
+ let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1");
+ send(json!({
+ "id": "runtime-request-1",
+ "method": "item/tool/requestUserInput",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": turn_id,
+ "itemId": "question-item-1",
+ "isBlocking": true,
+ "title": "Deployment input",
+ "questions": [{
+ "id": "environment",
+ "header": "Environment",
+ "question": "Where should we deploy?",
+ "options": [
+ {"label": "Staging", "description": "Deploy safely."},
+ {"label": "Production", "description": "Deploy directly."}
+ ]
+ }]
+ }
+ }))
+}
+
+fn send_runtime_request_flood(state: &FakeState, interrupt_count: u64) -> io::Result<()> {
+ let turn_id = state.active_turn_id.as_deref().unwrap_or("provider-turn-1");
+ for index in 0..160_u64 {
+ send(json!({
+ "id": format!("runtime-flood-{interrupt_count}-{index}"),
+ "method": "item/tool/requestUserInput",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": turn_id,
+ "itemId": format!("question-item-{interrupt_count}-{index}"),
+ "isBlocking": true,
+ "title": "Bounded cleanup input",
+ "questions": [{
+ "id": "environment",
+ "header": "Environment",
+ "question": "Where should we deploy?",
+ "options": [{"label": "Staging", "description": "Deploy safely."}],
+ }],
+ },
+ }))?;
+ }
+ Ok(())
+}
+
fn run() -> Result<(), Box<dyn std::error::Error>> {
let args = std::env::args().skip(1).collect::<Vec<_>>();
let state_path =
@@ -136,6 +196,19 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
let exit_after_tool_call_completion = args
.iter()
.any(|value| value == "--exit-after-tool-call-completion");
+ let emit_tool_call_on_resume = args
+ .iter()
+ .any(|value| value == "--emit-tool-call-on-resume");
+ let resume_unowned_turn_when_marked = args
+ .iter()
+ .any(|value| value == "--resume-unowned-turn-when-marked");
+ let replay_completed_tool_call_count = argument(&args, "--replay-completed-tool-call-count")
+ .map(|value| value.parse::<u64>())
+ .transpose()?
+ .unwrap_or_default();
+ let finish_turn_with_pending_tool = args
+ .iter()
+ .any(|value| value == "--finish-turn-with-pending-tool");
let require_dynamic_tool = args.iter().any(|value| value == "--require-dynamic-tool");
let hold_turn = args.iter().any(|value| value == "--hold-turn");
let exit_after_turn_start = args.iter().any(|value| value == "--exit-after-turn-start");
@@ -163,6 +236,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
let missing_id_second_turn_start = args
.iter()
.any(|value| value == "--missing-id-second-turn-start");
+ let missing_id_live_turn_start = args
+ .iter()
+ .any(|value| value == "--missing-id-live-turn-start");
let fail_after_accepting_second_turn_before_response = args
.iter()
.any(|value| value == "--fail-after-accepting-second-turn-before-response");
@@ -184,32 +260,93 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
let conflicting_ambiguous_second_turn = args
.iter()
.any(|value| value == "--conflicting-ambiguous-second-turn");
+ let ambiguous_older_reused_turn = args
+ .iter()
+ .any(|value| value == "--ambiguous-older-reused-turn");
let omit_ambiguous_turn_started = args
.iter()
.any(|value| value == "--omit-ambiguous-turn-started");
let fail_after_thread_read = args.iter().any(|value| value == "--fail-after-thread-read");
+ let fail_first_interrupt = args.iter().any(|value| value == "--fail-first-interrupt");
+ let accept_interrupt_without_terminal_once = args
+ .iter()
+ .any(|value| value == "--accept-interrupt-without-terminal-once");
+ let accept_interrupt_without_terminal = args
+ .iter()
+ .any(|value| value == "--accept-interrupt-without-terminal");
+ let flood_runtime_requests_on_interrupt = args
+ .iter()
+ .any(|value| value == "--flood-runtime-requests-on-interrupt");
+ let interrupt_terminal_delay_ms = argument(&args, "--interrupt-terminal-delay-ms")
+ .map(|value| value.parse::<u64>())
+ .transpose()?;
let exit_after_thread_read = args.iter().any(|value| value == "--exit-after-thread-read");
let fail_after_turn_completion_delay_ms =
argument(&args, "--fail-after-turn-completion-delay-ms")
.map(|value| value.parse::<u64>())
.transpose()?;
+ let delayed_tool_after_failed_turn = args
+ .iter()
+ .any(|value| value == "--delayed-tool-after-failed-turn");
+ let delayed_tool_after_next_turn_start = args
+ .iter()
+ .any(|value| value == "--delayed-tool-after-next-turn-start");
+ let delayed_tool_after_third_turn_start = args
+ .iter()
+ .any(|value| value == "--delayed-tool-after-third-turn-start");
+ let delayed_tool_after_second_turn_completion = args
+ .iter()
+ .any(|value| value == "--delayed-tool-after-second-turn-completion");
+ let tool_after_reused_turn_start = args
+ .iter()
+ .any(|value| value == "--tool-after-reused-turn-start");
+ let tool_after_older_reused_turn_start = args
+ .iter()
+ .any(|value| value == "--tool-after-older-reused-turn-start");
+ let question_before_failed_turn = args
+ .iter()
+ .any(|value| value == "--question-before-failed-turn");
+ let reuse_question_id = args.iter().any(|value| value == "--reuse-question-id");
let pre_response_notification = args
.iter()
.any(|value| value == "--notification-before-response");
let mut state = load_state(&state_path);
let mut turn_start_count = 0_u64;
+ let mut interrupt_count = 0_u64;
+ let mut delayed_interrupt_terminal_scheduled = false;
+ let mut answered_questions = 0u8;
+ let mut replayed_completed_tool_calls = 0_u64;
for line in io::stdin().lock().lines() {
let message: Value = serde_json::from_str(&line?)?;
+ if message.get("method").is_none()
+ && message
+ .get("id")
+ .and_then(Value::as_str)
+ .is_some_and(|id| id.starts_with("runtime-flood-"))
+ {
+ let outcome = if message.get("error").is_some() {
+ "runtime-response:rejected"
+ } else {
+ "runtime-response:cancelled"
+ };
+ log_call(call_log.as_deref(), outcome)?;
+ continue;
+ }
if message.get("method").is_none() && message.get("id") == Some(&json!("runtime-request-1"))
{
+ if reuse_question_id && answered_questions == 0 {
+ answered_questions = 1;
+ send_question(&state)?;
+ continue;
+ }
finish_turn(&state_path, &mut state, "completed")?;
continue;
}
if message.get("method").is_none() && message.get("id") == Some(&json!("tool-request-1")) {
if message.pointer("/result/success") == Some(&json!(false)) {
log_call(call_log.as_deref(), "tool-response:failure")?;
- if state.active_turn_id.is_some() {
+ if state.active_turn_id.is_some() && !hold_turn {
finish_turn(&state_path, &mut state, "failed")?;
}
continue;
@@ -225,7 +362,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
if result != json!({"ok": true, "task": {"id": "task-1"}}) {
return Err("semantic tool response changed the operation result".into());
}
- if replay_completed_tool_call {
+ log_call(call_log.as_deref(), &format!("tool-response:{text}"))?;
+ if replay_completed_tool_call && replayed_completed_tool_calls == 0 {
+ replayed_completed_tool_calls += 1;
send(json!({
"id": "tool-request-replay",
"method": "item/tool/call",
@@ -237,9 +376,22 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
"arguments": {}
}
}))?;
- continue;
+ } else if replayed_completed_tool_calls < replay_completed_tool_call_count {
+ replayed_completed_tool_calls += 1;
+ send(json!({
+ "id": "tool-request-1",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": state.active_turn_id,
+ "callId": "semantic-call-1",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))?;
+ } else if !hold_turn {
+ finish_turn(&state_path, &mut state, "completed")?;
}
- finish_turn(&state_path, &mut state, "completed")?;
continue;
}
let Some(method) = message.get("method").and_then(Value::as_str) else {
@@ -275,10 +427,31 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
if require_dynamic_tool && !has_task_context_tool(&message) {
return Err("thread/resume omitted the authorized dynamic tool".into());
}
+ let unowned_turn_marker = state_path.with_file_name("resume-unowned-turn");
+ if resume_unowned_turn_when_marked && unowned_turn_marker.exists() {
+ state.active_turn_id = Some("provider-turn-unowned".to_owned());
+ save_state(&state_path, &state)?;
+ fs::remove_file(unowned_turn_marker)?;
+ }
send(json!({
"id": id,
"result": {"thread": {"id": state.thread_id, "sessionId": "codex-account-session"}}
}))?;
+ if emit_tool_call_on_resume {
+ if let Some(turn_id) = state.active_turn_id.as_deref() {
+ send(json!({
+ "id": "tool-request-1",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": turn_id,
+ "callId": "semantic-call-1",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))?;
+ }
+ }
}
"thread/read" => {
let turns = state
@@ -319,17 +492,31 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
&& (complete_ambiguous_second_turn
|| complete_ambiguous_second_turn_before_response
|| conflicting_ambiguous_second_turn);
- let provider_turn_id = if emits_ambiguous_turn_evidence
+ let provider_turn_id = if turn_start_count == 3 && ambiguous_older_reused_turn {
+ "provider-turn-1".to_owned()
+ } else if emits_ambiguous_turn_evidence
|| (turn_start_count == 2
&& (retain_ambiguous_second_turn_active
|| hold_ambiguous_second_turn_after_item))
{
- "provider-turn-2"
+ "provider-turn-2".to_owned()
+ } else if tool_after_reused_turn_start
+ || (tool_after_older_reused_turn_start && turn_start_count == 3)
+ {
+ "provider-turn-1".to_owned()
} else {
- "provider-turn-1"
+ format!("provider-turn-{turn_start_count}")
};
- state.active_turn_id = Some(provider_turn_id.to_owned());
+ state.active_turn_id = Some(provider_turn_id.clone());
save_state(&state_path, &state)?;
+ if ambiguous_older_reused_turn && turn_start_count == 3 {
+ send(json!({
+ "method": "turn/started",
+ "params": {"turn": {"id": provider_turn_id}}
+ }))?;
+ send(json!({"id": id, "error": {}}))?;
+ continue;
+ }
if complete_ambiguous_second_turn_before_response && turn_start_count == 2 {
emit_ambiguous_turn_evidence(
&state_path,
@@ -355,6 +542,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
return Ok(());
}
if malformed_error_second_turn_start && turn_start_count == 2 {
+ if hold_ambiguous_second_turn_after_item {
+ emit_ambiguous_turn_item()?;
+ }
send(json!({"id": id, "error": {}}))?;
if emits_ambiguous_turn_evidence
&& !complete_ambiguous_second_turn_before_response
@@ -374,15 +564,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
"result": {"turn": {"status": "inProgress"}}
}))?;
if hold_ambiguous_second_turn_after_item {
- send(json!({
- "method": "item/completed",
- "params": {"item": {
- "id": "replacement-message-before-terminal",
- "type": "agentMessage",
- "status": "completed",
- "text": "Replacement output before terminal authority."
- }}
- }))?;
+ emit_ambiguous_turn_item()?;
continue;
}
if emits_ambiguous_turn_evidence
@@ -397,16 +579,103 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
return Err("configured failure after missing turn identity".into());
}
+ if missing_id_live_turn_start {
+ send(json!({
+ "id": id,
+ "result": {"turn": {"status": "inProgress"}}
+ }))?;
+ continue;
+ }
send(json!({
"id": id,
- "result": {"turn": {"id": "provider-turn-1", "status": "inProgress"}}
+ "result": {"turn": {"id": provider_turn_id, "status": "inProgress"}}
}))?;
send(json!({
"method": "turn/started",
- "params": {"turn": {"id": "provider-turn-1"}}
+ "params": {"turn": {"id": provider_turn_id}}
}))?;
if fail_after_second_turn_start && turn_start_count == 2 {
return Err("configured failure after second turn start".into());
+ } else if question_before_failed_turn {
+ send_question(&state)?;
+ send(json!({
+ "method": "turn/completed",
+ "params": {
+ "threadId": state.thread_id,
+ "turn": {"id": provider_turn_id, "status": "failed"}
+ }
+ }))?;
+ state.active_turn_id = None;
+ save_state(&state_path, &state)?;
+ } else if delayed_tool_after_failed_turn {
+ send(json!({
+ "method": "turn/failed",
+ "params": {
+ "threadId": state.thread_id,
+ "turn": {"id": provider_turn_id, "status": "failed"}
+ }
+ }))?;
+ state.active_turn_id = None;
+ save_state(&state_path, &state)?;
+ send(json!({
+ "id": "tool-request-delayed",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": provider_turn_id,
+ "callId": "semantic-call-delayed",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))?;
+ } else if delayed_tool_after_next_turn_start && turn_start_count == 2 {
+ send(json!({
+ "id": "tool-request-delayed",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": "provider-turn-1",
+ "callId": "semantic-call-delayed",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))?;
+ } else if delayed_tool_after_third_turn_start && turn_start_count == 3 {
+ send(json!({
+ "id": "tool-request-two-turns-delayed",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": "provider-turn-1",
+ "callId": "semantic-call-two-turns-delayed",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))?;
+ } else if tool_after_reused_turn_start && turn_start_count == 2 {
+ send(json!({
+ "id": "tool-request-reused-turn",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": "provider-turn-1",
+ "callId": "semantic-call-reused-turn",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))?;
+ } else if tool_after_older_reused_turn_start && turn_start_count == 3 {
+ send(json!({
+ "id": "tool-request-older-reused-turn",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": "provider-turn-1",
+ "callId": "semantic-call-older-reused-turn",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))?;
} else if exit_after_turn_start {
return Ok(());
} else if emit_tool_call {
@@ -415,41 +684,35 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
"method": "item/tool/call",
"params": {
"threadId": state.thread_id,
- "turnId": "provider-turn-1",
+ "turnId": provider_turn_id,
"callId": "semantic-call-1",
"tool": "get_task_context",
"arguments": {}
}
}))?;
- if complete_after_tool_call {
+ if complete_after_tool_call || finish_turn_with_pending_tool {
finish_turn(&state_path, &mut state, "completed")?;
if exit_after_tool_call_completion {
return Ok(());
}
}
} else if emit_question {
- send(json!({
- "id": "runtime-request-1",
- "method": "item/tool/requestUserInput",
- "params": {
- "threadId": state.thread_id,
- "turnId": "provider-turn-1",
- "itemId": "question-item-1",
- "isBlocking": true,
- "title": "Deployment input",
- "questions": [{
- "id": "environment",
- "header": "Environment",
- "question": "Where should we deploy?",
- "options": [
- {"label": "Staging", "description": "Deploy safely."},
- {"label": "Production", "description": "Deploy directly."}
- ]
- }]
- }
- }))?;
+ send_question(&state)?;
} else if !hold_turn {
finish_turn(&state_path, &mut state, "completed")?;
+ if delayed_tool_after_second_turn_completion && turn_start_count == 2 {
+ send(json!({
+ "id": "tool-request-idle-two-turns-delayed",
+ "method": "item/tool/call",
+ "params": {
+ "threadId": state.thread_id,
+ "turnId": "provider-turn-1",
+ "callId": "semantic-call-idle-two-turns-delayed",
+ "tool": "get_task_context",
+ "arguments": {}
+ }
+ }))?;
+ }
if emit_post_completion_warning {
send(json!({
"method": "warning",
@@ -477,8 +740,44 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
"turn/steer" => send(json!({"id": id, "result": {"accepted": true}}))?,
"turn/interrupt" => {
- send(json!({"id": id, "result": {"accepted": true}}))?;
- finish_turn(&state_path, &mut state, "interrupted")?;
+ interrupt_count += 1;
+ if fail_first_interrupt && interrupt_count == 1 {
+ send(json!({
+ "id": id,
+ "error": {"code": -32001, "message": "configured interrupt failure"}
+ }))?;
+ } else {
+ send(json!({"id": id, "result": {"accepted": true}}))?;
+ if flood_runtime_requests_on_interrupt {
+ send_runtime_request_flood(&state, interrupt_count)?;
+ }
+ if !accept_interrupt_without_terminal
+ && !(accept_interrupt_without_terminal_once
+ && interrupt_count == if fail_first_interrupt { 2 } else { 1 })
+ {
+ if let Some(delay_ms) = interrupt_terminal_delay_ms {
+ if !delayed_interrupt_terminal_scheduled {
+ delayed_interrupt_terminal_scheduled = true;
+ let delayed_state_path = state_path.clone();
+ let mut delayed_state = state.clone();
+ thread::spawn(move || {
+ thread::sleep(Duration::from_millis(delay_ms));
+ if let Err(error) = finish_turn(
+ &delayed_state_path,
+ &mut delayed_state,
+ "interrupted",
+ ) {
+ eprintln!(
+ "failed to emit delayed interrupt terminal: {error}"
+ );
+ }
+ });
+ }
+ } else {
+ finish_turn(&state_path, &mut state, "interrupted")?;
+ }
+ }
+ }
}
_ if id.is_some() => send(json!({
"id": id,
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs
index 4764ed1e88..62aed56ea1 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs
@@ -50,28 +50,26 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> {
optional_u64(args, name).map(|value| Duration::from_millis(value.unwrap_or(default)))
};
let state_dir = PathBuf::from(value(args, "--state-dir")?);
- run_durable_runner(
- DurableRunnerConfig {
- connect_url: value(args, "--connect-url")?,
- state_dir: state_dir.clone(),
- runner_instance_id: value(args, "--runner-id")?,
- environment_lease_id: value(args, "--environment-lease-id")?,
- run_id: value(args, "--run-id")?,
- normalized_session_id: value(args, "--session-id")?,
- turn_id: value(args, "--turn-id")?,
- item_id: value(args, "--item-id")?,
- runner_version: value(args, "--runner-version")?,
- runner_digest: value(args, "--runner-digest")?,
- max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 16 * 1024 * 1024)?,
- p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 1024 * 1024)?,
- max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?,
- reconnect_delay: duration("--reconnect-delay-ms", 250)?,
- max_runtime: duration("--max-runtime-ms", 60 * 60 * 1000)?,
- },
- ticket,
- CodexCommandExecutor::new(state_dir),
- )
- .map_err(|error| LocalRunnerError::invalid(error.to_string()))
+ let config = DurableRunnerConfig {
+ connect_url: value(args, "--connect-url")?,
+ state_dir: state_dir.clone(),
+ runner_instance_id: value(args, "--runner-id")?,
+ environment_lease_id: value(args, "--environment-lease-id")?,
+ run_id: value(args, "--run-id")?,
+ normalized_session_id: value(args, "--session-id")?,
+ turn_id: value(args, "--turn-id")?,
+ item_id: value(args, "--item-id")?,
+ runner_version: value(args, "--runner-version")?,
+ runner_digest: value(args, "--runner-digest")?,
+ max_outbox_bytes: usize_value(args, "--max-outbox-bytes", 16 * 1024 * 1024)?,
+ p0_reserve_bytes: usize_value(args, "--p0-reserve-bytes", 1024 * 1024)?,
+ max_frame_bytes: usize_value(args, "--max-frame-bytes", 1024 * 1024)?,
+ reconnect_delay: duration("--reconnect-delay-ms", 250)?,
+ max_runtime: duration("--max-runtime-ms", 60 * 60 * 1000)?,
+ };
+ let executor = CodexCommandExecutor::with_runner_config(state_dir, &config);
+ run_durable_runner(config, ticket, executor)
+ .map_err(|error| LocalRunnerError::invalid(error.to_string()))
}
fn run() -> Result<(), LocalRunnerError> {
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
index 322236013d..249c13f0ae 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
@@ -4,18 +4,24 @@ use std::time::Duration;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
+use sha2::{Digest, Sha256};
use crate::durable::redact_text;
use crate::local_runner::LocalRunnerError;
use crate::process_supervisor::SupervisedProcess;
-use crate::provider_bridge::{AuthorizedTool, ToolResult};
+use crate::provider_bridge::{AuthorizedTool, DurableReplayFilter, ToolResult};
+use crate::provider_events::normalized_codex_terminal_event_type;
pub const CODEX_APP_SERVER_MAX_FRAME_BYTES: usize = 4 * 1024 * 1024;
const MAX_BUFFERED_MESSAGES: usize = 1_024;
+const MAX_BUFFERED_MESSAGE_BYTES: usize = 16 * 1024 * 1024;
const MAX_INSTRUCTIONS_BYTES: usize = 1024 * 1024;
const MAX_PENDING_TOOL_REQUESTS: usize = 4_096;
const MAX_PENDING_TOOL_REQUEST_BYTES: usize = 16 * 1024 * 1024;
const MAX_COMPLETED_TOOL_CALL_IDS: usize = 4_096;
+const MAX_PENDING_RUNTIME_REQUESTS: usize = 128;
+const MAX_PENDING_RUNTIME_REQUEST_BYTES: usize = 4 * 1024 * 1024;
+pub(crate) const MAX_SETTLED_PROVIDER_TURN_IDS: usize = 4_096;
type QuestionOptionLabels = BTreeMap<String, BTreeMap<String, String>>;
type QuestionSetMapping = (String, Value, QuestionOptionLabels);
@@ -135,11 +141,71 @@ struct CompletedTurnAuthority {
provider_turn_id: String,
}
+#[derive(Default)]
+struct SettledProviderTurnIds {
+ ids: BTreeSet<String>,
+ filter: DurableReplayFilter,
+}
+
+impl SettledProviderTurnIds {
+ fn insert(&mut self, provider_turn_id: String) -> bool {
+ if self.contains(&provider_turn_id) {
+ return true;
+ }
+ if self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS {
+ return false;
+ }
+ self.ids.insert(provider_turn_id)
+ }
+
+ fn contains(&self, provider_turn_id: &str) -> bool {
+ self.ids.contains(provider_turn_id)
+ }
+
+ fn at_capacity(&self) -> bool {
+ self.ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS || !self.filter.is_empty()
+ }
+
+ fn restore(&mut self, provider_turn_id: String) -> Result<(), LocalRunnerError> {
+ if !self.insert(provider_turn_id) {
+ return Err(LocalRunnerError::invalid(
+ "Codex restored provider turn identity epoch exceeded its exact capacity",
+ ));
+ }
+ Ok(())
+ }
+
+ fn restore_all(
+ &mut self,
+ provider_turn_ids: impl IntoIterator<Item = String>,
+ replay_filter: DurableReplayFilter,
+ ) -> Result<(), LocalRunnerError> {
+ replay_filter
+ .validate()
+ .map_err(|error| LocalRunnerError::invalid(error.to_string()))?;
+ self.filter = replay_filter;
+ for provider_turn_id in provider_turn_ids {
+ if !self.insert(provider_turn_id) {
+ return Err(LocalRunnerError::invalid(
+ "Codex restored provider turn identity epoch exceeded its exact capacity",
+ ));
+ }
+ }
+ Ok(())
+ }
+}
+
enum ProviderRequestError {
Rejected(LocalRunnerError),
Ambiguous(LocalRunnerError),
}
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) enum RejectedAcceptedTurn {
+ ReusedIdentity(String),
+ InvalidIdentity,
+}
+
impl ProviderRequestError {
fn into_inner(self) -> LocalRunnerError {
match self {
@@ -159,10 +225,12 @@ struct PendingToolRequest {
#[derive(Clone, Debug, PartialEq)]
struct PendingRuntimeRequest {
rpc_id: Value,
+ turn_id: String,
method: String,
params: Value,
question_set: Value,
option_labels: QuestionOptionLabels,
+ retained_bytes: usize,
}
struct BufferedProviderMessage {
@@ -178,22 +246,32 @@ enum AmbiguousTurnMessage {
pub struct CodexProvider {
process: SupervisedProcess,
+ config: CodexProviderConfig,
+ authorized_tools: Vec<AuthorizedTool>,
next_request_id: u64,
thread_id: String,
provider_session_id: Option<String>,
active_provider_turn_id: Option<String>,
pending_messages: VecDeque<BufferedProviderMessage>,
deferred_ambiguous_messages: VecDeque<BufferedProviderMessage>,
+ pending_message_bytes: usize,
authorized_tool_ids: BTreeSet<String>,
pending_tool_requests: BTreeMap<String, PendingToolRequest>,
completed_tool_call_ids: BTreeSet<String>,
+ durable_tool_call_replays: bool,
pending_tool_request_bytes: usize,
pending_runtime_requests: BTreeMap<String, PendingRuntimeRequest>,
+ pending_runtime_request_bytes: usize,
+ runtime_request_scope: [u8; 16],
+ next_runtime_request_sequence: u64,
expected_shutdown: bool,
process_generation: u64,
completed_turn_authority: Option<CompletedTurnAuthority>,
completion_reconciliation_pending: bool,
ambiguous_turn_start_pending: bool,
+ settled_provider_turn_ids: SettledProviderTurnIds,
+ rejected_accepted_turn: Option<RejectedAcceptedTurn>,
+ quarantined: bool,
}
impl CodexProvider {
@@ -212,20 +290,7 @@ impl CodexProvider {
Self::start_with_tools_for_generation(config, authorized_tools, resume_thread_id, 1)
}
- pub(crate) fn start_for_generation(
- config: &CodexProviderConfig,
- resume_thread_id: Option<&str>,
- process_generation: u64,
- ) -> Result<Self, LocalRunnerError> {
- Self::start_with_tools_for_generation(
- config,
- std::iter::empty(),
- resume_thread_id,
- process_generation,
- )
- }
-
- fn start_with_tools_for_generation(
+ pub(crate) fn start_with_tools_for_generation(
config: &CodexProviderConfig,
authorized_tools: impl IntoIterator<Item = AuthorizedTool>,
resume_thread_id: Option<&str>,
@@ -237,7 +302,9 @@ impl CodexProvider {
"Codex process generation must be positive",
));
}
- let (dynamic_tools, authorized_tool_ids) = codex_dynamic_tools(authorized_tools)?;
+ let authorized_tools = authorized_tools.into_iter().collect::<Vec<_>>();
+ let (dynamic_tools, authorized_tool_ids) =
+ codex_dynamic_tools(authorized_tools.iter().cloned())?;
let mut provider = Self {
process: SupervisedProcess::spawn(
&config.command,
@@ -245,22 +312,32 @@ impl CodexProvider {
Duration::from_secs(2),
CODEX_APP_SERVER_MAX_FRAME_BYTES,
)?,
+ config: config.clone(),
+ authorized_tools,
next_request_id: 1,
thread_id: String::new(),
provider_session_id: None,
active_provider_turn_id: None,
pending_messages: VecDeque::new(),
deferred_ambiguous_messages: VecDeque::new(),
+ pending_message_bytes: 0,
authorized_tool_ids,
pending_tool_requests: BTreeMap::new(),
completed_tool_call_ids: BTreeSet::new(),
+ durable_tool_call_replays: false,
pending_tool_request_bytes: 0,
pending_runtime_requests: BTreeMap::new(),
+ pending_runtime_request_bytes: 0,
+ runtime_request_scope: new_runtime_request_scope()?,
+ next_runtime_request_sequence: 1,
expected_shutdown: false,
process_generation,
completed_turn_authority: None,
completion_reconciliation_pending: false,
ambiguous_turn_start_pending: false,
+ settled_provider_turn_ids: SettledProviderTurnIds::default(),
+ rejected_accepted_turn: None,
+ quarantined: false,
};
let initialized = provider.request(
"initialize",
@@ -320,7 +397,7 @@ impl CodexProvider {
if resume_thread_id.is_some() {
let snapshot = provider.read_thread()?;
provider.active_provider_turn_id = latest_active_turn_id(&snapshot)
- .map(|turn_id| bounded_identifier(Some(&turn_id), "Codex turn id"))
+ .map(|provider_turn_id| bounded_provider_turn_id(Some(&provider_turn_id)))
.transpose()?;
}
Ok(provider)
@@ -330,6 +407,10 @@ impl CodexProvider {
self.process.id()
}
+ pub(crate) fn process_generation(&self) -> u64 {
+ self.process_generation
+ }
+
pub fn thread_id(&self) -> &str {
&self.thread_id
}
@@ -346,12 +427,19 @@ impl CodexProvider {
self.ambiguous_turn_start_pending
}
+ pub(crate) fn enable_durable_tool_call_replays(&mut self) {
+ // The durable backend validates the call id, operation, and input
+ // against its persisted receipt before returning a stored result.
+ // Direct provider consumers retain the stricter one-shot behavior.
+ self.durable_tool_call_replays = true;
+ }
+
pub(crate) fn restore_completed_turn_authority(
&mut self,
authoritative: bool,
process_generation: Option<u64>,
provider_turn_id: Option<&str>,
- ) {
+ ) -> Result<(), LocalRunnerError> {
self.completed_turn_authority = authoritative.then(|| CompletedTurnAuthority {
// Legacy state did not record the generation. Generation zero is
// deliberately older than every supervised process generation.
@@ -360,6 +448,10 @@ impl CodexProvider {
.unwrap_or("durable-completed-turn")
.to_owned(),
});
+ if let Some(authority) = self.completed_turn_authority.as_ref() {
+ self.settled_provider_turn_ids
+ .restore(authority.provider_turn_id.clone())?;
+ }
// Resuming a completed durable thread and reading its provider state
// is recovery, not new turn work. Keep the prior terminal authoritative
// until start_turn explicitly revokes it.
@@ -369,6 +461,16 @@ impl CodexProvider {
// of them supersedes a completed result. Only accepting a replacement
// turn identity revokes this authority.
self.completion_reconciliation_pending = false;
+ Ok(())
+ }
+
+ pub(crate) fn restore_settled_turn_identities(
+ &mut self,
+ provider_turn_ids: impl IntoIterator<Item = String>,
+ replay_filter: DurableReplayFilter,
+ ) -> Result<(), LocalRunnerError> {
+ self.settled_provider_turn_ids
+ .restore_all(provider_turn_ids, replay_filter)
}
pub(crate) fn completed_turn_authority(&self) -> Option<(u64, &str)> {
@@ -380,7 +482,87 @@ impl CodexProvider {
})
}
+ pub(crate) fn take_rejected_accepted_turn(&mut self) -> Option<RejectedAcceptedTurn> {
+ self.rejected_accepted_turn.take()
+ }
+
+ pub(crate) fn restart_idle_identity_epoch(&mut self) -> Result<(), LocalRunnerError> {
+ if self.active_provider_turn_id.is_some() || self.ambiguous_turn_start_pending {
+ return Err(LocalRunnerError::invalid(
+ "Codex provider identity epoch cannot rotate while work is active",
+ ));
+ }
+
+ let next_generation = self.process_generation.checked_add(1).ok_or_else(|| {
+ LocalRunnerError::invalid("Codex process generation exhausted during epoch rollover")
+ })?;
+ let config = self.config.clone();
+ let authorized_tools = self.authorized_tools.clone();
+ let thread_id = self.thread_id.clone();
+ let completed_turn_authority = self.completed_turn_authority.clone();
+ let completion_reconciliation_pending = self.completion_reconciliation_pending;
+ let durable_tool_call_replays = self.durable_tool_call_replays;
+
+ // Exact turn identities may be forgotten only after the provider
+ // process that could emit them is gone. Resume the same thread in a
+ // fresh process generation, then preserve prior completion authority
+ // until a replacement turn identity is actually accepted.
+ self.shutdown()?;
+ let mut replacement = Self::start_with_tools_for_generation(
+ &config,
+ authorized_tools,
+ Some(&thread_id),
+ next_generation,
+ )?;
+ replacement.durable_tool_call_replays = durable_tool_call_replays;
+ if replacement.active_provider_turn_id.is_some() {
+ // A terminal can race the provider's own durable idle-state write.
+ // This process has work Paperclip never dispatched in the new
+ // epoch, so revoke its request authority and reap it. Retain the
+ // exact ledger for diagnostics, but never expose the unexpected
+ // turn as ordinary active work or admit a replacement.
+ replacement.settled_provider_turn_ids =
+ std::mem::take(&mut self.settled_provider_turn_ids);
+ replacement.active_provider_turn_id = None;
+ replacement.ambiguous_turn_start_pending = true;
+ replacement.rejected_accepted_turn = Some(RejectedAcceptedTurn::InvalidIdentity);
+ replacement.quarantined = true;
+ replacement.pending_messages.clear();
+ replacement.deferred_ambiguous_messages.clear();
+ replacement.pending_message_bytes = 0;
+ let _ = replacement.cancel_pending_requests();
+ let _ = replacement.process.terminate_group();
+ replacement.expected_shutdown = false;
+ *self = replacement;
+ return Err(LocalRunnerError::invalid(
+ "Codex provider epoch rollover resumed unowned active work; the provider was terminated",
+ ));
+ }
+ if let Some(authority) = completed_turn_authority.as_ref() {
+ replacement.restore_completed_turn_authority(
+ true,
+ Some(authority.process_generation),
+ Some(&authority.provider_turn_id),
+ )?;
+ }
+ replacement.completion_reconciliation_pending = completion_reconciliation_pending;
+ *self = replacement;
+ Ok(())
+ }
+
+ fn rollover_settled_turn_epoch_if_needed(&mut self) -> Result<(), LocalRunnerError> {
+ if !self.settled_provider_turn_ids.at_capacity() {
+ return Ok(());
+ }
+ self.restart_idle_identity_epoch()
+ }
+
pub fn start_turn(&mut self, message: &str, cwd: &str) -> Result<Value, LocalRunnerError> {
+ if self.quarantined {
+ return Err(LocalRunnerError::invalid(
+ "Codex provider is quarantined after unsafe recovered work",
+ ));
+ }
if self.active_provider_turn_id.is_some() {
return Err(LocalRunnerError::invalid(
"Codex already has an active provider turn",
@@ -396,6 +578,7 @@ impl CodexProvider {
"Codex turn text is empty or exceeds the 1 MiB limit",
));
}
+ self.rollover_settled_turn_epoch_if_needed()?;
// Preserve the prior durable result until a replacement turn identity
// is accepted. A rejected, ambiguous, or transport-failed attempt does
// not prove that replacement work superseded the completed turn.
@@ -403,6 +586,7 @@ impl CodexProvider {
self.completion_reconciliation_pending = false;
let prior_buffered_message_count = self.pending_messages.len();
self.ambiguous_turn_start_pending = true;
+ let runtime_request_scope = new_runtime_request_scope()?;
let result = match self.request_classified(
"turn/start",
json!({
@@ -433,16 +617,32 @@ impl CodexProvider {
}
Err(ProviderRequestError::Ambiguous(error)) => return Err(error),
};
- let provider_turn_id = bounded_identifier(
- result
- .pointer("/turn/id")
- .or_else(|| result.get("turnId"))
- .and_then(Value::as_str),
- "Codex turn id",
- )?;
+ let provider_turn_id = result
+ .pointer("/turn/id")
+ .or_else(|| result.get("turnId"))
+ .and_then(Value::as_str);
+ let provider_turn_id = match bounded_provider_turn_id(provider_turn_id) {
+ Ok(provider_turn_id) => provider_turn_id,
+ Err(error) => {
+ // A successful turn/start response means the provider may
+ // already be executing the work. Without a bounded identity,
+ // runnerd cannot durably bind, interrupt, or reconcile it.
+ // Terminate the process and let the durable backend close the
+ // run before returning the protocol error.
+ self.rejected_accepted_turn = Some(RejectedAcceptedTurn::InvalidIdentity);
+ self.expected_shutdown = true;
+ self.completed_turn_authority = None;
+ let _ = self.cancel_pending_requests();
+ let _ = self.process.terminate_group();
+ return Err(error);
+ }
+ };
+ if self.settled_provider_turn_ids.contains(&provider_turn_id) {
+ return Err(self.reject_accepted_reused_turn_identity(provider_turn_id));
+ }
// Only a validated provider turn identity proves that replacement
// work exists and supersedes the prior completed result.
- self.accept_replacement_turn(provider_turn_id);
+ self.accept_replacement_turn(provider_turn_id, runtime_request_scope);
Ok(result)
}
@@ -471,17 +671,12 @@ impl CodexProvider {
))
})?;
- if self
- .completed_turn_authority
- .as_ref()
- .is_some_and(|authority| authority.provider_turn_id == provider_turn_id)
- {
- return Err(LocalRunnerError::invalid(format!(
- "Codex {method} notification reused the previously completed turn id while resolving an ambiguous turn start"
- )));
+ if self.settled_provider_turn_ids.contains(&provider_turn_id) {
+ return Err(self.reject_accepted_reused_turn_identity(provider_turn_id));
}
- self.accept_replacement_turn(provider_turn_id.clone());
+ let runtime_request_scope = new_runtime_request_scope()?;
+ self.accept_replacement_turn(provider_turn_id.clone(), runtime_request_scope);
if method == "turn/started" && message.get("id").is_none() {
Ok(AmbiguousTurnMessage::ReconciledWithStart)
} else {
@@ -489,13 +684,39 @@ impl CodexProvider {
}
}
- fn accept_replacement_turn(&mut self, provider_turn_id: String) {
+ fn accept_replacement_turn(
+ &mut self,
+ provider_turn_id: String,
+ runtime_request_scope: [u8; 16],
+ ) {
self.ambiguous_turn_start_pending = false;
self.expected_shutdown = false;
self.completed_turn_authority = None;
self.completion_reconciliation_pending = false;
self.completed_tool_call_ids.clear();
+ // Retain the prior settled identity while the next turn runs. Besides
+ // recognizing delayed prior-turn requests, this fails closed if a
+ // provider ambiguously reuses the same turn id for fresh work.
self.active_provider_turn_id = Some(provider_turn_id);
+ self.runtime_request_scope = runtime_request_scope;
+ }
+
+ fn reject_accepted_reused_turn_identity(
+ &mut self,
+ provider_turn_id: String,
+ ) -> LocalRunnerError {
+ // Both a successful response and identity-bearing evidence after an
+ // ambiguous response prove the provider accepted work. A settled
+ // identity cannot durably own that work, so terminate its process
+ // before returning instead of leaving an untracked turn alive.
+ self.rejected_accepted_turn = Some(RejectedAcceptedTurn::ReusedIdentity(provider_turn_id));
+ self.expected_shutdown = true;
+ self.completed_turn_authority = None;
+ let _ = self.cancel_pending_requests();
+ let _ = self.process.terminate_group();
+ LocalRunnerError::invalid(
+ "Codex reused a settled provider turn identity after accepting work; the provider was terminated",
+ )
}
pub fn steer_turn(&mut self, message: &str) -> Result<Value, LocalRunnerError> {
@@ -554,15 +775,84 @@ impl CodexProvider {
.ok_or_else(|| {
LocalRunnerError::invalid("runtime response has no pending Codex request")
})?;
+ if self.active_provider_turn_id.as_deref() != Some(pending.turn_id.as_str()) {
+ return Err(LocalRunnerError::invalid(
+ "runtime response belongs to another Codex turn",
+ ));
+ }
let result = codex_question_response(&pending, response)?;
self.process
.send(&json!({"id": pending.rpc_id, "result": result}))?;
- self.pending_runtime_requests.remove(request_id);
+ if let Some(completed) = self.pending_runtime_requests.remove(request_id) {
+ self.pending_runtime_request_bytes = self
+ .pending_runtime_request_bytes
+ .saturating_sub(completed.retained_bytes);
+ }
Ok(())
}
+ fn reject_post_terminal_request(
+ &mut self,
+ rpc_id: Value,
+ method: &str,
+ ) -> Result<Option<CodexProviderEvent>, LocalRunnerError> {
+ let message = format!(
+ "ignored delayed {} request after the Codex turn terminated",
+ bounded_method(method)
+ );
+ let response = if method == "item/tool/call" {
+ json!({
+ "id": rpc_id,
+ "result": codex_tool_failure("the Codex turn has already terminated"),
+ })
+ } else {
+ json!({
+ "id": rpc_id,
+ "error": {"code": -32000, "message": "the Codex turn has already terminated"},
+ })
+ };
+ // The terminal notification is already authoritative and may be
+ // waiting in the durable outbox. A courtesy rejection must not turn a
+ // provider that has closed stdin into a fatal polling error.
+ let _ = self.process.send(&response);
+ Ok(Some(CodexProviderEvent::Notification {
+ method: "warning".to_owned(),
+ params: json!({"message": message, "providerMethod": bounded_method(method)}),
+ }))
+ }
+
pub fn poll(&mut self) -> Result<Option<CodexProviderEvent>, LocalRunnerError> {
+ if self.quarantined {
+ // Never interpret provider-originated requests after fail-closed
+ // quarantine. Drain output only so process termination cannot
+ // deadlock on a full pipe, then surface an unequivocal failure.
+ if self
+ .process
+ .receive_stdout_line(Duration::from_millis(1))?
+ .is_some()
+ {
+ return Ok(None);
+ }
+ return Ok(self
+ .process
+ .try_wait()?
+ .map(|exit| CodexProviderEvent::Exited {
+ exit_code: exit.exit_code,
+ success: false,
+ completed_turn_authoritative: false,
+ completed_turn_observed_by_process: false,
+ completion_reconciles_exit: false,
+ process_generation: self.process_generation,
+ completed_turn_process_generation: None,
+ }));
+ }
let buffered = self.pending_messages.pop_front();
+ if let Some(buffered) = buffered.as_ref() {
+ self.pending_message_bytes = self.pending_message_bytes.saturating_sub(json_size(
+ &buffered.value,
+ "buffered Codex provider message",
+ )?);
+ }
let message = if let Some(buffered) = buffered {
buffered.value
} else {
@@ -625,11 +915,24 @@ impl CodexProvider {
match self.classify_ambiguous_turn_message(&message)? {
AmbiguousTurnMessage::Ready => {}
AmbiguousTurnMessage::Deferred => {
- if self.deferred_ambiguous_messages.len() >= MAX_BUFFERED_MESSAGES {
+ if self
+ .pending_messages
+ .len()
+ .saturating_add(self.deferred_ambiguous_messages.len())
+ >= MAX_BUFFERED_MESSAGES
+ {
return Err(LocalRunnerError::invalid(
"Codex emitted too many messages before resolving an ambiguous turn start",
));
}
+ let retained_bytes = json_size(&message, "buffered Codex provider message")?;
+ self.pending_message_bytes =
+ retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes)
+ .ok_or_else(|| {
+ LocalRunnerError::invalid(
+ "Codex buffered messages exceed the 16 MiB aggregate limit",
+ )
+ })?;
self.deferred_ambiguous_messages
.push_back(BufferedProviderMessage { value: message });
return Ok(None);
@@ -641,6 +944,14 @@ impl CodexProvider {
}
AmbiguousTurnMessage::ReconciledNeedsStart { provider_turn_id } => {
let mut replay = std::mem::take(&mut self.deferred_ambiguous_messages);
+ let retained_bytes = json_size(&message, "buffered Codex provider message")?;
+ self.pending_message_bytes =
+ retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes)
+ .ok_or_else(|| {
+ LocalRunnerError::invalid(
+ "Codex buffered messages exceed the 16 MiB aggregate limit",
+ )
+ })?;
replay.push_back(BufferedProviderMessage { value: message });
replay.append(&mut self.pending_messages);
self.pending_messages = replay;
@@ -666,6 +977,13 @@ impl CodexProvider {
"Codex tool call named another thread",
));
}
+ if request_targets_non_active_turn(
+ self.active_provider_turn_id.as_deref(),
+ &self.settled_provider_turn_ids,
+ &params,
+ ) {
+ return self.reject_post_terminal_request(rpc_id, method);
+ }
let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| {
LocalRunnerError::invalid("Codex tool call arrived outside an active turn")
})?;
@@ -719,7 +1037,8 @@ impl CodexProvider {
input: input.clone(),
retained_bytes,
};
- if self.completed_tool_call_ids.contains(&call_id) {
+ let completed_replay = self.completed_tool_call_ids.contains(&call_id);
+ if completed_replay && !self.durable_tool_call_replays {
return Err(LocalRunnerError::invalid(
"Codex reused a completed tool call id",
));
@@ -746,7 +1065,9 @@ impl CodexProvider {
"Codex emitted too many pending tool calls",
));
}
- if self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS {
+ if !completed_replay
+ && self.completed_tool_call_ids.len() >= MAX_COMPLETED_TOOL_CALL_IDS
+ {
return Err(LocalRunnerError::invalid(
"Codex emitted too many completed tool calls in one turn",
));
@@ -770,36 +1091,93 @@ impl CodexProvider {
"Codex runtime request named another thread",
));
}
- let active_turn_id = self.active_provider_turn_id.as_deref().ok_or_else(|| {
+ if request_targets_non_active_turn(
+ self.active_provider_turn_id.as_deref(),
+ &self.settled_provider_turn_ids,
+ &params,
+ ) {
+ return self.reject_post_terminal_request(rpc_id, method);
+ }
+ let active_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| {
LocalRunnerError::invalid(
"Codex runtime request arrived outside an active turn",
)
})?;
- if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id) {
+ if params.get("turnId").and_then(Value::as_str) != Some(active_turn_id.as_str()) {
return Err(LocalRunnerError::invalid(
"Codex runtime request named another turn",
));
}
- let (request_id, question_set, option_labels) =
+ let (provider_request_id, question_set, option_labels) =
codex_question_set(&rpc_id, &params)?;
+ let retained_bytes = pending_runtime_request_size(
+ &rpc_id,
+ &active_turn_id,
+ method,
+ &params,
+ &question_set,
+ &option_labels,
+ )?;
let pending = PendingRuntimeRequest {
- rpc_id,
+ rpc_id: rpc_id.clone(),
+ turn_id: active_turn_id.clone(),
method: method.to_owned(),
params,
question_set: question_set.clone(),
option_labels,
+ retained_bytes,
};
- if let Some(existing) = self.pending_runtime_requests.get(&request_id) {
+ if let Some(existing) = self
+ .pending_runtime_requests
+ .values()
+ .find(|existing| existing.rpc_id == rpc_id)
+ {
if existing != &pending {
return Err(LocalRunnerError::invalid(
"Codex reused a runtime request id with different input",
));
}
return Ok(None);
- } else {
- self.pending_runtime_requests
- .insert(request_id.clone(), pending);
}
+ let retained_request_bytes = retain_pending_runtime_request_bytes(
+ self.pending_runtime_request_bytes,
+ retained_bytes,
+ );
+ if self.pending_runtime_requests.len() >= MAX_PENDING_RUNTIME_REQUESTS
+ || retained_request_bytes.is_none()
+ {
+ self.process.send(&json!({
+ "id": rpc_id,
+ "error": {
+ "code": -32000,
+ "message": "Paperclip rejected this runtime request because the pending input capacity was reached",
+ },
+ }))?;
+ return Ok(Some(CodexProviderEvent::Notification {
+ method: "warning".to_owned(),
+ params: json!({
+ "message": "rejected a Codex runtime request at the bounded pending-input limit",
+ "providerMethod": "item/tool/requestUserInput",
+ }),
+ }));
+ }
+ let request_sequence = self.next_runtime_request_sequence;
+ self.next_runtime_request_sequence = self
+ .next_runtime_request_sequence
+ .checked_add(1)
+ .ok_or_else(|| {
+ LocalRunnerError::invalid("Codex runtime request sequence overflowed")
+ })?;
+ let request_id = scoped_runtime_request_id(
+ &self.runtime_request_scope,
+ &active_turn_id,
+ &provider_request_id,
+ request_sequence,
+ );
+ self.pending_runtime_requests
+ .insert(request_id.clone(), pending);
+ self.pending_runtime_request_bytes =
+ retained_request_bytes.expect("bounded runtime request bytes checked above");
return Ok(Some(CodexProviderEvent::RuntimeRequest {
request_id,
question_set,
@@ -817,24 +1195,60 @@ impl CodexProvider {
if let Some(method) = message.get("method").and_then(Value::as_str) {
let params = message.get("params").cloned().unwrap_or(Value::Null);
+ let terminal_event_type = normalized_codex_terminal_event_type(method, &params);
+ let notification_turn_id = params
+ .get("turnId")
+ .or_else(|| params.pointer("/turn/id"))
+ .and_then(Value::as_str);
+ if terminal_event_type.is_some()
+ && notification_turn_id.is_some()
+ && notification_turn_id != self.active_provider_turn_id.as_deref()
+ && notification_turn_id
+ .is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id))
+ {
+ return Ok(Some(CodexProviderEvent::Notification {
+ method: "warning".to_owned(),
+ params: json!({
+ "message": "ignored a terminal notification for a non-active Codex turn",
+ "providerMethod": bounded_method(method),
+ }),
+ }));
+ }
validate_notification_binding(
&self.thread_id,
self.active_provider_turn_id.as_deref(),
&params,
)?;
- if method == "turn/completed" {
- let provider_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| {
- LocalRunnerError::invalid(
- "Codex completion arrived outside an active provider turn",
- )
- })?;
+ if let Some(terminal_event_type) = terminal_event_type {
+ if self.active_provider_turn_id.is_none() {
+ return Err(LocalRunnerError::invalid(
+ "Codex terminal arrived outside an active provider turn",
+ ));
+ }
+ let provider_turn_id = self
+ .active_provider_turn_id
+ .clone()
+ .expect("active provider turn checked above");
+ let completed_turn_authority = if terminal_event_type == "turn.completed" {
+ Some(CompletedTurnAuthority {
+ process_generation: self.process_generation,
+ provider_turn_id: provider_turn_id.clone(),
+ })
+ } else {
+ None
+ };
+ if !self
+ .settled_provider_turn_ids
+ .insert(provider_turn_id.clone())
+ {
+ return Err(LocalRunnerError::invalid(
+ "Codex provider turn identity epoch reached its exact capacity",
+ ));
+ }
self.active_provider_turn_id = None;
self.expected_shutdown = true;
- self.completed_turn_authority = Some(CompletedTurnAuthority {
- process_generation: self.process_generation,
- provider_turn_id,
- });
- self.completion_reconciliation_pending = true;
+ self.completed_turn_authority = completed_turn_authority;
+ self.completion_reconciliation_pending = terminal_event_type == "turn.completed";
// The provider terminal is authoritative once received. Clear
// local request ownership and attempt courtesy responses, but
// a provider that already closed stdin must not turn the
@@ -898,6 +1312,7 @@ impl CodexProvider {
fn cancel_pending_requests(&mut self) -> Result<(), LocalRunnerError> {
let pending_runtime = std::mem::take(&mut self.pending_runtime_requests);
let pending = std::mem::take(&mut self.pending_tool_requests);
+ self.pending_runtime_request_bytes = 0;
self.pending_tool_request_bytes = 0;
let mut first_error = None;
for request in pending_runtime.into_values() {
@@ -967,17 +1382,44 @@ impl CodexProvider {
}
return Ok(message.get("result").cloned().unwrap_or(Value::Null));
}
- if self.pending_messages.len() >= MAX_BUFFERED_MESSAGES {
+ if self
+ .pending_messages
+ .len()
+ .saturating_add(self.deferred_ambiguous_messages.len())
+ >= MAX_BUFFERED_MESSAGES
+ {
return Err(ProviderRequestError::Ambiguous(LocalRunnerError::invalid(
"Codex emitted too many messages before a request response",
)));
}
+ let retained_bytes = json_size(&message, "buffered Codex provider message")
+ .map_err(ProviderRequestError::Ambiguous)?;
+ let next_retained_bytes =
+ retain_buffered_message_bytes(self.pending_message_bytes, retained_bytes)
+ .ok_or_else(|| {
+ ProviderRequestError::Ambiguous(LocalRunnerError::invalid(
+ "Codex buffered messages exceed the 16 MiB aggregate limit",
+ ))
+ })?;
self.pending_messages
.push_back(BufferedProviderMessage { value: message });
+ self.pending_message_bytes = next_retained_bytes;
}
}
}
+fn json_size(value: &Value, label: &str) -> Result<usize, LocalRunnerError> {
+ serde_json::to_vec(value)
+ .map(|bytes| bytes.len())
+ .map_err(|error| LocalRunnerError::invalid(format!("{label} is not serializable: {error}")))
+}
+
+fn retain_buffered_message_bytes(current: usize, incoming: usize) -> Option<usize> {
+ current
+ .checked_add(incoming)
+ .filter(|total| *total <= MAX_BUFFERED_MESSAGE_BYTES)
+}
+
fn pending_tool_request_size(
parts: impl IntoIterator<Item = usize>,
) -> Result<usize, LocalRunnerError> {
@@ -1002,6 +1444,29 @@ fn retain_pending_tool_request_bytes(
})
}
+fn pending_runtime_request_size(
+ rpc_id: &Value,
+ turn_id: &str,
+ method: &str,
+ params: &Value,
+ question_set: &Value,
+ option_labels: &QuestionOptionLabels,
+) -> Result<usize, LocalRunnerError> {
+ serde_json::to_vec(&(rpc_id, turn_id, method, params, question_set, option_labels))
+ .map(|encoded| encoded.len())
+ .map_err(|error| {
+ LocalRunnerError::invalid(format!(
+ "Codex pending runtime request is not serializable: {error}"
+ ))
+ })
+}
+
+fn retain_pending_runtime_request_bytes(current: usize, incoming: usize) -> Option<usize> {
+ current
+ .checked_add(incoming)
+ .filter(|total| *total <= MAX_PENDING_RUNTIME_REQUEST_BYTES)
+}
+
fn codex_dynamic_tools(
authorized_tools: impl IntoIterator<Item = AuthorizedTool>,
) -> Result<(Vec<Value>, BTreeSet<String>), LocalRunnerError> {
@@ -1079,12 +1544,31 @@ fn codex_dynamic_tools(
fn bounded_identifier(value: Option<&str>, label: &str) -> Result<String, LocalRunnerError> {
let value = value.ok_or_else(|| LocalRunnerError::invalid(format!("{label} is required")))?;
- if value.is_empty() || value.len() > 160 || value.chars().any(char::is_control) {
+ let mut characters = value.chars();
+ let valid_first = characters
+ .next()
+ .is_some_and(|character| character.is_ascii_alphanumeric());
+ let valid_rest = characters.all(|character| {
+ character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '.' | ':')
+ });
+ if value.len() > 160 || !valid_first || !valid_rest {
return Err(LocalRunnerError::invalid(format!("{label} is invalid")));
}
Ok(value.to_owned())
}
+fn bounded_provider_turn_id(value: Option<&str>) -> Result<String, LocalRunnerError> {
+ let value = value
+ .filter(|value| !value.is_empty())
+ .ok_or_else(|| LocalRunnerError::invalid("Codex turn/start omitted turn.id"))?;
+ if value.len() > 240 || value.chars().any(char::is_control) {
+ return Err(LocalRunnerError::invalid(
+ "Codex turn/start returned an invalid turn.id",
+ ));
+ }
+ Ok(value.to_owned())
+}
+
fn codex_tool_failure(message: &str) -> Value {
json!({
"success": false,
@@ -1160,6 +1644,17 @@ fn notification_turn_id(params: &Value) -> Option<&str> {
.filter(|value| !value.is_empty())
}
+fn request_targets_non_active_turn(
+ active_turn_id: Option<&str>,
+ settled_turn_ids: &SettledProviderTurnIds,
+ params: &Value,
+) -> bool {
+ let requested_turn_id = params.get("turnId").and_then(Value::as_str);
+ requested_turn_id.is_some_and(|requested| {
+ settled_turn_ids.contains(requested) || active_turn_id != Some(requested)
+ })
+}
+
fn latest_active_turn_id(snapshot: &Value) -> Option<String> {
snapshot
.pointer("/thread/turns")
@@ -1296,6 +1791,33 @@ fn codex_question_set(
))
}
+fn new_runtime_request_scope() -> Result<[u8; 16], LocalRunnerError> {
+ let mut scope = [0u8; 16];
+ getrandom::fill(&mut scope).map_err(|error| {
+ LocalRunnerError::invalid(format!(
+ "failed to mint Codex runtime request scope: {error}"
+ ))
+ })?;
+ Ok(scope)
+}
+
+fn scoped_runtime_request_id(
+ scope: &[u8; 16],
+ turn_id: &str,
+ provider_request_id: &str,
+ request_sequence: u64,
+) -> String {
+ let mut digest = Sha256::new();
+ digest.update(scope);
+ digest.update([0]);
+ digest.update(turn_id.as_bytes());
+ digest.update([0]);
+ digest.update(provider_request_id.as_bytes());
+ digest.update([0]);
+ digest.update(request_sequence.to_be_bytes());
+ format!("runtime-request-{:x}", digest.finalize())
+}
+
fn codex_question_response(
pending: &PendingRuntimeRequest,
response: &Value,
@@ -1436,10 +1958,12 @@ mod tests {
assert_eq!(question_set["schema"], "paperclip.question_set.v1");
let pending = PendingRuntimeRequest {
rpc_id: json!(41),
+ turn_id: "turn-1".to_owned(),
method: "item/tool/requestUserInput".to_owned(),
params: Value::Null,
question_set,
option_labels: labels,
+ retained_bytes: 0,
};
let native = codex_question_response(
&pending,
@@ -1461,6 +1985,23 @@ mod tests {
}),
)
.is_err());
+ let scope = [7u8; 16];
+ assert_eq!(
+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),
+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),
+ );
+ assert_ne!(
+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),
+ scoped_runtime_request_id(&scope, "turn-2", "41", 1),
+ );
+ assert_ne!(
+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),
+ scoped_runtime_request_id(&[8u8; 16], "turn-1", "41", 1),
+ );
+ assert_ne!(
+ scoped_runtime_request_id(&scope, "turn-1", "41", 1),
+ scoped_runtime_request_id(&scope, "turn-1", "41", 2),
+ );
assert!(codex_question_response(
&pending,
&json!({
@@ -1503,6 +2044,87 @@ mod tests {
.is_ok());
}
+ #[test]
+ fn rejects_requests_bound_to_any_non_active_turn_nonfatally() {
+ let mut turn_one_settled = SettledProviderTurnIds::default();
+ turn_one_settled.insert("turn-1".to_owned());
+ let mut turn_two_settled = SettledProviderTurnIds::default();
+ turn_two_settled.insert("turn-2".to_owned());
+ let no_settled_turns = SettledProviderTurnIds::default();
+ assert!(request_targets_non_active_turn(
+ Some("turn-2"),
+ &turn_one_settled,
+ &json!({"turnId": "turn-1"}),
+ ));
+ assert!(request_targets_non_active_turn(
+ Some("turn-2"),
+ &turn_one_settled,
+ &json!({"turnId": "turn-0"}),
+ ));
+ assert!(request_targets_non_active_turn(
+ None,
+ &turn_two_settled,
+ &json!({"turnId": "turn-1"}),
+ ));
+ assert!(request_targets_non_active_turn(
+ Some("turn-1"),
+ &turn_one_settled,
+ &json!({"turnId": "turn-1"}),
+ ));
+ assert!(!request_targets_non_active_turn(
+ Some("turn-2"),
+ &turn_one_settled,
+ &json!({"turnId": "turn-2"}),
+ ));
+ assert!(!request_targets_non_active_turn(
+ None,
+ &no_settled_turns,
+ &json!({}),
+ ));
+ }
+
+ #[test]
+ fn settled_provider_turn_history_never_evicts_exact_identities() {
+ let mut settled = SettledProviderTurnIds::default();
+ for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS {
+ assert!(settled.insert(format!("turn-{index}")));
+ }
+
+ assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS);
+ assert!(settled.contains("turn-0"));
+ assert!(settled.contains("turn-1"));
+ assert!(settled.at_capacity());
+ assert!(!settled.insert(format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}")));
+ assert!(!settled.contains(&format!("turn-{MAX_SETTLED_PROVIDER_TURN_IDS}")));
+ assert_eq!(settled.ids.len(), MAX_SETTLED_PROVIDER_TURN_IDS);
+ assert!(settled.contains("turn-0"));
+ assert!(settled.filter.is_empty());
+ }
+
+ #[test]
+ fn restored_provider_turn_identity_is_retained_in_release_builds() {
+ let mut settled = SettledProviderTurnIds::default();
+
+ settled.restore("turn-restored".to_owned()).unwrap();
+
+ assert!(settled.contains("turn-restored"));
+ }
+
+ #[test]
+ fn restores_the_complete_durable_provider_turn_ledger() {
+ let mut settled = SettledProviderTurnIds::default();
+
+ settled
+ .restore_all(
+ ["turn-older".to_owned(), "turn-latest".to_owned()],
+ DurableReplayFilter::default(),
+ )
+ .unwrap();
+
+ assert!(settled.contains("turn-older"));
+ assert!(settled.contains("turn-latest"));
+ }
+
#[test]
fn bounds_all_retained_pending_tool_request_data_in_aggregate() {
let request_bytes = pending_tool_request_size([1, 2, 3, 4]).unwrap();
@@ -1516,4 +2138,30 @@ mod tests {
assert!(retain_pending_tool_request_bytes(usize::MAX, 1).is_err());
assert!(pending_tool_request_size([usize::MAX, 1]).is_err());
}
+
+ #[test]
+ fn bounds_all_retained_runtime_request_data_in_aggregate() {
+ assert_eq!(
+ retain_pending_runtime_request_bytes(MAX_PENDING_RUNTIME_REQUEST_BYTES - 10, 10,),
+ Some(MAX_PENDING_RUNTIME_REQUEST_BYTES)
+ );
+ assert_eq!(
+ retain_pending_runtime_request_bytes(MAX_PENDING_RUNTIME_REQUEST_BYTES, 1),
+ None
+ );
+ assert_eq!(retain_pending_runtime_request_bytes(usize::MAX, 1), None);
+ }
+
+ #[test]
+ fn bounds_messages_buffered_while_waiting_for_a_response() {
+ assert_eq!(
+ retain_buffered_message_bytes(MAX_BUFFERED_MESSAGE_BYTES - 10, 10),
+ Some(MAX_BUFFERED_MESSAGE_BYTES)
+ );
+ assert_eq!(
+ retain_buffered_message_bytes(MAX_BUFFERED_MESSAGE_BYTES, 1),
+ None
+ );
+ assert_eq!(retain_buffered_message_bytes(usize::MAX, 1), None);
+ }
}
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs
index ef3ec61456..3bbf2f9311 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs
@@ -9,12 +9,14 @@ use std::time::Duration;
pub use runner::{run_durable_runner, CommandExecution, CommandExecutor, PolledEvent};
pub(crate) use state::{
- create_private_temporary_file, open_private_regular_file, redact_text, verify_private_directory,
+ create_private_temporary_file, open_private_regular_file, redact_text, sanitize_value,
+ verify_private_directory,
};
pub use state::{
Command, CommandDisposition, DurableState, DurableStateStore, EventPriority,
StoredCommandResult, StoredOutboxEvent,
};
+pub(crate) use transport::current_unix_ms;
pub const PROTOCOL: &str = "paperclip.runner";
pub const PROTOCOL_VERSION: u64 = 1;
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs
index 50c6df265c..0e27f28f3c 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs
@@ -1078,7 +1078,23 @@ fn sensitive_key(key: &str) -> bool {
.any(|needle| normalized.contains(needle))
}
-fn sanitize_value(value: &Value) -> Value {
+fn protocol_authorization_boundary(key: &str, value: &Value) -> bool {
+ key.eq_ignore_ascii_case("authorizationBoundary")
+ && value.as_str().is_some_and(|boundary| {
+ matches!(
+ boundary,
+ "company"
+ | "actor"
+ | "active_task"
+ | "grant"
+ | "governed_action"
+ | "lock"
+ | "revision"
+ )
+ })
+}
+
+pub(crate) fn sanitize_value(value: &Value) -> Value {
match value {
Value::Object(object) => Value::Object(
object
@@ -1086,7 +1102,9 @@ fn sanitize_value(value: &Value) -> Value {
.map(|(key, value)| {
(
key.clone(),
- if sensitive_key(key) {
+ if protocol_authorization_boundary(key, value) {
+ value.clone()
+ } else if sensitive_key(key) {
Value::String("[REDACTED]".to_owned())
} else {
sanitize_value(value)
@@ -1323,6 +1341,21 @@ mod tests {
);
}
+ #[test]
+ fn protocol_authorization_boundary_is_not_redacted_as_a_credential() {
+ let sanitized = sanitize_value(&json!({
+ "authorizationBoundary": "active_task",
+ "nested": {"authorizationBoundary": "Bearer secret-value"},
+ "authorization": "Bearer secret-value",
+ }));
+ assert_eq!(sanitized["authorizationBoundary"], json!("active_task"));
+ assert_eq!(sanitized["authorization"], json!("[REDACTED]"));
+ assert_eq!(
+ sanitized["nested"]["authorizationBoundary"],
+ json!("[REDACTED]")
+ );
+ }
+
#[test]
fn outbox_reserves_capacity_for_p0_and_bounds_frames() {
let mut bounds_config = config(PathBuf::from("unused"));
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
index 860362da03..5c037db769 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
@@ -11,17 +11,69 @@ use std::os::unix::fs::PermissionsExt;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
-use crate::codex_provider::{CodexProvider, CodexProviderConfig, CodexProviderEvent};
+use crate::codex_provider::{
+ CodexProvider, CodexProviderConfig, CodexProviderEvent, RejectedAcceptedTurn,
+ MAX_SETTLED_PROVIDER_TURN_IDS,
+};
use crate::durable::{
- create_private_temporary_file, open_private_regular_file, verify_private_directory, Command,
- CommandExecution, CommandExecutor, DurableRunnerError, EventPriority, PolledEvent,
+ create_private_temporary_file, current_unix_ms, open_private_regular_file, sanitize_value,
+ verify_private_directory, Command, CommandExecution, CommandExecutor, DurableRunnerConfig,
+ DurableRunnerError, EventPriority, PolledEvent,
+};
+use crate::provider_bridge::{
+ authorized_tool_catalog_digest, semantic_value_digest, AuthorizedToolSet, DurableReplayFilter,
+ PendingToolCall, ProviderBridgeError, ProviderToolBridge, ToolResult, MAX_PENDING_CALLS,
+ TOOL_SET_SCHEMA,
+};
+use crate::provider_events::{
+ normalize_codex_notification, normalized_codex_terminal_event_type, NormalizedProviderEvent,
};
-use crate::provider_events::{normalize_codex_notification, NormalizedProviderEvent};
const PROVIDER_STATE_SCHEMA: &str = "paperclip.runner.codex-provider-state.v1";
const PROVIDER_STATE_FILE: &str = "codex-provider-state.json";
-const MAX_PROVIDER_STATE_BYTES: u64 = 2 * 1024 * 1024;
+const MAX_PROVIDER_STATE_BYTES: u64 = 16 * 1024 * 1024;
const MAX_EVENTS_PER_POLL: usize = 128;
+// One accepted semantic call can produce an input and a result event. Normal
+// traffic cannot consume the additional capacity required to diagnose a
+// receipt-limit stop, settle every retained call, and record the provider plus
+// run terminal events.
+const MAX_REGULAR_QUEUED_PROVIDER_EVENTS: usize = 2 * MAX_PENDING_CALLS + 3;
+const MAX_RECEIPT_LIMIT_TERMINAL_RESERVE: usize = MAX_PENDING_CALLS + 4;
+// During a receipt-limit stop, provider polling continues even when older
+// events remain unacknowledged so an already-buffered authoritative terminal
+// wins over the deadline fallback. Reserve one complete poll of cleanup events
+// in addition to the semantic-result and terminal envelopes.
+const MAX_TERMINAL_SETTLEMENT_EVENTS: usize = MAX_PENDING_CALLS + MAX_EVENTS_PER_POLL + 4;
+const MAX_QUEUED_PROVIDER_EVENTS: usize =
+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS + MAX_TERMINAL_SETTLEMENT_EVENTS;
+const MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS: u8 = 3;
+const RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS: u64 = 2_000;
+const RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS: u64 = 30_000;
+
+fn receipt_limit_deadline_after(timeout_ms: u64) -> Result<u64, DurableRunnerError> {
+ current_unix_ms()?.checked_add(timeout_ms).ok_or_else(|| {
+ DurableRunnerError::invalid("Codex receipt-limit interruption deadline overflowed")
+ })
+}
+
+#[derive(Clone, Debug)]
+struct ProviderEventIdentity {
+ run_id: String,
+ normalized_session_id: String,
+ turn_id: String,
+ item_id: String,
+}
+
+impl ProviderEventIdentity {
+ fn from_config(config: &DurableRunnerConfig) -> Self {
+ Self {
+ run_id: config.run_id.clone(),
+ normalized_session_id: config.normalized_session_id.clone(),
+ turn_id: config.turn_id.clone(),
+ item_id: config.item_id.clone(),
+ }
+ }
+}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase")]
@@ -70,6 +122,106 @@ fn completion_contract(
Ok(Some(binding))
}
+fn authorized_tool_set(payload: &Value) -> Result<AuthorizedToolSet, DurableRunnerError> {
+ if let Some(value) = payload.get("authorizedTools") {
+ return serde_json::from_value(value.clone()).map_err(|error| {
+ DurableRunnerError::invalid(format!("run.prepare authorizedTools is invalid: {error}"))
+ });
+ }
+ let operations = Vec::new();
+ let catalog_digest = authorized_tool_catalog_digest(&operations).map_err(|error| {
+ DurableRunnerError::invalid(format!("empty authorized tool set is invalid: {error}"))
+ })?;
+ Ok(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest,
+ operations,
+ })
+}
+
+fn semantic_correlation(identity: &ProviderEventIdentity) -> Value {
+ json!({
+ "runId": identity.run_id,
+ "normalizedSessionId": identity.normalized_session_id,
+ "turnId": identity.turn_id,
+ "itemId": identity.item_id,
+ })
+}
+
+fn semantic_input_event(
+ identity: &ProviderEventIdentity,
+ call: &PendingToolCall,
+) -> NormalizedProviderEvent {
+ let safe_input = sanitize_value(&call.input);
+ NormalizedProviderEvent {
+ event_type: "semantic_tool.input".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "semantic_tool": {
+ "schema": "paperclip.prp.semantic_tool.v1",
+ "schemaVersion": 1,
+ "phase": "input",
+ "operationId": call.operation_id,
+ "callId": call.call_id,
+ "correlation": semantic_correlation(identity),
+ "idempotencyKey": Value::Null,
+ "content": {
+ "digest": semantic_value_digest(&safe_input),
+ "redactionDisposition": "digest_only",
+ "references": [],
+ },
+ "input": safe_input,
+ },
+ }),
+ }
+}
+
+fn semantic_result_event(
+ identity: &ProviderEventIdentity,
+ result: &ToolResult,
+) -> NormalizedProviderEvent {
+ let safe_result = sanitize_value(&result.result);
+ let envelope = safe_result
+ .get("resultReceipt")
+ .filter(|receipt| {
+ receipt.get("schema").and_then(Value::as_str)
+ == Some("paperclip.prp.semantic_tool.v1")
+ && receipt.get("phase").and_then(Value::as_str) == Some("result")
+ && receipt.get("operationId").and_then(Value::as_str)
+ == Some(result.operation_id.as_str())
+ && receipt.get("callId").and_then(Value::as_str) == Some(result.call_id.as_str())
+ && receipt.get("correlation") == Some(&semantic_correlation(identity))
+ })
+ .cloned()
+ .unwrap_or_else(|| {
+ json!({
+ "schema": "paperclip.prp.semantic_tool.v1",
+ "schemaVersion": 1,
+ "phase": "result",
+ "operationId": result.operation_id,
+ "callId": result.call_id,
+ "correlation": semantic_correlation(identity),
+ "idempotencyKey": Value::Null,
+ "content": {
+ "digest": semantic_value_digest(&safe_result),
+ "redactionDisposition": "digest_only",
+ "references": [],
+ },
+ "outcome": if result.is_error { "failed" } else { "succeeded" },
+ "code": if result.is_error { "semantic_tool_failed" } else { "semantic_tool_succeeded" },
+ "retryable": false,
+ "authorizationBoundary": "active_task",
+ "operationReceiptId": format!("operation_{}", result.call_id),
+ })
+ });
+ NormalizedProviderEvent {
+ event_type: "semantic_tool.result".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({"semantic_tool": envelope}),
+ }
+}
+
fn terminal_events(state: &CodexProviderState, event_type: &str) -> Vec<NormalizedProviderEvent> {
let Some(contract) = state.completion_contract.as_ref() else {
return Vec::new();
@@ -158,6 +310,8 @@ struct CodexProviderState {
#[serde(default)]
completion_contract: Option<CompletionContractBinding>,
#[serde(default)]
+ tool_bridge: ProviderToolBridge,
+ #[serde(default)]
thread_id: Option<String>,
#[serde(default)]
provider_session_id: Option<String>,
@@ -173,17 +327,69 @@ struct CodexProviderState {
completed_turn_process_generation: Option<u64>,
#[serde(default)]
completed_provider_turn_id: Option<String>,
+ // Unlike the live provider process, the durable run survives restarts.
+ // Terminal identities stay exact for one process-generation epoch. At the
+ // bound, an idle process is reaped before this epoch is rotated.
+ #[serde(default)]
+ settled_provider_turn_ids: std::collections::BTreeSet<String>,
+ #[serde(default)]
+ settled_provider_turn_filter: DurableReplayFilter,
+ #[serde(default)]
+ receipt_limit_diagnostic_emitted: bool,
+ #[serde(default)]
+ receipt_limit_interrupt_pending: bool,
+ #[serde(default)]
+ receipt_limit_interrupt_accepted: bool,
+ #[serde(default)]
+ receipt_limit_interrupt_attempts: u8,
+ #[serde(default)]
+ receipt_limit_interrupt_deadline_unix_ms: Option<u64>,
last_agent_message: Option<String>,
#[serde(default)]
pending_events: VecDeque<PolledEvent>,
+ #[serde(default)]
+ queued_events: VecDeque<PolledEvent>,
#[serde(default = "initial_provider_event_seq")]
next_provider_event_seq: u64,
}
+#[derive(Debug, PartialEq)]
+enum ToolCallAdmission {
+ CompletedReplay(ToolResult),
+ PendingReplay,
+ Pending(PendingToolCall),
+}
+
+fn settled_provider_turn_contains(
+ identities: &std::collections::BTreeSet<String>,
+ _filter: &DurableReplayFilter,
+ provider_turn_id: &str,
+) -> bool {
+ identities.contains(provider_turn_id)
+}
+
+fn remember_settled_provider_turn(
+ identities: &mut std::collections::BTreeSet<String>,
+ _filter: &mut DurableReplayFilter,
+ provider_turn_id: String,
+) -> Result<(), DurableRunnerError> {
+ if identities.contains(&provider_turn_id) {
+ return Ok(());
+ }
+ if identities.len() >= MAX_SETTLED_PROVIDER_TURN_IDS {
+ return Err(DurableRunnerError::invalid(
+ "Codex provider turn identity epoch reached its exact capacity",
+ ));
+ }
+ identities.insert(provider_turn_id);
+ Ok(())
+}
+
impl CodexProviderState {
fn new(
config: CodexProviderConfig,
completion_contract: Option<CompletionContractBinding>,
+ tool_bridge: ProviderToolBridge,
) -> Self {
let thread_id = config.provider_session_id.clone();
Self {
@@ -191,6 +397,7 @@ impl CodexProviderState {
lifecycle: "prepared".to_owned(),
config,
completion_contract,
+ tool_bridge,
thread_id,
provider_session_id: None,
active_provider_turn_id: None,
@@ -199,8 +406,16 @@ impl CodexProviderState {
provider_process_generation: 0,
completed_turn_process_generation: None,
completed_provider_turn_id: None,
+ settled_provider_turn_ids: std::collections::BTreeSet::new(),
+ settled_provider_turn_filter: DurableReplayFilter::default(),
+ receipt_limit_diagnostic_emitted: false,
+ receipt_limit_interrupt_pending: false,
+ receipt_limit_interrupt_accepted: false,
+ receipt_limit_interrupt_attempts: 0,
+ receipt_limit_interrupt_deadline_unix_ms: None,
last_agent_message: None,
pending_events: VecDeque::new(),
+ queued_events: VecDeque::new(),
next_provider_event_seq: initial_provider_event_seq(),
}
}
@@ -209,6 +424,9 @@ impl CodexProviderState {
self.config
.validate()
.map_err(|error| DurableRunnerError::invalid(error.to_string()))?;
+ self.tool_bridge.validate_recovered().map_err(|error| {
+ DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}"))
+ })?;
let mut pending_event_ids = HashSet::new();
if self.schema != PROVIDER_STATE_SCHEMA
|| !matches!(
@@ -262,24 +480,59 @@ impl CodexProviderState {
|| (!self.completed_turn_authoritative
&& (self.completed_turn_process_generation.is_some()
|| self.completed_provider_turn_id.is_some()))
+ || self.settled_provider_turn_ids.len() > MAX_SETTLED_PROVIDER_TURN_IDS
+ || self.settled_provider_turn_filter.validate().is_err()
+ || self
+ .settled_provider_turn_ids
+ .iter()
+ .any(|provider_turn_id| {
+ provider_turn_id.is_empty()
+ || provider_turn_id.len() > 240
+ || provider_turn_id.chars().any(char::is_control)
+ })
+ || self
+ .active_provider_turn_id
+ .as_ref()
+ .is_some_and(|provider_turn_id| {
+ settled_provider_turn_contains(
+ &self.settled_provider_turn_ids,
+ &self.settled_provider_turn_filter,
+ provider_turn_id,
+ )
+ })
|| self
.completed_turn_process_generation
.is_some_and(|generation| generation > self.provider_process_generation)
+ || (self.receipt_limit_diagnostic_emitted && self.active_provider_turn_id.is_none())
+ || (self.receipt_limit_interrupt_pending
+ && (!self.receipt_limit_diagnostic_emitted
+ || self.active_provider_turn_id.is_none()))
+ || (self.receipt_limit_interrupt_accepted && !self.receipt_limit_interrupt_pending)
+ || self.receipt_limit_interrupt_attempts > MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS
+ || (!self.receipt_limit_interrupt_pending && self.receipt_limit_interrupt_attempts != 0)
+ || self
+ .receipt_limit_interrupt_deadline_unix_ms
+ .is_some_and(|deadline| deadline == 0 || !self.receipt_limit_interrupt_pending)
|| (matches!(
self.lifecycle.as_str(),
"prepared" | "session_open" | "closed"
) && self.active_provider_turn_id.is_some())
|| self.next_provider_event_seq == 0
|| self.pending_events.len() > MAX_EVENTS_PER_POLL + 3
- || self.pending_events.iter().any(|event| {
- provider_event_sequence(&event.executor_event_id)
- .is_none_or(|sequence| sequence >= self.next_provider_event_seq)
- || !pending_event_ids.insert(event.executor_event_id.as_str())
- || event.event_type.is_empty()
- || event.event_type.len() > 160
- || event.event_type.chars().any(char::is_control)
- || !event.payload.is_object()
- })
+ || self.queued_events.len() > MAX_QUEUED_PROVIDER_EVENTS
+ || self
+ .pending_events
+ .iter()
+ .chain(self.queued_events.iter())
+ .any(|event| {
+ provider_event_sequence(&event.executor_event_id)
+ .is_none_or(|sequence| sequence >= self.next_provider_event_seq)
+ || !pending_event_ids.insert(event.executor_event_id.as_str())
+ || event.event_type.is_empty()
+ || event.event_type.len() > 160
+ || event.event_type.chars().any(char::is_control)
+ || !event.payload.is_object()
+ })
{
return Err(DurableRunnerError::invalid(
"Codex provider state is malformed or inconsistent",
@@ -288,20 +541,131 @@ impl CodexProviderState {
Ok(())
}
- fn push_event(&mut self, event: NormalizedProviderEvent) -> Result<(), DurableRunnerError> {
+ fn push_event_with_limit(
+ &mut self,
+ event: NormalizedProviderEvent,
+ max_queued_events: usize,
+ ) -> Result<(), DurableRunnerError> {
+ let queue_event =
+ !self.queued_events.is_empty() || self.pending_events.len() >= MAX_EVENTS_PER_POLL;
+ if queue_event && self.queued_events.len() >= max_queued_events {
+ return Err(DurableRunnerError::invalid(
+ "Codex provider event backlog exceeds its durable limit",
+ ));
+ }
let sequence = self.next_provider_event_seq;
self.next_provider_event_seq = sequence
.checked_add(1)
.ok_or_else(|| DurableRunnerError::invalid("provider event sequence exhausted"))?;
- self.pending_events.push_back(PolledEvent {
+ let event = PolledEvent {
executor_event_id: provider_event_id(sequence),
event_type: event.event_type,
priority: event.priority,
payload: event.payload,
- });
+ };
+ if queue_event {
+ self.queued_events.push_back(event);
+ } else {
+ self.pending_events.push_back(event);
+ }
+ Ok(())
+ }
+
+ fn begin_receipt_limit_stop(
+ &mut self,
+ call_id: String,
+ operation_id: String,
+ deadline_unix_ms: u64,
+ ) -> Result<bool, DurableRunnerError> {
+ if self.receipt_limit_diagnostic_emitted {
+ return Ok(self.receipt_limit_interrupt_pending);
+ }
+ self.push_terminal_event(NormalizedProviderEvent {
+ event_type: "harness.diagnostic".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "provider": "codex",
+ "code": "semantic_tool_turn_receipt_limit",
+ "operationId": operation_id,
+ "callId": call_id,
+ "message": "The active provider turn reached its durable semantic-tool receipt limit and was interrupted",
+ "paperclipExecuted": false,
+ }),
+ })?;
+ self.receipt_limit_diagnostic_emitted = true;
+ self.receipt_limit_interrupt_pending = true;
+ self.receipt_limit_interrupt_attempts = 0;
+ self.receipt_limit_interrupt_deadline_unix_ms = Some(deadline_unix_ms);
+ Ok(true)
+ }
+
+ fn record_receipt_limit_interrupt_attempt(&mut self) -> Result<(), DurableRunnerError> {
+ if self.receipt_limit_interrupt_attempts >= MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS {
+ return Err(DurableRunnerError::invalid(
+ "Codex receipt-limit interruption exceeded its durable retry bound",
+ ));
+ }
+ self.receipt_limit_interrupt_attempts += 1;
Ok(())
}
+ fn mark_receipt_limit_interrupt_accepted(&mut self, accepted_deadline_unix_ms: u64) {
+ // Provider restoration can reconcile the active turn as already
+ // settled while an interruption command is in flight. In that case
+ // `interrupt_turn` returns `already_settled` and recovery has already
+ // cleared the durable retry marker. Do not recreate an accepted state
+ // without a pending interruption or active turn.
+ if !self.receipt_limit_interrupt_pending || self.active_provider_turn_id.is_none() {
+ return;
+ }
+ if !self.receipt_limit_interrupt_accepted {
+ self.receipt_limit_interrupt_deadline_unix_ms = Some(
+ self.receipt_limit_interrupt_deadline_unix_ms
+ .unwrap_or_default()
+ .max(accepted_deadline_unix_ms),
+ );
+ }
+ self.receipt_limit_interrupt_accepted = true;
+ }
+
+ fn push_event(&mut self, event: NormalizedProviderEvent) -> Result<(), DurableRunnerError> {
+ self.push_event_with_limit(event, MAX_REGULAR_QUEUED_PROVIDER_EVENTS)
+ }
+
+ fn push_terminal_event(
+ &mut self,
+ event: NormalizedProviderEvent,
+ ) -> Result<(), DurableRunnerError> {
+ self.push_event_with_limit(event, MAX_QUEUED_PROVIDER_EVENTS)
+ }
+
+ fn push_receipt_limit_cleanup_event(
+ &mut self,
+ event: NormalizedProviderEvent,
+ ) -> Result<(), DurableRunnerError> {
+ let queue_event =
+ !self.queued_events.is_empty() || self.pending_events.len() >= MAX_EVENTS_PER_POLL;
+ if queue_event
+ && self.queued_events.len()
+ >= MAX_QUEUED_PROVIDER_EVENTS - MAX_RECEIPT_LIMIT_TERMINAL_RESERVE
+ {
+ // Continue draining the provider so an authoritative terminal can
+ // still be observed, but never let cleanup chatter consume the
+ // semantic-result and terminal-event reserve.
+ return Ok(());
+ }
+ self.push_terminal_event(event)
+ }
+
+ fn refill_pending_events(&mut self) {
+ while self.pending_events.len() < MAX_EVENTS_PER_POLL {
+ let Some(event) = self.queued_events.pop_front() else {
+ break;
+ };
+ self.pending_events.push_back(event);
+ }
+ }
+
fn extend_events(
&mut self,
events: impl IntoIterator<Item = NormalizedProviderEvent>,
@@ -312,6 +676,43 @@ impl CodexProviderState {
Ok(())
}
+ fn admit_tool_call(
+ &mut self,
+ call_id: &str,
+ operation_id: &str,
+ input: &Value,
+ ) -> Result<ToolCallAdmission, ProviderBridgeError> {
+ if let Some(result) = self
+ .tool_bridge
+ .replay_result(call_id, operation_id, input)?
+ {
+ // An exact completed replay is a transport retry. Its input and
+ // result receipts are already durable, so recording another event
+ // would make an otherwise idempotent replay consume bounded event
+ // capacity and could prevent returning the stored result.
+ return Ok(ToolCallAdmission::CompletedReplay(result));
+ }
+
+ let pending_replay = self
+ .tool_bridge
+ .pending_calls()
+ .any(|pending| pending.call_id == call_id);
+ let call = self.tool_bridge.begin_call(
+ call_id.to_owned(),
+ operation_id.to_owned(),
+ input.clone(),
+ )?;
+ if pending_replay {
+ // The first input receipt is already durable. An exact pending
+ // replay is only the provider re-establishing its request after a
+ // process restart; appending another event would make the retry
+ // consume bounded backlog capacity without adding information.
+ Ok(ToolCallAdmission::PendingReplay)
+ } else {
+ Ok(ToolCallAdmission::Pending(call))
+ }
+ }
+
fn reconcile_active_provider_turn(&mut self, active_provider_turn_id: Option<String>) {
self.active_provider_turn_id = active_provider_turn_id;
if self.active_provider_turn_id.is_some() {
@@ -330,12 +731,53 @@ impl CodexProviderState {
"session_open".to_owned()
};
}
+
+ fn settle_active_provider_turn_identity(&mut self) -> Result<(), DurableRunnerError> {
+ let provider_turn_id = self.active_provider_turn_id.clone().ok_or_else(|| {
+ DurableRunnerError::invalid("Codex terminal omitted its active provider turn identity")
+ })?;
+ remember_settled_provider_turn(
+ &mut self.settled_provider_turn_ids,
+ &mut self.settled_provider_turn_filter,
+ provider_turn_id,
+ )?;
+ Ok(())
+ }
+
+ fn recovered_settled_provider_turn_ids(
+ &self,
+ ) -> Result<(std::collections::BTreeSet<String>, DurableReplayFilter), DurableRunnerError> {
+ let mut settled_provider_turn_ids = self.settled_provider_turn_ids.clone();
+ let mut settled_provider_turn_filter = self.settled_provider_turn_filter.clone();
+ // State written before the durable set was introduced retained only
+ // the latest completed identity. Fold that legacy authority into the
+ // new ledger before the provider is allowed to accept replacement work.
+ if let Some(provider_turn_id) = self.completed_provider_turn_id.clone() {
+ remember_settled_provider_turn(
+ &mut settled_provider_turn_ids,
+ &mut settled_provider_turn_filter,
+ provider_turn_id,
+ )?;
+ }
+ Ok((settled_provider_turn_ids, settled_provider_turn_filter))
+ }
+
+ fn extend_terminal_events(
+ &mut self,
+ events: impl IntoIterator<Item = NormalizedProviderEvent>,
+ ) -> Result<(), DurableRunnerError> {
+ for event in events {
+ self.push_terminal_event(event)?;
+ }
+ Ok(())
+ }
}
pub struct CodexCommandExecutor {
state_dir: PathBuf,
state: Option<CodexProviderState>,
provider: Option<CodexProvider>,
+ event_identity: Option<ProviderEventIdentity>,
restore_checked: bool,
}
@@ -345,10 +787,17 @@ impl CodexCommandExecutor {
state_dir: state_dir.into(),
state: None,
provider: None,
+ event_identity: None,
restore_checked: false,
}
}
+ pub fn with_runner_config(state_dir: impl Into<PathBuf>, config: &DurableRunnerConfig) -> Self {
+ let mut executor = Self::new(state_dir);
+ executor.event_identity = Some(ProviderEventIdentity::from_config(config));
+ executor
+ }
+
fn state_path(&self) -> PathBuf {
self.state_dir.join(PROVIDER_STATE_FILE)
}
@@ -380,9 +829,14 @@ impl CodexCommandExecutor {
file.read_to_end(&mut input).map_err(|error| {
DurableRunnerError::invalid(format!("failed to read Codex provider state: {error}"))
})?;
- let state: CodexProviderState = serde_json::from_slice(&input).map_err(|error| {
+ let mut state: CodexProviderState = serde_json::from_slice(&input).map_err(|error| {
DurableRunnerError::invalid(format!("Codex provider state is malformed: {error}"))
})?;
+ state.tool_bridge.attach_existing_run().map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "Codex semantic tool state could not be reattached: {error}"
+ ))
+ })?;
state.validate()?;
self.state = Some(state);
self.restore_provider_if_needed()
@@ -413,15 +867,139 @@ impl CodexCommandExecutor {
let completed_turn_process_generation = state.completed_turn_process_generation;
let completed_provider_turn_id = state.completed_provider_turn_id.clone();
let ambiguous_turn_start_pending = state.ambiguous_turn_start_pending;
- let mut provider = CodexProvider::start_for_generation(
+ let tool_replay_history_blocks_admission =
+ state.tool_bridge.replay_history_blocks_admission();
+ let tool_receipt_epoch_has_active_receipts = state.tool_bridge.has_active_receipts();
+ let (settled_provider_turn_ids, settled_provider_turn_filter) =
+ state.recovered_settled_provider_turn_ids()?;
+ let provider_epoch_requires_rollover = settled_provider_turn_ids.len()
+ >= MAX_SETTLED_PROVIDER_TURN_IDS
+ || !settled_provider_turn_filter.is_empty();
+ let mut provider = CodexProvider::start_with_tools_for_generation(
&state.config,
+ state.tool_bridge.authorized_tools().cloned(),
Some(&thread_id),
process_generation,
)
.map_err(|error| {
DurableRunnerError::invalid(format!("failed to resume Codex provider: {error}"))
})?;
+ provider.enable_durable_tool_call_replays();
+ provider
+ .restore_settled_turn_identities(
+ settled_provider_turn_ids.iter().cloned(),
+ settled_provider_turn_filter.clone(),
+ )
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to restore Codex provider turn identities: {error}"
+ ))
+ })?;
let recovered_active_turn_id = provider.active_provider_turn_id().map(str::to_owned);
+ let legacy_epoch_is_ambiguous = (provider_epoch_requires_rollover
+ && (ambiguous_turn_start_pending || recovered_active_turn_id.is_some()))
+ || (tool_replay_history_blocks_admission
+ && (ambiguous_turn_start_pending
+ || recovered_active_turn_id.is_some()
+ || (previous_active_turn_id.is_some()
+ && tool_receipt_epoch_has_active_receipts)));
+ if legacy_epoch_is_ambiguous {
+ // A saturated legacy epoch cannot prove that recovered work can
+ // be identified and settled exactly. Reap the resumed process
+ // generation and close the run instead of risking duplicate work.
+ let provider_reported_active = recovered_active_turn_id.is_some();
+ let provider_shutdown_failed = provider.shutdown().is_err();
+ drop(provider);
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available during legacy recovery");
+ state.provider_process_generation = process_generation;
+ state.settled_provider_turn_ids = settled_provider_turn_ids;
+ state.settled_provider_turn_filter = settled_provider_turn_filter;
+ state.active_provider_turn_id = None;
+ state.ambiguous_turn_start_pending = false;
+ state.completed_turn_authoritative = false;
+ state.completed_turn_process_generation = None;
+ state.completed_provider_turn_id = None;
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
+ state.last_agent_message = None;
+ state.lifecycle = "closed".to_owned();
+ let _ = state.push_terminal_event(NormalizedProviderEvent {
+ event_type: "harness.diagnostic".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "provider": "codex",
+ "code": "legacy_provider_turn_epoch_ambiguous",
+ "message": "Codex recovery could not safely identify and settle active work from a saturated legacy replay epoch; Paperclip terminated the provider and closed the durable run",
+ "paperclipAccepted": false,
+ "providerReportedActive": provider_reported_active,
+ "ambiguousStartPending": ambiguous_turn_start_pending,
+ "providerShutdownFailed": provider_shutdown_failed,
+ }),
+ });
+ self.save_state()?;
+ return Ok(());
+ }
+ if let Some(reused_provider_turn_id) = recovered_active_turn_id
+ .as_ref()
+ .filter(|provider_turn_id| {
+ settled_provider_turn_contains(
+ &settled_provider_turn_ids,
+ &settled_provider_turn_filter,
+ provider_turn_id,
+ )
+ })
+ .cloned()
+ {
+ // The durable terminal ledger is authoritative. A resumed provider
+ // that reports one of those identities as active is contradictory
+ // and may still be mutating the workspace. Terminate that process
+ // generation and persist the run closed before exposing recovery
+ // to the controller; otherwise this path would reopen settled work.
+ let provider_shutdown_failed = provider.shutdown().is_err();
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available during recovery");
+ state.provider_process_generation = process_generation;
+ state.settled_provider_turn_ids = settled_provider_turn_ids;
+ state.settled_provider_turn_filter = settled_provider_turn_filter;
+ state.active_provider_turn_id = None;
+ state.ambiguous_turn_start_pending = false;
+ state.completed_turn_authoritative = false;
+ state.completed_turn_process_generation = None;
+ state.completed_provider_turn_id = None;
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
+ state.last_agent_message = None;
+ state.lifecycle = "closed".to_owned();
+ // Closing the provider is the safety boundary. Preserve that
+ // durable transition even when an already-full event backlog has
+ // no room for an additional diagnostic.
+ let _ = state.push_terminal_event(NormalizedProviderEvent {
+ event_type: "harness.diagnostic".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "provider": "codex",
+ "code": "provider_turn_identity_reused",
+ "providerTurnId": reused_provider_turn_id,
+ "message": "Codex recovery reported a previously settled turn identity as active; Paperclip terminated the provider and closed the durable run",
+ "paperclipAccepted": false,
+ "providerReportedActive": true,
+ "providerShutdownFailed": provider_shutdown_failed,
+ }),
+ });
+ self.save_state()?;
+ return Ok(());
+ }
if ambiguous_turn_start_pending {
let recovered_turn_id = recovered_active_turn_id.as_deref().ok_or_else(|| {
DurableRunnerError::invalid(
@@ -434,37 +1012,99 @@ impl CodexCommandExecutor {
));
}
}
- provider.restore_completed_turn_authority(
- completed_turn_authoritative
- && recovered_active_turn_id.is_none()
- && !ambiguous_turn_start_pending,
- completed_turn_process_generation,
- completed_provider_turn_id.as_deref(),
- );
+ provider
+ .restore_completed_turn_authority(
+ completed_turn_authoritative
+ && recovered_active_turn_id.is_none()
+ && !ambiguous_turn_start_pending,
+ completed_turn_process_generation,
+ completed_provider_turn_id.as_deref(),
+ )
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to restore Codex completion authority: {error}"
+ ))
+ })?;
self.provider = Some(provider);
- self.state
- .as_mut()
- .expect("Codex state remains available during recovery")
- .provider_process_generation = process_generation;
+ {
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available during recovery");
+ state.provider_process_generation = process_generation;
+ state.settled_provider_turn_ids = settled_provider_turn_ids;
+ state.settled_provider_turn_filter = settled_provider_turn_filter;
+ }
if provider_had_exited
|| ambiguous_turn_start_pending
|| recovered_active_turn_id != previous_active_turn_id
{
+ let recovered_turn_ended =
+ previous_active_turn_id.is_some() && recovered_active_turn_id.is_none();
+ let identity = self.event_identity.clone();
let state = self
.state
.as_mut()
.expect("Codex state remains available during recovery");
+ if recovered_turn_ended {
+ if !provider_epoch_requires_rollover {
+ state.settle_active_provider_turn_identity()?;
+ }
+ let settled = state
+ .tool_bridge
+ .settle_turn("provider_turn_terminated")
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to settle semantic tools during recovery: {error}"
+ ))
+ })?;
+ if !settled.is_empty() {
+ let identity = identity.as_ref().ok_or_else(|| {
+ DurableRunnerError::invalid(
+ "Codex semantic tool events require the durable runner identity",
+ )
+ })?;
+ for result in settled {
+ state.push_terminal_event(semantic_result_event(identity, &result))?;
+ }
+ }
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
+ }
state.reconcile_active_provider_turn(recovered_active_turn_id.clone());
- state.push_event(NormalizedProviderEvent {
+ let reconciled = NormalizedProviderEvent {
event_type: "session.reconciled".to_owned(),
priority: EventPriority::P0,
payload: json!({
"provider": "codex",
"providerSessionId": thread_id,
- "previousProviderTurnId": previous_active_turn_id,
- "activeProviderTurnId": recovered_active_turn_id,
+ "previousProviderTurnId": previous_active_turn_id.clone(),
+ "activeProviderTurnId": recovered_active_turn_id.clone(),
}),
- })?;
+ };
+ if recovered_turn_ended {
+ state.push_terminal_event(reconciled)?;
+ // A turn that disappeared while runnerd was offline has no
+ // trustworthy success notification to replay. Terminate it
+ // conservatively so the controller cannot wait forever or
+ // mistake an unknown outcome for success.
+ state.push_terminal_event(NormalizedProviderEvent {
+ event_type: "turn.failed".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "provider": "codex",
+ "providerTurnId": previous_active_turn_id,
+ "status": "failed",
+ "providerTerminalObserved": false,
+ }),
+ })?;
+ state.extend_terminal_events(terminal_events(state, "turn.failed"))?;
+ } else {
+ state.push_event(reconciled)?;
+ }
}
self.save_state()?;
Ok(())
@@ -502,7 +1142,7 @@ impl CodexCommandExecutor {
})?;
if bytes.len() as u64 > MAX_PROVIDER_STATE_BYTES {
return Err(DurableRunnerError::invalid(
- "Codex provider state exceeds the 2 MiB limit",
+ "Codex provider state exceeds the 16 MiB limit",
));
}
let (temporary, mut file) = create_private_temporary_file(&path)?;
@@ -542,7 +1182,8 @@ impl CodexCommandExecutor {
.validate()
.map_err(|error| DurableRunnerError::invalid(error.to_string()))?;
let completion_contract = completion_contract(payload)?;
- if let Some(state) = &self.state {
+ let tool_set = authorized_tool_set(payload)?;
+ if let Some(state) = self.state.as_mut() {
if state.config != config || state.completion_contract != completion_contract {
return Err(DurableRunnerError::invalid(
"Codex provider or completion contract changed across the durable run",
@@ -553,8 +1194,33 @@ impl CodexCommandExecutor {
"Codex provider session is already closed",
));
}
+ if state.tool_bridge.has_catalog() {
+ state
+ .tool_bridge
+ .verify_tool_set(&tool_set)
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "run.prepare tool contract changed: {error}"
+ ))
+ })?;
+ } else {
+ state.tool_bridge.prepare(tool_set).map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "run.prepare tool contract rejected: {error}"
+ ))
+ })?;
+ self.save_state()?;
+ }
} else {
- self.state = Some(CodexProviderState::new(config, completion_contract));
+ let mut tool_bridge = ProviderToolBridge::default();
+ tool_bridge.prepare(tool_set).map_err(|error| {
+ DurableRunnerError::invalid(format!("run.prepare tool contract rejected: {error}"))
+ })?;
+ self.state = Some(CodexProviderState::new(
+ config,
+ completion_contract,
+ tool_bridge,
+ ));
self.save_state()?;
}
Ok(CommandExecution::result(json!({
@@ -579,24 +1245,50 @@ impl CodexCommandExecutor {
.provider_process_generation
.checked_add(1)
.ok_or_else(|| DurableRunnerError::invalid("Codex process generation exhausted"))?;
- let mut provider = CodexProvider::start_for_generation(
+ let (settled_provider_turn_ids, settled_provider_turn_filter) =
+ state.recovered_settled_provider_turn_ids()?;
+ let mut provider = CodexProvider::start_with_tools_for_generation(
&state.config,
+ state.tool_bridge.authorized_tools().cloned(),
state.thread_id.as_deref(),
process_generation,
)
.map_err(|error| {
DurableRunnerError::invalid(format!("failed to start Codex provider: {error}"))
})?;
- provider.restore_completed_turn_authority(
- state.completed_turn_authoritative && provider.active_provider_turn_id().is_none(),
- state.completed_turn_process_generation,
- state.completed_provider_turn_id.as_deref(),
- );
+ provider.enable_durable_tool_call_replays();
+ provider
+ .restore_settled_turn_identities(
+ settled_provider_turn_ids.iter().cloned(),
+ settled_provider_turn_filter.clone(),
+ )
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to restore Codex provider turn identities: {error}"
+ ))
+ })?;
+ provider
+ .restore_completed_turn_authority(
+ state.completed_turn_authoritative
+ && provider.active_provider_turn_id().is_none(),
+ state.completed_turn_process_generation,
+ state.completed_provider_turn_id.as_deref(),
+ )
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to restore Codex completion authority: {error}"
+ ))
+ })?;
self.provider = Some(provider);
- self.state
- .as_mut()
- .expect("Codex state remains available after provider start")
- .provider_process_generation = process_generation;
+ {
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available after provider start");
+ state.provider_process_generation = process_generation;
+ state.settled_provider_turn_ids = settled_provider_turn_ids;
+ state.settled_provider_turn_filter = settled_provider_turn_filter;
+ }
self.save_state()?;
}
self.provider
@@ -604,6 +1296,21 @@ impl CodexCommandExecutor {
.ok_or_else(|| DurableRunnerError::invalid("Codex provider is unavailable"))
}
+ fn verify_attached_tools(&self, payload: &Value) -> Result<(), DurableRunnerError> {
+ if payload.get("authorizedTools").is_none() {
+ return Ok(());
+ }
+ let tool_set = authorized_tool_set(payload)?;
+ self.state
+ .as_ref()
+ .ok_or_else(|| DurableRunnerError::invalid("Codex provider has not been prepared"))?
+ .tool_bridge
+ .verify_tool_set(&tool_set)
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!("run.attach tool contract changed: {error}"))
+ })
+ }
+
fn open_session(&mut self) -> Result<CommandExecution, DurableRunnerError> {
self.restore_provider_if_needed()?;
if self
@@ -636,6 +1343,11 @@ impl CodexCommandExecutor {
state.thread_id = Some(thread_id.clone());
state.provider_session_id = provider_session_id.clone();
state.active_provider_turn_id = None;
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
state.lifecycle = "session_open".to_owned();
state.config.provider_version.clone()
};
@@ -667,47 +1379,244 @@ impl CodexCommandExecutor {
})
}
- fn start_turn(&mut self, payload: &Value) -> Result<CommandExecution, DurableRunnerError> {
- self.restore_provider_if_needed()?;
- if self
- .state
+ fn close_after_rejected_provider_acceptance(
+ &mut self,
+ rejected_accepted_turn: &RejectedAcceptedTurn,
+ ) -> Result<(), DurableRunnerError> {
+ let provider_process_generation = self
+ .provider
.as_ref()
- .is_some_and(|state| state.active_provider_turn_id.is_some())
- {
- return Err(DurableRunnerError::invalid(
- "Codex already has an active provider turn",
- ));
+ .map(CodexProvider::process_generation);
+ // The provider has already been terminated. Drop its quarantined
+ // handle before persisting the closure so recovery can never resume
+ // work that Codex accepted without Paperclip accepting its identity.
+ self.provider = None;
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available after rejected provider acceptance");
+ if let Some(provider_process_generation) = provider_process_generation {
+ state.provider_process_generation = provider_process_generation;
}
- let text = payload
- .get("text")
- .and_then(Value::as_str)
- .ok_or_else(|| DurableRunnerError::invalid("turn.start payload.text is required"))?;
- let cwd = self
+ state.active_provider_turn_id = None;
+ state.ambiguous_turn_start_pending = false;
+ state.completed_turn_authoritative = false;
+ state.completed_turn_process_generation = None;
+ state.completed_provider_turn_id = None;
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
+ state.last_agent_message = None;
+ state.lifecycle = "closed".to_owned();
+ // Closure is the safety boundary. Preserve it even if a saturated
+ // event queue cannot retain this additional diagnostic.
+ let _ = state.push_terminal_event(NormalizedProviderEvent {
+ event_type: "harness.diagnostic".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "provider": "codex",
+ "code": match rejected_accepted_turn {
+ RejectedAcceptedTurn::ReusedIdentity(_) => "provider_turn_identity_reused",
+ RejectedAcceptedTurn::InvalidIdentity => "provider_turn_identity_invalid",
+ },
+ "providerTurnId": match rejected_accepted_turn {
+ RejectedAcceptedTurn::ReusedIdentity(provider_turn_id) => json!(provider_turn_id),
+ RejectedAcceptedTurn::InvalidIdentity => Value::Null,
+ },
+ "message": match rejected_accepted_turn {
+ RejectedAcceptedTurn::ReusedIdentity(_) => "Codex accepted work with a previously settled turn identity; Paperclip terminated the provider and closed the durable run",
+ RejectedAcceptedTurn::InvalidIdentity => "Codex accepted work without a valid bounded turn identity; Paperclip terminated the provider and closed the durable run",
+ },
+ "paperclipAccepted": false,
+ "providerAccepted": true,
+ }),
+ });
+ self.save_state()
+ }
+
+ fn rollover_provider_identity_epochs_if_needed(&mut self) -> Result<(), DurableRunnerError> {
+ let tool_rollover_required = self
.state
.as_ref()
- .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?
- .config
- .cwd
- .clone();
- self.ensure_provider()?;
- {
- let state = self
- .state
- .as_mut()
- .expect("Codex state remains available before turn/start dispatch");
- state.ambiguous_turn_start_pending = true;
+ .is_some_and(|state| state.tool_bridge.replay_history_blocks_admission());
+ let rollover_required = self.state.as_ref().is_some_and(|state| {
+ state.settled_provider_turn_ids.len() >= MAX_SETTLED_PROVIDER_TURN_IDS
+ || !state.settled_provider_turn_filter.is_empty()
+ || tool_rollover_required
+ });
+ if !rollover_required {
+ return Ok(());
}
- self.save_state()?;
- let (start_result, completion_authority_retained, ambiguous_turn_start_pending) = {
- let provider = self.ensure_provider()?;
+ let rollover_is_safe = self.state.as_ref().is_some_and(|state| {
+ state.active_provider_turn_id.is_none()
+ && !state.ambiguous_turn_start_pending
+ && state.lifecycle == "session_open"
+ });
+ if !rollover_is_safe {
+ return Err(DurableRunnerError::invalid(
+ "Codex provider identity epoch cannot rotate while work is active",
+ ));
+ }
+
+ let (restart_result, process_generation, rejected_accepted_turn) = {
+ let provider = self.provider.as_mut().ok_or_else(|| {
+ DurableRunnerError::invalid(
+ "Codex provider identity epoch cannot rotate without an attached process",
+ )
+ })?;
+ let restart_result = provider.restart_idle_identity_epoch();
+ (
+ restart_result,
+ provider.process_generation(),
+ provider.take_rejected_accepted_turn(),
+ )
+ };
+ if let Err(error) = restart_result {
+ if let Some(rejected_accepted_turn) = rejected_accepted_turn {
+ let failure_kind = match &rejected_accepted_turn {
+ RejectedAcceptedTurn::ReusedIdentity(_) => "accepted identity reuse",
+ RejectedAcceptedTurn::InvalidIdentity => "an invalid accepted identity",
+ };
+ self.close_after_rejected_provider_acceptance(&rejected_accepted_turn)?;
+ return Err(DurableRunnerError::invalid(format!(
+ "Codex identity epoch rollover failed closed after {failure_kind}: {error}"
+ )));
+ }
+ return Err(DurableRunnerError::invalid(format!(
+ "failed to rotate the completed Codex identity epoch: {error}"
+ )));
+ };
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available during identity epoch rollover");
+ state.provider_process_generation = process_generation;
+ state.settled_provider_turn_ids.clear();
+ if let Some(completed_provider_turn_id) = state.completed_provider_turn_id.clone() {
+ // The replacement process restored this still-authoritative
+ // terminal into its fresh epoch. Mirror that one tombstone in the
+ // durable ledger until accepting replacement work revokes the
+ // completion authority.
+ state
+ .settled_provider_turn_ids
+ .insert(completed_provider_turn_id);
+ }
+ state.settled_provider_turn_filter = DurableReplayFilter::default();
+ if tool_rollover_required {
+ state
+ .tool_bridge
+ .rollover_replay_epoch_after_provider_restart()
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to rotate Codex semantic tool replay authority: {error}"
+ ))
+ })?;
+ }
+ self.save_state()?;
+ Ok(())
+ }
+
+ fn start_turn(&mut self, payload: &Value) -> Result<CommandExecution, DurableRunnerError> {
+ self.restore_provider_if_needed()?;
+ if self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.lifecycle == "closed")
+ {
+ return Err(DurableRunnerError::invalid(
+ "Codex provider session is closed",
+ ));
+ }
+ if self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.active_provider_turn_id.is_some())
+ {
+ return Err(DurableRunnerError::invalid(
+ "Codex already has an active provider turn",
+ ));
+ }
+ if self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.ambiguous_turn_start_pending)
+ {
+ return Err(DurableRunnerError::invalid(
+ "Codex has an unresolved ambiguous provider turn start",
+ ));
+ }
+ if self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.queued_events.len() >= MAX_REGULAR_QUEUED_PROVIDER_EVENTS)
+ {
+ return Err(DurableRunnerError::invalid(
+ "cannot start a new Codex turn until terminal events are acknowledged",
+ ));
+ }
+ self.state
+ .as_mut()
+ .expect("Codex state remains available before turn receipt preparation")
+ .tool_bridge
+ .prepare_turn()
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "Codex semantic tool receipts could not prepare the next turn: {error}"
+ ))
+ })?;
+ self.rollover_provider_identity_epochs_if_needed()?;
+ let text = payload
+ .get("text")
+ .and_then(Value::as_str)
+ .ok_or_else(|| DurableRunnerError::invalid("turn.start payload.text is required"))?;
+ let cwd = self
+ .state
+ .as_ref()
+ .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?
+ .config
+ .cwd
+ .clone();
+ self.ensure_provider()?;
+ {
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available before turn/start dispatch");
+ state.ambiguous_turn_start_pending = true;
+ }
+ self.save_state()?;
+ let (
+ start_result,
+ completion_authority_retained,
+ ambiguous_turn_start_pending,
+ rejected_accepted_turn,
+ ) = {
+ let provider = self.ensure_provider()?;
let result = provider.start_turn(text, &cwd);
(
result,
provider.completed_turn_authority().is_some(),
provider.ambiguous_turn_start_pending(),
+ provider.take_rejected_accepted_turn(),
)
};
if let Err(error) = start_result {
+ if let Some(rejected_accepted_turn) = rejected_accepted_turn {
+ // Codex accepted this work before disclosing a usable durable
+ // identity. The provider has already been terminated; close
+ // this run before returning so recovery cannot resume the
+ // untracked turn from the provider's thread snapshot.
+ self.close_after_rejected_provider_acceptance(&rejected_accepted_turn)?;
+ let failure_kind = match &rejected_accepted_turn {
+ RejectedAcceptedTurn::ReusedIdentity(_) => "accepted identity reuse",
+ RejectedAcceptedTurn::InvalidIdentity => "an invalid accepted identity",
+ };
+ return Err(DurableRunnerError::invalid(format!(
+ "Codex turn/start failed closed after {failure_kind}: {error}"
+ )));
+ }
let state = self
.state
.as_mut()
@@ -748,6 +1657,11 @@ impl CodexCommandExecutor {
state.completed_turn_authoritative = false;
state.completed_turn_process_generation = None;
state.completed_provider_turn_id = None;
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
state.last_agent_message = None;
state.lifecycle = "turn_active".to_owned();
self.save_state()?;
@@ -773,6 +1687,30 @@ impl CodexCommandExecutor {
"reason": reason,
})));
}
+ let has_pending_tools = self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.tool_bridge.pending_calls().next().is_some());
+ if has_pending_tools {
+ let identity = self.event_identity()?;
+ let mut next_state = self
+ .state
+ .clone()
+ .expect("Codex state remains available during interruption");
+ let cancelled = next_state
+ .tool_bridge
+ .cancel_pending_calls("provider_turn_stopped")
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to cancel pending semantic tools: {error}"
+ ))
+ })?;
+ for result in cancelled {
+ next_state.push_terminal_event(semantic_result_event(&identity, &result))?;
+ }
+ self.persist_state(&next_state)?;
+ self.state = Some(next_state);
+ }
self.ensure_provider()?.interrupt_turn().map_err(|error| {
DurableRunnerError::invalid(format!("Codex turn interrupt failed: {error}"))
})?;
@@ -799,6 +1737,15 @@ impl CodexCommandExecutor {
.get("requestId")
.and_then(Value::as_str)
.ok_or_else(|| DurableRunnerError::invalid("request.resolve requires requestId"))?;
+ if self
+ .state
+ .as_ref()
+ .is_none_or(|state| state.active_provider_turn_id.is_none())
+ {
+ return Err(DurableRunnerError::invalid(
+ "cannot resolve a Codex runtime request outside an active turn",
+ ));
+ }
let response = payload
.get("response")
.ok_or_else(|| DurableRunnerError::invalid("request.resolve requires response"))?;
@@ -817,6 +1764,377 @@ impl CodexCommandExecutor {
})
}
+ fn event_identity(&self) -> Result<ProviderEventIdentity, DurableRunnerError> {
+ self.event_identity.clone().ok_or_else(|| {
+ DurableRunnerError::invalid(
+ "Codex semantic tool events require the durable runner identity",
+ )
+ })
+ }
+
+ fn reject_tool_call(
+ &mut self,
+ call_id: String,
+ operation_id: String,
+ reason: String,
+ ) -> Result<(), DurableRunnerError> {
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available for a rejected tool call");
+ let event = NormalizedProviderEvent {
+ event_type: "harness.diagnostic".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "provider": "codex",
+ "code": "semantic_tool_denied",
+ "operationId": operation_id,
+ "callId": call_id,
+ "message": reason,
+ "paperclipExecuted": false,
+ }),
+ };
+ if state.receipt_limit_interrupt_pending {
+ state.push_receipt_limit_cleanup_event(event)?;
+ } else {
+ state.push_event(event)?;
+ }
+ self.save_state()?;
+ let rejection = ToolResult {
+ call_id,
+ operation_id,
+ result: json!({
+ "error": {
+ "code": "invalid_tool_call",
+ "message": "Paperclip rejected this semantic tool call",
+ "retryable": false,
+ },
+ }),
+ is_error: true,
+ };
+ self.provider
+ .as_mut()
+ .expect("provider remains present while rejecting its tool call")
+ .deliver_tool_result(&rejection)
+ .map_err(|delivery_error| {
+ DurableRunnerError::invalid(format!(
+ "failed to return the semantic tool rejection: {delivery_error}"
+ ))
+ })
+ }
+
+ fn stop_turn_at_tool_receipt_limit(
+ &mut self,
+ call_id: String,
+ operation_id: String,
+ ) -> Result<(), DurableRunnerError> {
+ let deadline_unix_ms =
+ receipt_limit_deadline_after(RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS)?;
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available at its tool receipt limit");
+ let interrupt_pending = state.begin_receipt_limit_stop(
+ call_id.clone(),
+ operation_id.clone(),
+ deadline_unix_ms,
+ )?;
+ let first_interrupt_attempt =
+ interrupt_pending && state.receipt_limit_interrupt_attempts == 0;
+ if interrupt_pending {
+ self.save_state()?;
+ }
+ // The durable diagnostic owns the turn-level failure, while every
+ // buffered JSON-RPC call still receives an explicit provider error.
+ // Deliver the rejection before requesting interruption: Codex may
+ // close the transport as part of the interrupt, and a failed courtesy
+ // RPC must never abort runnerd's terminal polling loop.
+ let rejection = ToolResult {
+ call_id,
+ operation_id,
+ result: json!({
+ "error": {
+ "code": "semantic_tool_turn_receipt_limit",
+ "message": "Paperclip stopped this turn at its durable semantic-tool receipt limit",
+ "retryable": false,
+ },
+ }),
+ is_error: true,
+ };
+ if let Some(provider) = self.provider.as_mut() {
+ let _ = provider.deliver_tool_result(&rejection);
+ }
+ if first_interrupt_attempt {
+ // Keep the durable retry marker until a terminal notification is
+ // observed. Provider acceptance acknowledges only this RPC; it
+ // does not prove that the turn stopped. Later buffered calls may
+ // therefore retry the idempotent interruption instead of leaving
+ // a still-active receipt-exhausted turn permanently unstopped.
+ self.state
+ .as_mut()
+ .expect("Codex state remains available before receipt-limit interruption")
+ .record_receipt_limit_interrupt_attempt()?;
+ self.save_state()?;
+ match self.interrupt_turn("semantic_tool_turn_receipt_limit") {
+ Ok(_) => {
+ let accepted_deadline_unix_ms =
+ receipt_limit_deadline_after(RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS)?;
+ self.state
+ .as_mut()
+ .expect("Codex state remains available after receipt-limit interruption")
+ .mark_receipt_limit_interrupt_accepted(accepted_deadline_unix_ms);
+ self.save_state()?;
+ }
+ // The first interruption attempt is also best-effort. Its
+ // durable retry marker is already saved, and propagating the
+ // transport error here would terminate runnerd before it can
+ // poll the provider's terminal notification.
+ Err(_) => {}
+ }
+ }
+ Ok(())
+ }
+
+ fn settle_receipt_limit_interrupt_after_deadline(&mut self) -> Result<(), DurableRunnerError> {
+ let interrupt_accepted = self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.receipt_limit_interrupt_accepted);
+ let provider_shutdown_failed = self
+ .provider
+ .as_mut()
+ .is_some_and(|provider| provider.shutdown().is_err());
+ self.provider = None;
+ let identity = self.event_identity()?;
+ let state = self
+ .state
+ .as_mut()
+ .expect("Codex state remains available at its receipt-limit retry bound");
+ state.settle_active_provider_turn_identity()?;
+ let settled = state
+ .tool_bridge
+ .settle_turn("semantic_tool_turn_receipt_limit")
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to settle semantic tools at the receipt-limit retry bound: {error}"
+ ))
+ })?;
+ for result in settled {
+ state.push_terminal_event(semantic_result_event(&identity, &result))?;
+ }
+ state.active_provider_turn_id = None;
+ state.completed_turn_authoritative = false;
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
+ state.lifecycle = "provider_exited".to_owned();
+ let terminal_event_type = if interrupt_accepted {
+ "turn.interrupted"
+ } else {
+ "turn.failed"
+ };
+ state.push_terminal_event(NormalizedProviderEvent {
+ event_type: terminal_event_type.to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "provider": "codex",
+ "code": if interrupt_accepted {
+ "semantic_tool_turn_receipt_limit_interrupt_deadline"
+ } else {
+ "semantic_tool_turn_receipt_limit_interrupt_unconfirmed"
+ },
+ "message": if interrupt_accepted {
+ "Codex accepted the receipt-limit interruption but did not emit its terminal before the bounded shutdown deadline"
+ } else {
+ "Codex did not confirm terminal state after the bounded receipt-limit interruption attempts"
+ },
+ "interruptAccepted": interrupt_accepted,
+ "providerTerminalObserved": false,
+ "providerShutdownFailed": provider_shutdown_failed,
+ }),
+ })?;
+ let terminal = terminal_events(state, terminal_event_type);
+ state.extend_terminal_events(terminal)?;
+ self.save_state()
+ }
+
+ fn retry_receipt_limit_interrupt(&mut self) -> Result<(), DurableRunnerError> {
+ let (should_retry, attempts, persisted_deadline) =
+ self.state.as_ref().map_or((false, 0, None), |state| {
+ (
+ state.receipt_limit_interrupt_pending
+ && state.active_provider_turn_id.is_some(),
+ state.receipt_limit_interrupt_attempts,
+ state.receipt_limit_interrupt_deadline_unix_ms,
+ )
+ });
+ if !should_retry {
+ return Ok(());
+ }
+ let now_unix_ms = current_unix_ms()?;
+ let deadline_unix_ms = match persisted_deadline {
+ Some(deadline) => deadline,
+ None => {
+ // Older durable state did not record this additive field. Give
+ // an already-pending interruption one complete bounded window
+ // after recovery rather than falling back on poll count alone.
+ let timeout_ms = if self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.receipt_limit_interrupt_accepted)
+ {
+ RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS
+ } else {
+ RECEIPT_LIMIT_INTERRUPT_TERMINAL_DEADLINE_MS
+ };
+ let deadline = receipt_limit_deadline_after(timeout_ms)?;
+ self.state
+ .as_mut()
+ .expect("Codex state remains available while adding its receipt-limit deadline")
+ .receipt_limit_interrupt_deadline_unix_ms = Some(deadline);
+ self.save_state()?;
+ deadline
+ }
+ };
+ // Attempts bound network traffic, while the durable wall-clock deadline
+ // gives an accepted asynchronous interruption time to deliver its
+ // authoritative terminal. The provider is always polled once more below
+ // before an elapsed deadline is converted into the conservative fallback.
+ if attempts >= MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS || now_unix_ms >= deadline_unix_ms {
+ return Ok(());
+ }
+ // Polling is the autonomous recovery path until a terminal notification
+ // clears the durable marker. RPC acceptance alone does not establish
+ // that Codex stopped the turn, so accepted-but-unsettled interruptions
+ // remain idempotently retryable across polls and process restarts.
+ self.state
+ .as_mut()
+ .expect("Codex state remains available before receipt-limit retry")
+ .record_receipt_limit_interrupt_attempt()?;
+ self.save_state()?;
+ if self
+ .interrupt_turn("semantic_tool_turn_receipt_limit_retry")
+ .is_ok()
+ && self.state.as_ref().is_some_and(|state| {
+ state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some()
+ })
+ {
+ let accepted_deadline_unix_ms =
+ receipt_limit_deadline_after(RECEIPT_LIMIT_ACCEPTED_TERMINAL_DEADLINE_MS)?;
+ self.state
+ .as_mut()
+ .expect("Codex state remains available after receipt-limit retry")
+ .mark_receipt_limit_interrupt_accepted(accepted_deadline_unix_ms);
+ self.save_state()?;
+ }
+ Ok(())
+ }
+
+ fn settle_receipt_limit_interrupt_if_deadline_elapsed(
+ &mut self,
+ ) -> Result<(), DurableRunnerError> {
+ let deadline = self.state.as_ref().and_then(|state| {
+ (state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some())
+ .then_some(state.receipt_limit_interrupt_deadline_unix_ms)
+ .flatten()
+ });
+ let Some(deadline) = deadline else {
+ return Ok(());
+ };
+ if current_unix_ms()? >= deadline {
+ self.settle_receipt_limit_interrupt_after_deadline()?;
+ }
+ Ok(())
+ }
+
+ fn handle_tool_call(
+ &mut self,
+ call_id: String,
+ operation_id: String,
+ input: Value,
+ ) -> Result<(), DurableRunnerError> {
+ let identity = self.event_identity()?;
+ let admission = self
+ .state
+ .as_mut()
+ .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?
+ .admit_tool_call(&call_id, &operation_id, &input);
+ match admission {
+ Ok(ToolCallAdmission::CompletedReplay(result)) => {
+ self.provider
+ .as_mut()
+ .expect("provider remains present while handling its tool call")
+ .deliver_tool_result(&result)
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to replay a durable semantic tool result: {error}"
+ ))
+ })?;
+ return Ok(());
+ }
+ Ok(ToolCallAdmission::PendingReplay) => return Ok(()),
+ Err(error) => {
+ if error.is_active_turn_receipt_limit() {
+ return self.stop_turn_at_tool_receipt_limit(call_id, operation_id);
+ }
+ return self.reject_tool_call(call_id, operation_id, error.to_string());
+ }
+ Ok(ToolCallAdmission::Pending(call)) => {
+ self.state
+ .as_mut()
+ .expect("Codex state remains available while accepting a tool call")
+ .push_event(semantic_input_event(&identity, &call))?;
+ self.save_state()
+ }
+ }
+ }
+
+ fn deliver_semantic_result(
+ &mut self,
+ payload: &Value,
+ ) -> Result<CommandExecution, DurableRunnerError> {
+ let result: ToolResult = serde_json::from_value(payload.clone()).map_err(|error| {
+ DurableRunnerError::invalid(format!("semantic tool result is invalid: {error}"))
+ })?;
+ let identity = self.event_identity()?;
+ let was_completed = self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.tool_bridge.has_completed_call(&result.call_id));
+ let mut next_state = self
+ .state
+ .clone()
+ .ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?;
+ next_state
+ .tool_bridge
+ .apply_result(result.clone())
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!("semantic tool result was rejected: {error}"))
+ })?;
+ if was_completed {
+ return Ok(CommandExecution::result(json!({
+ "status": "duplicate",
+ "callId": result.call_id,
+ })));
+ }
+ next_state.push_event(semantic_result_event(&identity, &result))?;
+ self.persist_state(&next_state)?;
+ self.state = Some(next_state);
+ self.ensure_provider()?
+ .deliver_tool_result(&result)
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to return semantic tool result to Codex: {error}"
+ ))
+ })?;
+ Ok(CommandExecution::result(json!({
+ "status": "delivered",
+ "callId": result.call_id,
+ })))
+ }
+
fn close_session(&mut self) -> Result<CommandExecution, DurableRunnerError> {
if let Some(provider) = self.provider.as_mut() {
provider.shutdown().map_err(|error| {
@@ -830,6 +2148,11 @@ impl CodexCommandExecutor {
.ok_or_else(|| DurableRunnerError::invalid("Codex provider is not prepared"))?;
state.active_provider_turn_id = None;
state.ambiguous_turn_start_pending = false;
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
state.lifecycle = "closed".to_owned();
let thread_id = state.thread_id.clone();
self.save_state()?;
@@ -865,15 +2188,24 @@ impl CodexCommandExecutor {
// an ambiguous-start failure cannot degrade into an empty successful
// poll on the same executor.
self.restore_provider_if_needed()?;
- if self
- .state
- .as_ref()
- .is_some_and(|state| !state.pending_events.is_empty())
+ // Receipt-limit interruption is autonomous recovery. It must advance
+ // even while older durable events await acknowledgement, otherwise a
+ // slow or disconnected controller can keep an exhausted provider turn
+ // alive forever. Terminal settlement uses the reserved event capacity.
+ self.retry_receipt_limit_interrupt()?;
+ let receipt_limit_terminal_poll = self.state.as_ref().is_some_and(|state| {
+ state.receipt_limit_interrupt_pending && state.active_provider_turn_id.is_some()
+ });
+ if !receipt_limit_terminal_poll
+ && self
+ .state
+ .as_ref()
+ .is_some_and(|state| !state.pending_events.is_empty())
{
return Ok(());
}
if self.provider.is_none() {
- return Ok(());
+ return self.settle_receipt_limit_interrupt_if_deadline_elapsed();
}
for _ in 0..MAX_EVENTS_PER_POLL {
let event = self
@@ -889,11 +2221,9 @@ impl CodexCommandExecutor {
CodexProviderEvent::ToolCall {
call_id,
operation_id,
- ..
+ input,
} => {
- return Err(DurableRunnerError::invalid(format!(
- "Codex emitted semantic tool call {call_id} for {operation_id} before the durable tool bridge was attached"
- )));
+ self.handle_tool_call(call_id, operation_id, input)?;
}
CodexProviderEvent::Notification { method, params } => {
let active_provider_turn_id = if method == "turn/started" {
@@ -904,14 +2234,17 @@ impl CodexCommandExecutor {
} else {
None
};
- let completed_turn_authority = if method == "turn/completed" {
- self.provider
- .as_ref()
- .and_then(CodexProvider::completed_turn_authority)
- .map(|(generation, turn_id)| (generation, turn_id.to_owned()))
- } else {
- None
- };
+ let normalized_terminal_type =
+ normalized_codex_terminal_event_type(&method, &params);
+ let completed_turn_authority =
+ if normalized_terminal_type == Some("turn.completed") {
+ self.provider
+ .as_ref()
+ .and_then(CodexProvider::completed_turn_authority)
+ .map(|(generation, turn_id)| (generation, turn_id.to_owned()))
+ } else {
+ None
+ };
let normalized = normalize_codex_notification(&method, &params);
let terminal_event_type = normalized
.iter()
@@ -926,6 +2259,7 @@ impl CodexCommandExecutor {
| "turn.interrupted"
)
});
+ let identity = self.event_identity.clone();
let state = self
.state
.as_mut()
@@ -948,23 +2282,63 @@ impl CodexCommandExecutor {
})?;
state.reconcile_active_provider_turn(Some(provider_turn_id));
}
- if method == "turn/completed" {
- let (process_generation, provider_turn_id) = completed_turn_authority
- .ok_or_else(|| {
+ if terminal_event_type.is_some() {
+ state.settle_active_provider_turn_identity()?;
+ let settled = state
+ .tool_bridge
+ .settle_turn("provider_turn_terminated")
+ .map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to settle semantic tools at turn termination: {error}"
+ ))
+ })?;
+ if !settled.is_empty() {
+ let identity = identity.as_ref().ok_or_else(|| {
DurableRunnerError::invalid(
- "Codex completion omitted process and turn authority",
+ "Codex semantic tool events require the durable runner identity",
)
})?;
+ for result in settled {
+ state.push_terminal_event(semantic_result_event(
+ identity, &result,
+ ))?;
+ }
+ }
state.active_provider_turn_id = None;
- state.completed_turn_authoritative = true;
- state.completed_turn_process_generation = Some(process_generation);
- state.completed_provider_turn_id = Some(provider_turn_id);
+ if terminal_event_type.as_deref() == Some("turn.completed") {
+ let (process_generation, provider_turn_id) = completed_turn_authority
+ .ok_or_else(|| {
+ DurableRunnerError::invalid(
+ "Codex completion omitted process and turn authority",
+ )
+ })?;
+ state.completed_turn_authoritative = true;
+ state.completed_turn_process_generation = Some(process_generation);
+ state.completed_provider_turn_id = Some(provider_turn_id);
+ } else {
+ state.completed_turn_authoritative = false;
+ state.completed_turn_process_generation = None;
+ state.completed_provider_turn_id = None;
+ }
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
state.ambiguous_turn_start_pending = false;
state.lifecycle = "session_open".to_owned();
}
- state.extend_events(normalized)?;
+ if terminal_event_type.is_some() {
+ state.extend_terminal_events(normalized)?;
+ } else if receipt_limit_terminal_poll {
+ for event in normalized {
+ state.push_receipt_limit_cleanup_event(event)?;
+ }
+ } else {
+ state.extend_events(normalized)?;
+ }
if let Some(event_type) = terminal_event_type {
- state.extend_events(terminal_events(state, &event_type))?;
+ state.extend_terminal_events(terminal_events(state, &event_type))?;
}
self.save_state()?;
}
@@ -977,29 +2351,35 @@ impl CodexCommandExecutor {
.or_else(|| question_set.pointer("/questions/0/prompt"))
.and_then(Value::as_str)
.unwrap_or("Codex needs your input");
- self.state
+ let state = self
+ .state
.as_mut()
- .expect("Codex state remains available while polling")
- .push_event(NormalizedProviderEvent {
- event_type: "runtime_request.created".to_owned(),
- priority: EventPriority::P0,
- payload: json!({
- "request": {
- "schema": "paperclip.runtime_request.v2",
- "requestKind": "runtime",
- "requestId": request_id,
- "type": "input",
- "status": "pending",
- "prompt": prompt,
- "input": question_set,
- "origin": {
- "adapter": "codex-app-server",
- "provider": "codex",
- "method": "item/tool/requestUserInput",
- },
+ .expect("Codex state remains available while polling");
+ let event = NormalizedProviderEvent {
+ event_type: "runtime_request.created".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({
+ "request": {
+ "schema": "paperclip.runtime_request.v2",
+ "requestKind": "runtime",
+ "requestId": request_id,
+ "type": "input",
+ "status": "pending",
+ "prompt": prompt,
+ "input": question_set,
+ "origin": {
+ "adapter": "codex-app-server",
+ "provider": "codex",
+ "method": "item/tool/requestUserInput",
},
- }),
- })?;
+ },
+ }),
+ };
+ if receipt_limit_terminal_poll {
+ state.push_receipt_limit_cleanup_event(event)?;
+ } else {
+ state.push_event(event)?;
+ }
self.save_state()?;
}
CodexProviderEvent::Exited {
@@ -1022,7 +2402,7 @@ impl CodexCommandExecutor {
// whether this separate session exit belongs to that
// completion or is a later idle-provider failure.
state.lifecycle = "provider_exited".to_owned();
- state.push_event(NormalizedProviderEvent {
+ state.push_terminal_event(NormalizedProviderEvent {
// A completed turn remains authoritative, while the
// reusable provider session independently becomes
// unavailable. Avoid emitting session.failed for
@@ -1053,7 +2433,11 @@ impl CodexCommandExecutor {
}
}
}
- Ok(())
+ // Check the durable deadline only after a complete provider poll. A
+ // terminal that arrived after interruption acceptance but before this
+ // observation remains authoritative even when several fast controller
+ // polls have already exhausted the interruption-attempt budget.
+ self.settle_receipt_limit_interrupt_if_deadline_elapsed()
}
}
@@ -1066,6 +2450,7 @@ impl CommandExecutor for CodexCommandExecutor {
if self.state.is_none() && command.payload.get("provider").is_some() {
self.prepare(&command.payload)?;
}
+ self.verify_attached_tools(&command.payload)?;
let mut execution = self.open_session()?;
execution.events.push((
"run.attached".to_owned(),
@@ -1081,6 +2466,7 @@ impl CommandExecutor for CodexCommandExecutor {
self.interrupt_turn(&command.command_type)
}
"request.resolve" => self.resolve_request(&command.payload),
+ "semantic_tool.result" => self.deliver_semantic_result(&command.payload),
"session.snapshot" => self.snapshot(),
"session.close" | "session.destroy" => self.close_session(),
"runner.drain" | "runner.suspend" | "runner.shutdown" => {
@@ -1119,6 +2505,7 @@ impl CommandExecutor for CodexCommandExecutor {
));
}
next_state.pending_events.drain(..count);
+ next_state.refill_pending_events();
self.persist_state(&next_state)?;
self.state = Some(next_state);
Ok(())
@@ -1160,6 +2547,7 @@ mod tests {
approval_policy: "never".to_owned(),
},
completion_contract: None,
+ tool_bridge: ProviderToolBridge::default(),
thread_id: Some("thread-1".to_owned()),
provider_session_id: None,
active_provider_turn_id: None,
@@ -1168,8 +2556,16 @@ mod tests {
provider_process_generation: 0,
completed_turn_process_generation: None,
completed_provider_turn_id: None,
+ settled_provider_turn_ids: std::collections::BTreeSet::new(),
+ settled_provider_turn_filter: DurableReplayFilter::default(),
+ receipt_limit_diagnostic_emitted: false,
+ receipt_limit_interrupt_pending: false,
+ receipt_limit_interrupt_accepted: false,
+ receipt_limit_interrupt_attempts: 0,
+ receipt_limit_interrupt_deadline_unix_ms: None,
last_agent_message: None,
pending_events: VecDeque::new(),
+ queued_events: VecDeque::new(),
next_provider_event_seq: initial_provider_event_seq(),
};
assert!(state.validate().is_err());
@@ -1194,6 +2590,7 @@ mod tests {
approval_policy: "never".to_owned(),
},
None,
+ ProviderToolBridge::default(),
);
state.thread_id = Some("thread-1".to_owned());
state.lifecycle = "session_open".to_owned();
@@ -1236,6 +2633,7 @@ mod tests {
revision: "1".to_owned(),
criterion_ids: vec!["objective".to_owned()],
}),
+ ProviderToolBridge::default(),
);
state.last_agent_message = Some("Finished the requested work.".to_owned());
let events = terminal_events(&state, "turn.completed");
@@ -1244,4 +2642,752 @@ mod tests {
assert_eq!(events[1].event_type, "run.terminal");
assert_eq!(events[1].payload["runTerminalState"], "succeeded");
}
+
+ #[test]
+ fn semantic_input_digest_covers_the_transmitted_redacted_value() {
+ let identity = ProviderEventIdentity {
+ run_id: "run-1".to_owned(),
+ normalized_session_id: "session-1".to_owned(),
+ turn_id: "turn-1".to_owned(),
+ item_id: "item-1".to_owned(),
+ };
+ let call = PendingToolCall {
+ call_id: "call-1".to_owned(),
+ operation_id: "get_task_context".to_owned(),
+ input: json!({"password": "do-not-persist", "safe": true}),
+ };
+ let event = semantic_input_event(&identity, &call);
+ let transmitted = &event.payload["semantic_tool"]["input"];
+ assert_eq!(transmitted["password"], "[REDACTED]");
+ assert_eq!(
+ event.payload["semantic_tool"]["content"]["digest"],
+ semantic_value_digest(transmitted)
+ );
+ }
+
+ #[test]
+ fn receipt_limit_diagnostic_is_durable_and_turn_idempotent() {
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ ProviderToolBridge::default(),
+ );
+ state.thread_id = Some("thread-1".to_owned());
+ state.active_provider_turn_id = Some("turn-1".to_owned());
+ state.lifecycle = "turn_active".to_owned();
+
+ assert!(state
+ .begin_receipt_limit_stop("call-first".to_owned(), "tool.first".to_owned(), 10_000)
+ .unwrap());
+ state.mark_receipt_limit_interrupt_accepted(50_000);
+ assert!(state
+ .begin_receipt_limit_stop("call-second".to_owned(), "tool.second".to_owned(), 20_000)
+ .unwrap());
+ assert_eq!(state.pending_events.len(), 1);
+ assert_eq!(state.pending_events[0].payload["callId"], "call-first");
+ assert_eq!(state.receipt_limit_interrupt_deadline_unix_ms, Some(50_000));
+
+ let mut recovered: CodexProviderState =
+ serde_json::from_slice(&serde_json::to_vec(&state).unwrap()).unwrap();
+ recovered.validate().unwrap();
+ assert!(recovered.receipt_limit_interrupt_accepted);
+ assert_eq!(
+ recovered.receipt_limit_interrupt_deadline_unix_ms,
+ Some(50_000)
+ );
+ assert!(recovered
+ .begin_receipt_limit_stop(
+ "call-after-restart".to_owned(),
+ "tool.third".to_owned(),
+ 30_000,
+ )
+ .unwrap());
+ assert_eq!(recovered.pending_events.len(), 1);
+ // Only a terminal notification clears the retry marker. Accepting an
+ // interrupt request does not prove that the provider stopped.
+ assert!(recovered
+ .begin_receipt_limit_stop(
+ "call-after-success".to_owned(),
+ "tool.fourth".to_owned(),
+ 40_000,
+ )
+ .unwrap());
+ assert_eq!(recovered.pending_events.len(), 1);
+ }
+
+ #[test]
+ fn settled_receipt_limit_interrupt_cannot_be_marked_accepted() {
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ ProviderToolBridge::default(),
+ );
+ state.thread_id = Some("thread-1".to_owned());
+ state.active_provider_turn_id = Some("turn-1".to_owned());
+ state.lifecycle = "turn_active".to_owned();
+ state
+ .begin_receipt_limit_stop("call-1".to_owned(), "tool.one".to_owned(), 10_000)
+ .unwrap();
+ state.record_receipt_limit_interrupt_attempt().unwrap();
+
+ // Recovery observed that the turn ended before the interrupt RPC was
+ // issued and cleared the receipt-limit interruption state.
+ state.settle_active_provider_turn_identity().unwrap();
+ state.active_provider_turn_id = None;
+ state.lifecycle = "session_open".to_owned();
+ state.receipt_limit_diagnostic_emitted = false;
+ state.receipt_limit_interrupt_pending = false;
+ state.receipt_limit_interrupt_accepted = false;
+ state.receipt_limit_interrupt_attempts = 0;
+ state.receipt_limit_interrupt_deadline_unix_ms = None;
+
+ state.mark_receipt_limit_interrupt_accepted(50_000);
+
+ assert!(!state.receipt_limit_interrupt_accepted);
+ assert!(state.receipt_limit_interrupt_deadline_unix_ms.is_none());
+ state.validate().unwrap();
+ }
+
+ #[test]
+ fn regular_backlog_preserves_receipt_limit_and_terminal_settlement_capacity() {
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ ProviderToolBridge::default(),
+ );
+ let identity = ProviderEventIdentity {
+ run_id: "run-1".to_owned(),
+ normalized_session_id: "session-1".to_owned(),
+ turn_id: "turn-1".to_owned(),
+ item_id: "item-1".to_owned(),
+ };
+ let ordinary_event = || NormalizedProviderEvent {
+ event_type: "harness.diagnostic".to_owned(),
+ priority: EventPriority::P1,
+ payload: json!({"code": "ordinary_backlog"}),
+ };
+ for _ in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) {
+ state.push_event(ordinary_event()).unwrap();
+ }
+ assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL);
+ assert_eq!(
+ state.queued_events.len(),
+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS
+ );
+ assert!(state.push_event(ordinary_event()).is_err());
+
+ for _ in 0..MAX_EVENTS_PER_POLL {
+ state
+ .push_receipt_limit_cleanup_event(ordinary_event())
+ .unwrap();
+ }
+ let cleanup_boundary = state.queued_events.len();
+ state
+ .push_receipt_limit_cleanup_event(ordinary_event())
+ .expect("cleanup overflow is dropped while preserving terminal capacity");
+ assert_eq!(state.queued_events.len(), cleanup_boundary);
+
+ for index in 0..MAX_PENDING_CALLS {
+ state
+ .push_terminal_event(semantic_result_event(
+ &identity,
+ &ToolResult {
+ call_id: format!("call-{index}"),
+ operation_id: "get_task_context".to_owned(),
+ result: json!({"error": {"code": "provider_turn_terminated"}}),
+ is_error: true,
+ },
+ ))
+ .unwrap();
+ }
+ for event_type in [
+ "harness.diagnostic",
+ "turn.completed",
+ "run.result.proposed",
+ "run.terminal",
+ ] {
+ state
+ .push_terminal_event(NormalizedProviderEvent {
+ event_type: event_type.to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({"terminal": true}),
+ })
+ .unwrap();
+ }
+
+ assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL);
+ assert_eq!(state.queued_events.len(), MAX_QUEUED_PROVIDER_EVENTS);
+ state.validate().unwrap();
+ assert!(state.push_terminal_event(ordinary_event()).is_err());
+ assert!(serde_json::to_vec(&state).unwrap().len() as u64 <= MAX_PROVIDER_STATE_BYTES);
+ }
+
+ #[test]
+ fn completed_replays_are_read_only_at_the_regular_event_boundary() {
+ let operation = crate::provider_bridge::AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation],
+ })
+ .unwrap();
+ bridge
+ .begin_call(
+ "call-replayed".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .unwrap();
+ let replayed_result = ToolResult {
+ call_id: "call-replayed".to_owned(),
+ operation_id: "get_task_context".to_owned(),
+ result: json!({"ok": true}),
+ is_error: false,
+ };
+ bridge.apply_result(replayed_result.clone()).unwrap();
+
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ bridge,
+ );
+ let identity = ProviderEventIdentity {
+ run_id: "run-1".to_owned(),
+ normalized_session_id: "session-1".to_owned(),
+ turn_id: "turn-1".to_owned(),
+ item_id: "item-1".to_owned(),
+ };
+
+ // Keep the pending window occupied, then model the input and result
+ // receipts retained by the maximum 4,096 completed calls. Only three
+ // regular queued-event slots remain at this boundary.
+ for index in 0..MAX_EVENTS_PER_POLL {
+ state
+ .push_event(NormalizedProviderEvent {
+ event_type: "provider.notice.recorded".to_owned(),
+ priority: EventPriority::P1,
+ payload: json!({"index": index}),
+ })
+ .unwrap();
+ }
+ for index in 0..MAX_PENDING_CALLS {
+ let call = PendingToolCall {
+ call_id: format!("call-{index}"),
+ operation_id: "get_task_context".to_owned(),
+ input: json!({}),
+ };
+ state
+ .push_event(semantic_input_event(&identity, &call))
+ .unwrap();
+ state
+ .push_event(semantic_result_event(
+ &identity,
+ &ToolResult {
+ call_id: call.call_id,
+ operation_id: call.operation_id,
+ result: json!({"ok": true}),
+ is_error: false,
+ },
+ ))
+ .unwrap();
+ }
+ assert_eq!(
+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS - state.queued_events.len(),
+ 3
+ );
+ let pending_len = state.pending_events.len();
+ let queued_len = state.queued_events.len();
+ let next_sequence = state.next_provider_event_seq;
+
+ for _ in 0..4 {
+ assert_eq!(
+ state
+ .admit_tool_call("call-replayed", "get_task_context", &json!({}))
+ .unwrap(),
+ ToolCallAdmission::CompletedReplay(replayed_result.clone())
+ );
+ }
+ assert_eq!(state.pending_events.len(), pending_len);
+ assert_eq!(state.queued_events.len(), queued_len);
+ assert_eq!(state.next_provider_event_seq, next_sequence);
+ state.validate().unwrap();
+ }
+
+ #[test]
+ fn pending_replays_are_read_only_at_the_regular_event_boundary() {
+ let operation = crate::provider_bridge::AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation],
+ })
+ .unwrap();
+ bridge
+ .begin_call(
+ "call-pending".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .unwrap();
+
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ bridge,
+ );
+ for index in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) {
+ state
+ .push_event(NormalizedProviderEvent {
+ event_type: "provider.notice.recorded".to_owned(),
+ priority: EventPriority::P1,
+ payload: json!({"index": index}),
+ })
+ .unwrap();
+ }
+ assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL);
+ assert_eq!(
+ state.queued_events.len(),
+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS
+ );
+ let next_sequence = state.next_provider_event_seq;
+
+ for _ in 0..4 {
+ assert_eq!(
+ state
+ .admit_tool_call("call-pending", "get_task_context", &json!({}))
+ .unwrap(),
+ ToolCallAdmission::PendingReplay
+ );
+ }
+ assert_eq!(state.pending_events.len(), MAX_EVENTS_PER_POLL);
+ assert_eq!(
+ state.queued_events.len(),
+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS
+ );
+ assert_eq!(state.next_provider_event_seq, next_sequence);
+ state.validate().unwrap();
+ }
+
+ #[test]
+ fn exact_regular_backlog_capacity_rejects_turn_admission() {
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ ProviderToolBridge::default(),
+ );
+ state.lifecycle = "prepared".to_owned();
+ for _ in 0..(MAX_EVENTS_PER_POLL + MAX_REGULAR_QUEUED_PROVIDER_EVENTS) {
+ state
+ .push_event(NormalizedProviderEvent {
+ event_type: "harness.diagnostic".to_owned(),
+ priority: EventPriority::P1,
+ payload: json!({"code": "admission_boundary"}),
+ })
+ .unwrap();
+ }
+ assert_eq!(
+ state.queued_events.len(),
+ MAX_REGULAR_QUEUED_PROVIDER_EVENTS,
+ );
+ let mut executor = CodexCommandExecutor::new(PathBuf::from("unused-test-state"));
+ executor.state = Some(state);
+ executor.restore_checked = true;
+
+ let error = executor
+ .start_turn(&json!({"text": "must not reach the provider"}))
+ .expect_err("the exact regular backlog limit must reject admission");
+ assert!(error
+ .to_string()
+ .contains("until terminal events are acknowledged"));
+ assert!(executor.provider.is_none());
+ }
+
+ #[test]
+ fn transient_receipt_limit_clears_before_a_later_turn() {
+ let operation = crate::provider_bridge::AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation],
+ })
+ .unwrap();
+ let mut encoded = serde_json::to_value(&bridge).unwrap();
+ encoded["durableRunReceiptLimitReached"] = Value::Bool(true);
+ let mut bridge: ProviderToolBridge = serde_json::from_value(encoded).unwrap();
+ bridge.attach_existing_run().unwrap();
+
+ assert!(bridge.durable_run_receipt_limit_reached());
+ bridge.prepare_turn().unwrap();
+ assert!(!bridge.durable_run_receipt_limit_reached());
+ }
+
+ #[test]
+ fn restore_reattaches_the_durable_tool_result_byte_counter() {
+ let directory = std::env::temp_dir().join(format!(
+ "paperclip-provider-tool-byte-restore-{}",
+ std::process::id()
+ ));
+ let _ = fs::remove_dir_all(&directory);
+ fs::create_dir_all(&directory).unwrap();
+ let operation = crate::provider_bridge::AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation],
+ })
+ .unwrap();
+ bridge
+ .begin_call(
+ "call-1".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .unwrap();
+ bridge
+ .apply_result(ToolResult {
+ call_id: "call-1".to_owned(),
+ operation_id: "get_task_context".to_owned(),
+ result: json!({"ok": true}),
+ is_error: false,
+ })
+ .unwrap();
+ bridge.settle_turn("provider_turn_terminated").unwrap();
+ assert!(bridge.retained_result_bytes_for_test() > 0);
+
+ let state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ bridge,
+ );
+ let writer = CodexCommandExecutor::new(&directory);
+ writer.persist_state(&state).unwrap();
+
+ let mut recovered = CodexCommandExecutor::new(&directory);
+ recovered.restore().unwrap();
+ assert!(
+ recovered
+ .state
+ .as_ref()
+ .unwrap()
+ .tool_bridge
+ .retained_result_bytes_for_test()
+ > 0
+ );
+ fs::remove_dir_all(directory).unwrap();
+ }
+
+ #[test]
+ fn durable_provider_turn_ledger_backfills_legacy_completion_authority() {
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ ProviderToolBridge::default(),
+ );
+ state.completed_turn_authoritative = true;
+ state.completed_turn_process_generation = Some(1);
+ state.completed_provider_turn_id = Some("provider-turn-legacy".to_owned());
+ state.provider_process_generation = 1;
+
+ let (recovered, recovered_filter) = state.recovered_settled_provider_turn_ids().unwrap();
+
+ assert!(settled_provider_turn_contains(
+ &recovered,
+ &recovered_filter,
+ "provider-turn-legacy"
+ ));
+ }
+
+ #[test]
+ fn durable_provider_turn_ledger_never_evicts_within_an_epoch() {
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ ProviderToolBridge::default(),
+ );
+ state.thread_id = Some("thread-1".to_owned());
+ state.lifecycle = "turn_active".to_owned();
+ for index in 0..MAX_SETTLED_PROVIDER_TURN_IDS - 1 {
+ state
+ .settled_provider_turn_ids
+ .insert(format!("provider-turn-{index:04}"));
+ }
+ state.active_provider_turn_id = Some("provider-turn-final".to_owned());
+
+ state.settle_active_provider_turn_identity().unwrap();
+ state.active_provider_turn_id = None;
+ state.lifecycle = "session_open".to_owned();
+
+ assert_eq!(
+ state.settled_provider_turn_ids.len(),
+ MAX_SETTLED_PROVIDER_TURN_IDS
+ );
+ assert!(state.settled_provider_turn_filter.is_empty());
+ assert!(state
+ .settled_provider_turn_ids
+ .contains("provider-turn-final"));
+ assert!(state
+ .settled_provider_turn_ids
+ .contains("provider-turn-0000"));
+
+ state.lifecycle = "turn_active".to_owned();
+ state.active_provider_turn_id = Some("provider-turn-overflow".to_owned());
+ assert!(state.settle_active_provider_turn_identity().is_err());
+ assert!(!state
+ .settled_provider_turn_ids
+ .contains("provider-turn-overflow"));
+ state.active_provider_turn_id = None;
+ state.lifecycle = "session_open".to_owned();
+ state.validate().unwrap();
+ let recovered: CodexProviderState =
+ serde_json::from_str(&serde_json::to_string(&state).unwrap()).unwrap();
+ assert_eq!(
+ recovered.settled_provider_turn_ids,
+ state.settled_provider_turn_ids
+ );
+ assert_eq!(
+ recovered.settled_provider_turn_filter,
+ state.settled_provider_turn_filter
+ );
+ recovered.validate().unwrap();
+ }
+
+ #[test]
+ fn receipt_limit_deadline_settlement_preserves_unacknowledged_events() {
+ let directory = std::env::temp_dir().join(format!(
+ "paperclip-provider-receipt-deadline-{}",
+ std::process::id()
+ ));
+ let _ = fs::remove_dir_all(&directory);
+ fs::create_dir_all(&directory).unwrap();
+ let mut state = CodexProviderState::new(
+ CodexProviderConfig {
+ provider: "codex".to_owned(),
+ driver: "codex_app_server".to_owned(),
+ provider_version: "test".to_owned(),
+ command: PathBuf::from("codex"),
+ args: vec!["app-server".to_owned()],
+ cwd: std::env::current_dir()
+ .unwrap()
+ .to_string_lossy()
+ .into_owned(),
+ model: None,
+ provider_session_id: None,
+ instructions: String::new(),
+ approval_policy: "never".to_owned(),
+ },
+ None,
+ ProviderToolBridge::default(),
+ );
+ state.thread_id = Some("thread-1".to_owned());
+ state.active_provider_turn_id = Some("turn-1".to_owned());
+ state.lifecycle = "turn_active".to_owned();
+ state.receipt_limit_diagnostic_emitted = true;
+ state.receipt_limit_interrupt_pending = true;
+ state.receipt_limit_interrupt_attempts = MAX_RECEIPT_LIMIT_INTERRUPT_ATTEMPTS;
+ state.receipt_limit_interrupt_deadline_unix_ms = Some(1);
+ state
+ .push_event(NormalizedProviderEvent {
+ event_type: "provider.notice.recorded".to_owned(),
+ priority: EventPriority::P1,
+ payload: json!({"message": "awaiting acknowledgement"}),
+ })
+ .unwrap();
+ let mut executor = CodexCommandExecutor::new(&directory);
+ executor.state = Some(state);
+ executor.event_identity = Some(ProviderEventIdentity {
+ run_id: "run-1".to_owned(),
+ normalized_session_id: "session-1".to_owned(),
+ turn_id: "turn-1".to_owned(),
+ item_id: "item-1".to_owned(),
+ });
+ executor.restore_checked = true;
+
+ // Exercise receipt-limit recovery directly. `poll_provider` also
+ // restores a missing provider by design, while this unit test
+ // intentionally injects state without constructing a provider.
+ executor.retry_receipt_limit_interrupt().unwrap();
+ executor
+ .settle_receipt_limit_interrupt_if_deadline_elapsed()
+ .unwrap();
+
+ let state = executor.state.as_ref().unwrap();
+ assert_eq!(state.lifecycle, "provider_exited");
+ assert!(state.active_provider_turn_id.is_none());
+ assert!(!state.receipt_limit_interrupt_pending);
+ assert!(state.pending_events.iter().any(|event| {
+ event.event_type == "provider.notice.recorded"
+ && event.payload == json!({"message": "awaiting acknowledgement"})
+ }));
+ assert!(state
+ .pending_events
+ .iter()
+ .any(|event| event.event_type == "turn.failed"));
+ fs::remove_dir_all(directory).unwrap();
+ }
}
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs
index f5ef365baf..62213d3378 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs
@@ -1,9 +1,9 @@
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::error::Error;
use std::fmt::{self, Display, Formatter};
use serde::de::{self, MapAccess, Visitor};
-use serde::{Deserialize, Deserializer, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
use sha2::{Digest, Sha256};
@@ -12,18 +12,32 @@ pub const TOOL_CALL_SCHEMA: &str = "paperclip.prp.semantic_tool.v1";
pub const TOOL_RESULT_COMMAND: &str = "semantic_tool.result";
const MAX_AUTHORIZED_TOOLS: usize = 256;
const MAX_DESCRIPTION_BYTES: usize = 16 * 1024;
-const MAX_SCHEMA_BYTES: usize = 1024 * 1024;
-const MAX_TOOL_SET_BYTES: usize = 4 * 1024 * 1024;
-const MAX_TOOL_VALUE_BYTES: usize = 1024 * 1024;
-// Settled results are authoritative replay receipts and live for the durable
-// run. Bound their complete encoded map, while reserving enough room for every
-// active call to later produce a maximum-sized result. The 1 KiB allowance
-// covers the map key, bounded call/operation identities, JSON field names, and
-// escaping around a 1 MiB result value.
+const MAX_SCHEMA_BYTES: usize = 512 * 1024;
+// Leave room for the authenticated PRP command or event envelope inside the
+// default 1 MiB transport frame.
+const MAX_TOOL_SET_BYTES: usize = 768 * 1024;
+const MAX_TOOL_VALUE_BYTES: usize = 768 * 1024;
+const MAX_ACCEPTED_TOOL_VALUE_BYTES: usize = 4 * 1024 * 1024;
+const MAX_RETAINED_TOOL_VALUE_BYTES: usize = 8 * 1024 * 1024;
+// Exact settled receipts are authoritative for the durable run. Bound their
+// complete encoded representation independently of the raw-value budget so a
+// long sequence of tiny results cannot accumulate unbounded identity and JSON
+// envelope overhead.
const MAX_SETTLED_RESULT_BYTES: usize = 8 * 1024 * 1024;
-const MAX_SETTLED_RESULT_ENTRY_BYTES: usize = MAX_TOOL_VALUE_BYTES + 1024;
-const MAX_RETAINED_CALLS: usize = 4_096;
+pub(crate) const MAX_PENDING_CALLS: usize = 4_096;
+// Active-turn inputs and results are separately bounded. Settled identities
+// remain exact for a complete provider-process epoch so a delayed request
+// cannot replay an old call ID after crossing a turn boundary. At the bound,
+// the backend must reap the idle process before it rotates this ledger.
+const MAX_DURABLE_CALL_RECEIPTS: usize = 4_096;
const MAX_SETTLED_CALL_IDS: usize = 65_536;
+// Retain the legacy serialized filter shape for recovery compatibility. New
+// state never inserts probabilistic identities. A recovered non-empty filter
+// makes the durable run fail closed until attach_run because its identities
+// cannot safely be distinguished from fresh calls.
+const REPLAY_FILTER_WORDS: usize = 32_768;
+const ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE: &str =
+ "durable provider tool receipt limit reached for the active turn";
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase")]
@@ -62,26 +76,145 @@ pub struct ToolResult {
pub is_error: bool,
}
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+struct CompletedToolCall {
+ call: PendingToolCall,
+ result: ToolResult,
+}
+
+#[derive(Clone, Debug, Default, PartialEq)]
+struct SettledCallIds {
+ order: VecDeque<String>,
+ members: BTreeSet<String>,
+}
+
+// Preserve the existing on-disk array shape so durable state written before
+// this bounded ordering was introduced remains readable. New state records IDs
+// from oldest to newest, which makes pruning deterministic across recovery.
+impl Serialize for SettledCallIds {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ self.order.serialize(serializer)
+ }
+}
+
+impl<'de> Deserialize<'de> for SettledCallIds {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let values = VecDeque::<String>::deserialize(deserializer)?;
+ let mut settled = Self::default();
+ for value in values {
+ if settled.members.insert(value.clone()) {
+ settled.order.push_back(value);
+ }
+ }
+ Ok(settled)
+ }
+}
+
+impl SettledCallIds {
+ fn clear(&mut self) {
+ self.order.clear();
+ self.members.clear();
+ }
+
+ fn contains(&self, call_id: &str) -> bool {
+ self.members.contains(call_id)
+ }
+
+ fn is_empty(&self) -> bool {
+ self.order.is_empty()
+ }
+
+ fn len(&self) -> usize {
+ self.order.len()
+ }
+
+ fn iter(&self) -> impl Iterator<Item = &String> {
+ self.order.iter()
+ }
+
+ fn extend_recent(
+ &mut self,
+ call_ids: impl IntoIterator<Item = String>,
+ limit: usize,
+ ) -> Vec<String> {
+ for call_id in call_ids {
+ if self.members.insert(call_id.clone()) {
+ self.order.push_back(call_id);
+ }
+ }
+ let mut evicted = Vec::new();
+ while self.order.len() > limit {
+ if let Some(call_id) = self.order.pop_front() {
+ self.members.remove(&call_id);
+ evicted.push(call_id);
+ }
+ }
+ evicted
+ }
+}
+
+#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
+pub(crate) struct DurableReplayFilter {
+ words: Vec<u64>,
+}
+
+impl DurableReplayFilter {
+ pub(crate) fn is_empty(&self) -> bool {
+ self.words.is_empty()
+ }
+
+ pub(crate) fn validate(&self) -> Result<(), ProviderBridgeError> {
+ if self.words.is_empty() || self.words.len() == REPLAY_FILTER_WORDS {
+ Ok(())
+ } else {
+ Err(ProviderBridgeError::invalid(
+ "durable replay filter has an invalid size",
+ ))
+ }
+ }
+}
+
#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ProviderToolBridge {
authorized: BTreeMap<String, AuthorizedTool>,
+ #[serde(default)]
+ catalog_operations: Vec<AuthorizedTool>,
catalog_digest: Option<String>,
pending: BTreeMap<String, PendingToolCall>,
#[serde(deserialize_with = "deserialize_retained_results")]
- completed: BTreeMap<String, ToolResult>,
+ completed: BTreeMap<String, CompletedToolCall>,
+ // Keep authoritative result bodies through the provider's replay window;
+ // prepare_turn releases them after settlement.
#[serde(default, deserialize_with = "deserialize_retained_results")]
- settled_results: BTreeMap<String, ToolResult>,
+ settled_results: BTreeMap<String, CompletedToolCall>,
// Derived from completed + settled results. It is intentionally omitted
// from durable JSON and recomputed by attach_existing_run so old state and
// tampered counters cannot bypass the byte envelope.
#[serde(skip)]
retained_result_bytes: usize,
- // Compatibility tombstones for state written before settled results were
- // retained. They still fail closed on call-id reuse, but cannot replay a
- // value that the older state format discarded.
+ // Exact replay tombstones span every provider turn in the durable run.
#[serde(default)]
- settled_call_ids: BTreeSet<String>,
+ settled_call_ids: SettledCallIds,
+ #[serde(default)]
+ settled_call_filter: DurableReplayFilter,
+ // Resource exhaustion stops the active turn. prepare_turn releases bulky
+ // result payloads and recomputes this marker, but replay ambiguity remains
+ // fail-closed until attach_run or a verified idle provider-process restart
+ // establishes fresh replay authority.
+ #[serde(
+ default,
+ alias = "settledHistoryResetPending",
+ skip_serializing_if = "is_false"
+ )]
+ durable_run_receipt_limit_reached: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -91,6 +224,14 @@ impl ProviderBridgeError {
fn invalid(message: impl Into<String>) -> Self {
Self(message.into())
}
+
+ fn active_turn_receipt_limit() -> Self {
+ Self::invalid(ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE)
+ }
+
+ pub fn is_active_turn_receipt_limit(&self) -> bool {
+ self.0 == ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE
+ }
}
impl Display for ProviderBridgeError {
@@ -101,6 +242,10 @@ impl Display for ProviderBridgeError {
impl Error for ProviderBridgeError {}
+fn is_false(value: &bool) -> bool {
+ !*value
+}
+
impl ProviderToolBridge {
pub fn prepare(&mut self, tool_set: AuthorizedToolSet) -> Result<(), ProviderBridgeError> {
self.prepare_internal(tool_set, false)
@@ -117,6 +262,8 @@ impl ProviderToolBridge {
self.settled_results.clear();
self.retained_result_bytes = 0;
self.settled_call_ids.clear();
+ self.settled_call_filter = DurableReplayFilter::default();
+ self.durable_run_receipt_limit_reached = false;
Ok(())
}
@@ -125,92 +272,9 @@ impl ProviderToolBridge {
// preserve them so an interrupted dispatcher can resume or replay the
// authoritative result. `attach_run` remains the boundary that rejects
// carrying pending calls into a different run.
- if self
- .authorized
- .iter()
- .any(|(operation_id, tool)| operation_id != &tool.operation_id)
- {
- return Err(ProviderBridgeError::invalid(
- "recovered authorized tool identities are inconsistent",
- ));
- }
- let Some(catalog_digest) = self.catalog_digest.clone() else {
- if self.authorized.is_empty()
- && self.pending.is_empty()
- && self.completed.is_empty()
- && self.settled_results.is_empty()
- && self.settled_call_ids.is_empty()
- {
- self.retained_result_bytes = 0;
- return Ok(());
- }
- return Err(ProviderBridgeError::invalid(
- "recovered authorized tools omit the catalog digest",
- ));
- };
- let recovered_tool_set = AuthorizedToolSet {
- schema: TOOL_SET_SCHEMA.to_owned(),
- schema_version: 1,
- catalog_digest,
- operations: self.authorized.values().cloned().collect(),
- };
- validate_authorized_tool_set(&recovered_tool_set).map_err(|error| {
- ProviderBridgeError::invalid(format!(
- "recovered authorized tool catalog is invalid: {error}"
- ))
- })?;
- if self
- .settled_call_ids
- .len()
- .checked_add(self.settled_results.len())
- .and_then(|total| total.checked_add(self.pending.len()))
- .and_then(|total| total.checked_add(self.completed.len()))
- .is_none_or(|total| total > MAX_SETTLED_CALL_IDS)
- || self.pending.len().saturating_add(self.completed.len()) > MAX_RETAINED_CALLS
- || self
- .settled_call_ids
- .iter()
- .any(|call_id| !is_stable_call_id(call_id))
- || self.settled_results.iter().any(|(call_id, result)| {
- !is_stable_call_id(call_id)
- || call_id != &result.call_id
- || self.settled_call_ids.contains(call_id)
- || validate_retained_result(result).is_err()
- || validate_tool_result_contract(&self.authorized, result).is_err()
- })
- || self.settled_call_ids.iter().any(|call_id| {
- self.pending.contains_key(call_id) || self.completed.contains_key(call_id)
- })
- || self.settled_results.keys().any(|call_id| {
- self.pending.contains_key(call_id) || self.completed.contains_key(call_id)
- })
- || self.pending.iter().any(|(call_id, call)| {
- !is_stable_call_id(call_id)
- || call_id != &call.call_id
- || self.completed.contains_key(call_id)
- || self.settled_call_ids.contains(call_id)
- || self.settled_results.contains_key(call_id)
- || validate_pending_tool_call(&self.authorized, call).is_err()
- })
- || self.completed.iter().any(|(call_id, result)| {
- !is_stable_call_id(call_id)
- || call_id != &result.call_id
- || self.pending.contains_key(call_id)
- || validate_retained_result(result).is_err()
- || validate_tool_result_contract(&self.authorized, result).is_err()
- })
- {
- return Err(ProviderBridgeError::invalid(
- "recovered provider tool call state is invalid",
- ));
- }
+ self.validate_recovered()?;
self.retained_result_bytes =
retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?;
- self.ensure_settled_result_capacity(0).map_err(|_| {
- ProviderBridgeError::invalid(
- "recovered provider tool results exceed the durable byte limit",
- )
- })?;
Ok(())
}
@@ -235,6 +299,7 @@ impl ProviderToolBridge {
}
}
self.catalog_digest = Some(tool_set.catalog_digest);
+ self.catalog_operations = tool_set.operations.clone();
self.authorized = tool_set
.operations
.into_iter()
@@ -244,7 +309,245 @@ impl ProviderToolBridge {
}
pub fn authorized_tools(&self) -> impl Iterator<Item = &AuthorizedTool> {
- self.authorized.values()
+ self.catalog_operations.iter()
+ }
+
+ pub fn validate_recovered(&self) -> Result<(), ProviderBridgeError> {
+ let Some(catalog_digest) = self.catalog_digest.clone() else {
+ return if self.authorized.is_empty()
+ && self.catalog_operations.is_empty()
+ && self.pending.is_empty()
+ && self.completed.is_empty()
+ && self.settled_results.is_empty()
+ && self.settled_call_ids.is_empty()
+ && self.settled_call_filter.is_empty()
+ && !self.durable_run_receipt_limit_reached
+ {
+ Ok(())
+ } else {
+ Err(ProviderBridgeError::invalid(
+ "recovered provider tool bridge omitted its catalog identity",
+ ))
+ };
+ };
+ let mut expected = ProviderToolBridge::default();
+ expected.prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest,
+ operations: self.catalog_operations.clone(),
+ })?;
+ if self.authorized != expected.authorized {
+ return Err(ProviderBridgeError::invalid(
+ "recovered provider tool bridge changed its authorized catalog",
+ ));
+ }
+ let retained_receipts = self.pending.len().saturating_add(self.completed.len());
+ if self.pending.len() > MAX_PENDING_CALLS || retained_receipts > MAX_DURABLE_CALL_RECEIPTS {
+ return Err(ProviderBridgeError::invalid(
+ "recovered provider tool bridge exceeds its call limit",
+ ));
+ }
+ self.validate_retained_value_bytes()?;
+ self.settled_call_filter.validate()?;
+ if self.settled_call_ids.len() > MAX_SETTLED_CALL_IDS {
+ return Err(ProviderBridgeError::invalid(
+ "recovered provider tool bridge exceeds its settled call identity limit",
+ ));
+ }
+ if !self.settled_identity_capacity_allows(0) {
+ return Err(ProviderBridgeError::invalid(
+ "recovered provider tool bridge cannot settle its active call identities",
+ ));
+ }
+ for call_id in self.settled_call_ids.iter() {
+ validate_stable_id(call_id, "settled tool call id")?;
+ if self.pending.contains_key(call_id) || self.completed.contains_key(call_id) {
+ return Err(ProviderBridgeError::invalid(
+ "recovered settled provider tool call identity is inconsistent",
+ ));
+ }
+ }
+ let mut pending_validator = expected.clone();
+ for (call_id, call) in &self.pending {
+ if call_id != &call.call_id
+ || self.completed.contains_key(call_id)
+ || self.settled_results.contains_key(call_id)
+ || self.has_settled_call_id(call_id)
+ {
+ return Err(ProviderBridgeError::invalid(
+ "recovered provider tool call identity is inconsistent",
+ ));
+ }
+ pending_validator.begin_call(
+ call.call_id.clone(),
+ call.operation_id.clone(),
+ call.input.clone(),
+ )?;
+ }
+ for (call_id, completed) in &self.completed {
+ if call_id != &completed.call.call_id
+ || call_id != &completed.result.call_id
+ || completed.call.operation_id != completed.result.operation_id
+ || self.settled_results.contains_key(call_id)
+ || self.has_settled_call_id(call_id)
+ {
+ return Err(ProviderBridgeError::invalid(
+ "recovered completed tool call identity is inconsistent",
+ ));
+ }
+ let mut completed_validator = expected.clone();
+ completed_validator.begin_call(
+ completed.call.call_id.clone(),
+ completed.call.operation_id.clone(),
+ completed.call.input.clone(),
+ )?;
+ completed_validator.apply_result(completed.result.clone())?;
+ }
+ for (call_id, settled) in &self.settled_results {
+ if call_id != &settled.call.call_id
+ || call_id != &settled.result.call_id
+ || settled.call.operation_id != settled.result.operation_id
+ || !self.settled_call_ids.contains(call_id)
+ || self.pending.contains_key(call_id)
+ || self.completed.contains_key(call_id)
+ {
+ return Err(ProviderBridgeError::invalid(
+ "recovered settled tool result identity is inconsistent",
+ ));
+ }
+ let mut settled_validator = expected.clone();
+ settled_validator.begin_call(
+ settled.call.call_id.clone(),
+ settled.call.operation_id.clone(),
+ settled.call.input.clone(),
+ )?;
+ settled_validator.apply_result(settled.result.clone())?;
+ }
+ let retained_bytes =
+ retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?;
+ ensure_settled_result_capacity(retained_bytes, self.pending.values()).map_err(|_| {
+ ProviderBridgeError::invalid(
+ "recovered provider tool results exceed the durable byte limit",
+ )
+ })?;
+ Ok(())
+ }
+
+ pub fn verify_tool_set(&self, tool_set: &AuthorizedToolSet) -> Result<(), ProviderBridgeError> {
+ let mut expected = ProviderToolBridge::default();
+ expected.prepare(tool_set.clone())?;
+ if self.catalog_digest != expected.catalog_digest
+ || self.catalog_operations != expected.catalog_operations
+ || self.authorized != expected.authorized
+ {
+ return Err(ProviderBridgeError::invalid(
+ "authorized tool set changed across a durable session",
+ ));
+ }
+ Ok(())
+ }
+
+ pub fn has_catalog(&self) -> bool {
+ self.catalog_digest.is_some()
+ }
+
+ pub fn durable_run_receipt_limit_reached(&self) -> bool {
+ self.durable_run_receipt_limit_reached
+ }
+
+ fn has_settled_call_id(&self, call_id: &str) -> bool {
+ self.settled_call_ids.contains(call_id)
+ }
+
+ pub(crate) fn replay_history_blocks_admission(&self) -> bool {
+ self.settled_call_ids.len() >= MAX_SETTLED_CALL_IDS || !self.settled_call_filter.is_empty()
+ }
+
+ pub(crate) fn has_active_receipts(&self) -> bool {
+ !self.pending.is_empty() || !self.completed.is_empty()
+ }
+
+ /// Forget an exact replay epoch only after its provider process has been
+ /// reaped and an idle replacement generation has been established. The
+ /// process boundary is what prevents an old provider request from being
+ /// admitted after its tombstone is released; callers must not use this as
+ /// ordinary capacity eviction.
+ pub(crate) fn rollover_replay_epoch_after_provider_restart(
+ &mut self,
+ ) -> Result<(), ProviderBridgeError> {
+ if !self.replay_history_blocks_admission() {
+ return Err(ProviderBridgeError::invalid(
+ "cannot rotate provider tool replay authority before its exact capacity boundary",
+ ));
+ }
+ if !self.pending.is_empty()
+ || !self.completed.is_empty()
+ || !self.settled_results.is_empty()
+ || self.retained_result_bytes != 0
+ {
+ return Err(ProviderBridgeError::invalid(
+ "cannot rotate provider tool replay authority while receipts are retained",
+ ));
+ }
+ self.settled_call_ids.clear();
+ self.settled_call_filter = DurableReplayFilter::default();
+ self.durable_run_receipt_limit_reached = false;
+ Ok(())
+ }
+
+ fn settled_identity_capacity_allows(&self, additional_calls: usize) -> bool {
+ self.settled_call_ids
+ .len()
+ .checked_add(self.total_call_receipts())
+ .and_then(|total| total.checked_add(additional_calls))
+ .is_some_and(|total| total <= MAX_SETTLED_CALL_IDS)
+ }
+
+ pub fn prepare_turn(&mut self) -> Result<(), ProviderBridgeError> {
+ if !self.pending.is_empty() || !self.completed.is_empty() {
+ return Err(ProviderBridgeError::invalid(
+ "cannot prepare the next provider turn while tool calls are active",
+ ));
+ }
+ // The authoritative result bodies are needed only while the provider
+ // can replay the just-settled turn. Release that bulky data at the
+ // verified turn boundary, but retain every call-ID tombstone and any
+ // recovered legacy filter until the owning provider process is reaped.
+ self.settled_results.clear();
+ self.retained_result_bytes = 0;
+ self.durable_run_receipt_limit_reached = self.replay_history_blocks_admission();
+ Ok(())
+ }
+
+ #[cfg(test)]
+ pub(crate) fn retained_result_bytes_for_test(&self) -> usize {
+ self.retained_result_bytes
+ }
+
+ pub fn replay_result(
+ &self,
+ call_id: &str,
+ operation_id: &str,
+ input: &Value,
+ ) -> Result<Option<ToolResult>, ProviderBridgeError> {
+ if let Some(completed) = self
+ .completed
+ .get(call_id)
+ .or_else(|| self.settled_results.get(call_id))
+ {
+ if completed.call.operation_id != operation_id || &completed.call.input != input {
+ return Err(ProviderBridgeError::invalid(
+ "provider replayed a completed tool call with different input",
+ ));
+ }
+ return Ok(Some(completed.result.clone()));
+ }
+ Ok(None)
+ }
+
+ pub fn has_completed_call(&self, call_id: &str) -> bool {
+ self.completed.contains_key(call_id) || self.has_settled_call_id(call_id)
}
pub fn begin_call(
@@ -253,6 +556,24 @@ impl ProviderToolBridge {
operation_id: String,
input: Value,
) -> Result<PendingToolCall, ProviderBridgeError> {
+ validate_stable_id(&call_id, "tool call id")?;
+ validate_operation_id(&operation_id)?;
+ let authorized = self.authorized.get(&operation_id).ok_or_else(|| {
+ ProviderBridgeError::invalid(format!(
+ "provider requested unauthorized tool {operation_id}"
+ ))
+ })?;
+ let validator = jsonschema::validator_for(&authorized.input_schema).map_err(|_| {
+ ProviderBridgeError::invalid(format!(
+ "tool {operation_id} has an invalid durable input JSON Schema"
+ ))
+ })?;
+ if !validator.is_valid(&input) {
+ return Err(ProviderBridgeError::invalid(format!(
+ "provider arguments for {operation_id} failed JSON Schema validation"
+ )));
+ }
+ bounded_json(&input, MAX_TOOL_VALUE_BYTES, "provider tool input")?;
let call = PendingToolCall {
call_id: call_id.clone(),
operation_id,
@@ -270,55 +591,93 @@ impl ProviderToolBridge {
}
if self.completed.contains_key(&call_id)
|| self.settled_results.contains_key(&call_id)
- || self.settled_call_ids.contains(&call_id)
+ || self.has_settled_call_id(&call_id)
{
return Err(ProviderBridgeError::invalid(
"provider reused a completed tool call id",
));
}
- if self.pending.len().saturating_add(self.completed.len()) >= MAX_RETAINED_CALLS {
+ // A durable receipt ledger may be exactly full or legacy state may
+ // contain only a probabilistic summary of evicted identities. Neither
+ // state can admit more work safely in this provider-process epoch. The
+ // backend reaps an idle process before establishing a fresh epoch.
+ if self.replay_history_blocks_admission() {
+ self.durable_run_receipt_limit_reached = true;
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
+ if self.durable_run_receipt_limit_reached {
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
+ // Pending and completed calls become exact settled identities when the
+ // turn terminates. Reserve that identity capacity before dispatch so
+ // every admitted call can be settled without stranding active state.
+ if !self.settled_identity_capacity_allows(1) {
+ self.durable_run_receipt_limit_reached = true;
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
+ if self.pending.len() >= MAX_PENDING_CALLS {
return Err(ProviderBridgeError::invalid(
- "provider tool receipt limit reached for the active turn",
+ "concurrent provider tool call limit reached",
));
}
- // Reserve durable identity space before accepting work. Settlement can
- // then never fail merely because earlier turns filled the ledger and
- // leave completed receipts stranded in the active-turn budget.
- if self
- .settled_call_ids
- .len()
- .saturating_add(self.settled_results.len())
- .saturating_add(self.pending.len())
- .saturating_add(self.completed.len())
- >= MAX_SETTLED_CALL_IDS
+ if self.total_call_receipts() >= MAX_DURABLE_CALL_RECEIPTS {
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
+ let input_bytes = json_size(&call.input, "provider tool input")?;
+ let pending_bytes = self.pending_value_bytes()?;
+ if pending_bytes
+ .checked_add(input_bytes)
+ .is_none_or(|total| total > MAX_ACCEPTED_TOOL_VALUE_BYTES)
{
return Err(ProviderBridgeError::invalid(
- "durable provider tool call identity limit reached",
+ "retained provider tool values exceed the 4 MiB acceptance limit",
));
}
- // Reserve the worst-case encoded result before accepting the call.
- // This makes apply_result and settlement infallible with respect to
- // durable result capacity: accepted work can always retain its exact
- // authoritative replay value.
- self.ensure_settled_result_capacity(1)?;
+ let result_reserve = self
+ .pending
+ .len()
+ .saturating_add(1)
+ .checked_mul(MAX_TOOL_VALUE_BYTES)
+ .ok_or_else(|| ProviderBridgeError::invalid("provider tool result reserve overflow"))?;
+ let retained_value_bytes = self.retained_value_bytes()?;
+ if retained_value_bytes
+ .checked_add(input_bytes)
+ .and_then(|total| total.checked_add(MAX_TOOL_VALUE_BYTES))
+ .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES)
+ {
+ self.durable_run_receipt_limit_reached = true;
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
+ if retained_value_bytes
+ .checked_add(input_bytes)
+ .and_then(|total| total.checked_add(result_reserve))
+ .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES)
+ {
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
+ // Reserve the complete encoded receipt, including this exact input and
+ // a maximum-sized result, before accepting work. An admitted call can
+ // therefore always retain its authoritative result at settlement.
+ if ensure_settled_result_capacity(self.retained_result_bytes, std::iter::once(&call))
+ .is_err()
+ {
+ self.durable_run_receipt_limit_reached = true;
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
+ if self.ensure_settled_result_capacity(Some(&call)).is_err() {
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
self.pending.insert(call_id, call.clone());
Ok(call)
}
pub fn apply_result(&mut self, result: ToolResult) -> Result<Value, ProviderBridgeError> {
- if result.call_id.is_empty()
- || result.call_id.len() > 160
- || result.call_id.chars().any(char::is_control)
- {
- return Err(ProviderBridgeError::invalid(
- "tool result call id is invalid",
- ));
- }
+ validate_stable_id(&result.call_id, "tool result call id")?;
validate_operation_id(&result.operation_id)?;
bounded_json(&result.result, MAX_TOOL_VALUE_BYTES, "provider tool result")?;
if let Some(existing) = self.completed.get(&result.call_id) {
- return if existing == &result {
- Ok(existing.result.clone())
+ return if existing.result == result {
+ Ok(existing.result.result.clone())
} else {
Err(ProviderBridgeError::invalid(
"conflicting duplicate tool result",
@@ -326,20 +685,20 @@ impl ProviderToolBridge {
};
}
if let Some(existing) = self.settled_results.get(&result.call_id) {
- return if existing == &result {
- Ok(existing.result.clone())
+ return if existing.result == result {
+ Ok(existing.result.result.clone())
} else {
Err(ProviderBridgeError::invalid(
"conflicting duplicate settled tool result",
))
};
}
- if self.settled_call_ids.contains(&result.call_id) {
+ if self.has_settled_call_id(&result.call_id) {
return Err(ProviderBridgeError::invalid(
"legacy settled tool result cannot be replayed",
));
}
- let pending = self.pending.get(&result.call_id).ok_or_else(|| {
+ let pending = self.pending.get(&result.call_id).cloned().ok_or_else(|| {
ProviderBridgeError::invalid("tool result does not match a pending provider call")
})?;
if pending.operation_id != result.operation_id {
@@ -347,89 +706,218 @@ impl ProviderToolBridge {
"tool result operation does not match its call",
));
}
- validate_tool_result_contract(&self.authorized, &result)?;
- let result_bytes = retained_result_entry_bytes(&result.call_id, &result)?;
- let next_retained_bytes = self
- .retained_result_bytes
+ let authorized = self.authorized.get(&result.operation_id).ok_or_else(|| {
+ ProviderBridgeError::invalid("tool result operation is no longer authorized")
+ })?;
+ let validator = jsonschema::validator_for(&authorized.response_schema).map_err(|_| {
+ ProviderBridgeError::invalid(format!(
+ "tool {} has an invalid durable response JSON Schema",
+ result.operation_id
+ ))
+ })?;
+ let response = semantic_response_value(&result)?;
+ if !result.is_error {
+ // Paperclip semantic dispatchers return an authoritative envelope;
+ // provider contracts describe the operation-specific value inside
+ // `result`. Direct values remain valid for compatibility with v1
+ // peers that do not wrap their semantic result.
+ if let Some(response) = response {
+ if !validator.is_valid(response) {
+ return Err(ProviderBridgeError::invalid(format!(
+ "tool result for {} failed JSON Schema validation",
+ result.operation_id
+ )));
+ }
+ }
+ }
+ let result_bytes = json_size(&result.result, "provider tool result")?;
+ let retained_bytes = self.retained_value_bytes()?;
+ if retained_bytes
.checked_add(result_bytes)
- .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?;
- let remaining_pending_reserve = self
- .pending
- .len()
- .saturating_sub(1)
- .checked_mul(MAX_SETTLED_RESULT_ENTRY_BYTES)
- .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?;
- if next_retained_bytes
- .checked_add(remaining_pending_reserve)
- .is_none_or(|bytes| bytes > MAX_SETTLED_RESULT_BYTES)
+ .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES)
{
return Err(ProviderBridgeError::invalid(
- "durable provider tool result byte limit reached",
+ "retained provider tool values exceed the 8 MiB aggregate limit",
));
}
+ let completed = CompletedToolCall {
+ call: pending,
+ result: result.clone(),
+ };
+ let receipt_bytes = retained_result_entry_bytes(&result.call_id, &completed)?;
+ let next_retained_bytes = self
+ .retained_result_bytes
+ .checked_add(receipt_bytes)
+ .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?;
+ ensure_settled_result_capacity(
+ next_retained_bytes,
+ self.pending
+ .iter()
+ .filter(|(call_id, _)| *call_id != &result.call_id)
+ .map(|(_, call)| call),
+ )?;
self.pending.remove(&result.call_id);
self.retained_result_bytes = next_retained_bytes;
- self.completed
- .insert(result.call_id.clone(), result.clone());
+ self.completed.insert(result.call_id.clone(), completed);
Ok(result.result)
}
- pub fn settle_turn(&mut self) -> Result<(), ProviderBridgeError> {
- if !self.pending.is_empty() {
- return Err(ProviderBridgeError::invalid(
- "cannot settle provider tool receipts while calls are pending",
- ));
- }
- self.retained_result_bytes =
- retained_result_bytes(self.settled_results.iter().chain(self.completed.iter()))?;
- self.ensure_settled_result_capacity(0)?;
- // The identity capacity was reserved in `begin_call`, so moving the
- // authoritative receipts cannot fail a valid admitted turn. The check
- // above rejects only recovered state that bypassed attach validation.
- self.settled_results.append(&mut self.completed);
- Ok(())
- }
-
pub fn pending_calls(&self) -> impl Iterator<Item = &PendingToolCall> {
self.pending.values()
}
- fn ensure_settled_result_capacity(
- &self,
- additional_pending: usize,
- ) -> Result<(), ProviderBridgeError> {
- let pending_count = self
+ pub fn cancel_pending_calls(
+ &mut self,
+ code: &str,
+ ) -> Result<Vec<ToolResult>, ProviderBridgeError> {
+ let mut next = self.clone();
+ let results = next.cancel_pending_calls_internal(code)?;
+ *self = next;
+ Ok(results)
+ }
+
+ pub fn settle_turn(&mut self, code: &str) -> Result<Vec<ToolResult>, ProviderBridgeError> {
+ validate_stable_id(code, "tool cancellation code")?;
+ let results = self
.pending
- .len()
- .checked_add(additional_pending)
- .ok_or_else(|| {
- ProviderBridgeError::invalid("durable provider result count overflow")
- })?;
- let pending_reserve = pending_count
- .checked_mul(MAX_SETTLED_RESULT_ENTRY_BYTES)
- .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))?;
+ .values()
+ .map(|call| cancelled_tool_result(call, code))
+ .collect::<Vec<_>>();
+ let mut settled_entries = self.completed.clone();
+ for (call_id, call) in &self.pending {
+ settled_entries.insert(
+ call_id.clone(),
+ CompletedToolCall {
+ call: call.clone(),
+ result: cancelled_tool_result(call, code),
+ },
+ );
+ }
+ let next_retained_bytes =
+ retained_result_bytes(self.settled_results.iter().chain(settled_entries.iter()))?;
+ ensure_settled_result_capacity(next_retained_bytes, std::iter::empty())?;
+
+ // Byte capacity was reserved at admission. Keep every identity exact
+ // for the lifetime of this provider-process epoch; prepare_turn
+ // releases only the bulky result bodies.
+ let new_identity_count = settled_entries
+ .keys()
+ .filter(|call_id| !self.settled_call_ids.contains(call_id))
+ .count();
if self
- .retained_result_bytes
- .checked_add(pending_reserve)
- .is_none_or(|bytes| bytes > MAX_SETTLED_RESULT_BYTES)
+ .settled_call_ids
+ .len()
+ .checked_add(new_identity_count)
+ .is_none_or(|total| total > MAX_SETTLED_CALL_IDS)
{
+ return Err(ProviderBridgeError::active_turn_receipt_limit());
+ }
+ let evicted = self
+ .settled_call_ids
+ .extend_recent(settled_entries.keys().cloned(), MAX_SETTLED_CALL_IDS);
+ debug_assert!(evicted.is_empty());
+ self.pending.clear();
+ self.completed.clear();
+ self.settled_results.append(&mut settled_entries);
+ self.retained_result_bytes = next_retained_bytes;
+ Ok(results)
+ }
+
+ fn cancel_pending_calls_internal(
+ &mut self,
+ code: &str,
+ ) -> Result<Vec<ToolResult>, ProviderBridgeError> {
+ validate_stable_id(code, "tool cancellation code")?;
+ let pending = self.pending.values().cloned().collect::<Vec<_>>();
+ let mut results = Vec::with_capacity(pending.len());
+ for call in pending {
+ let result = cancelled_tool_result(&call, code);
+ self.apply_result(result.clone())?;
+ results.push(result);
+ }
+ Ok(results)
+ }
+
+ fn retained_value_bytes(&self) -> Result<usize, ProviderBridgeError> {
+ let pending = self
+ .pending
+ .values()
+ .map(|call| json_size(&call.input, "retained provider tool input"));
+ let completed = self.completed.values().flat_map(|entry| {
+ [
+ json_size(&entry.call.input, "retained provider tool input"),
+ json_size(&entry.result.result, "retained provider tool result"),
+ ]
+ });
+ let settled = self.settled_results.values().flat_map(|entry| {
+ [
+ json_size(&entry.call.input, "retained provider tool input"),
+ json_size(&entry.result.result, "retained provider tool result"),
+ ]
+ });
+ pending
+ .chain(completed)
+ .chain(settled)
+ .try_fold(0usize, |total, bytes| {
+ total.checked_add(bytes?).ok_or_else(|| {
+ ProviderBridgeError::invalid("retained provider tool values overflow")
+ })
+ })
+ }
+
+ fn pending_value_bytes(&self) -> Result<usize, ProviderBridgeError> {
+ self.pending.values().try_fold(0usize, |total, call| {
+ total
+ .checked_add(json_size(&call.input, "retained provider tool input")?)
+ .ok_or_else(|| {
+ ProviderBridgeError::invalid("retained provider tool values overflow")
+ })
+ })
+ }
+
+ fn total_call_receipts(&self) -> usize {
+ self.pending.len().saturating_add(self.completed.len())
+ }
+
+ fn validate_retained_value_bytes(&self) -> Result<(), ProviderBridgeError> {
+ if self.retained_value_bytes()? > MAX_RETAINED_TOOL_VALUE_BYTES {
return Err(ProviderBridgeError::invalid(
- "durable provider tool result byte limit reached",
+ "retained provider tool values exceed the 8 MiB aggregate limit",
));
}
Ok(())
}
+
+ fn ensure_settled_result_capacity(
+ &self,
+ additional_pending: Option<&PendingToolCall>,
+ ) -> Result<(), ProviderBridgeError> {
+ ensure_settled_result_capacity(
+ self.retained_result_bytes,
+ self.pending.values().chain(additional_pending.into_iter()),
+ )
+ }
}
-fn validate_retained_result(result: &ToolResult) -> Result<(), ProviderBridgeError> {
- if !is_stable_call_id(&result.call_id) {
+fn validate_retained_result(entry: &CompletedToolCall) -> Result<(), ProviderBridgeError> {
+ validate_stable_id(&entry.call.call_id, "retained tool call id")?;
+ validate_stable_id(&entry.result.call_id, "retained tool result call id")?;
+ validate_operation_id(&entry.call.operation_id)?;
+ validate_operation_id(&entry.result.operation_id)?;
+ if entry.call.call_id != entry.result.call_id
+ || entry.call.operation_id != entry.result.operation_id
+ {
return Err(ProviderBridgeError::invalid(
- "retained tool result call id is invalid",
+ "retained provider tool receipt identity is inconsistent",
));
}
- validate_operation_id(&result.operation_id)?;
bounded_json(
- &result.result,
+ &entry.call.input,
+ MAX_TOOL_VALUE_BYTES,
+ "retained provider tool input",
+ )?;
+ bounded_json(
+ &entry.result.result,
MAX_TOOL_VALUE_BYTES,
"retained provider tool result",
)
@@ -439,9 +927,7 @@ fn validate_pending_tool_call(
authorized: &BTreeMap<String, AuthorizedTool>,
call: &PendingToolCall,
) -> Result<(), ProviderBridgeError> {
- if !is_stable_call_id(&call.call_id) {
- return Err(ProviderBridgeError::invalid("tool call id is invalid"));
- }
+ validate_stable_id(&call.call_id, "tool call id")?;
validate_operation_id(&call.operation_id)?;
let tool = authorized.get(&call.operation_id).ok_or_else(|| {
ProviderBridgeError::invalid(format!(
@@ -571,16 +1057,16 @@ fn validate_authorized_tool_set(tool_set: &AuthorizedToolSet) -> Result<(), Prov
fn retained_result_entry_bytes(
call_id: &str,
- result: &ToolResult,
+ result: &CompletedToolCall,
) -> Result<usize, ProviderBridgeError> {
// A two-item tuple has the same delimiter cost as a one-entry JSON map.
// Summing tuples therefore equals one entry exactly and conservatively
// overcounts a multi-entry map by one byte per additional receipt.
- encoded_json_bytes(&(call_id, result), "retained provider tool result")
+ json_size(&(call_id, result), "retained provider tool result")
}
fn retained_result_bytes<'a>(
- results: impl IntoIterator<Item = (&'a String, &'a ToolResult)>,
+ results: impl IntoIterator<Item = (&'a String, &'a CompletedToolCall)>,
) -> Result<usize, ProviderBridgeError> {
results
.into_iter()
@@ -593,16 +1079,53 @@ fn retained_result_bytes<'a>(
})
}
+fn pending_result_reserve_bytes(call: &PendingToolCall) -> Result<usize, ProviderBridgeError> {
+ let placeholder = CompletedToolCall {
+ call: call.clone(),
+ result: ToolResult {
+ call_id: call.call_id.clone(),
+ operation_id: call.operation_id.clone(),
+ result: Value::Null,
+ is_error: false,
+ },
+ };
+ retained_result_entry_bytes(&call.call_id, &placeholder)?
+ .checked_sub(json_size(&Value::Null, "provider tool result reserve")?)
+ .and_then(|bytes| bytes.checked_add(MAX_TOOL_VALUE_BYTES))
+ .ok_or_else(|| ProviderBridgeError::invalid("durable provider result size overflow"))
+}
+
+fn ensure_settled_result_capacity<'a>(
+ retained_bytes: usize,
+ pending: impl IntoIterator<Item = &'a PendingToolCall>,
+) -> Result<(), ProviderBridgeError> {
+ let reserved_bytes = pending
+ .into_iter()
+ .try_fold(retained_bytes, |total, call| {
+ total
+ .checked_add(pending_result_reserve_bytes(call)?)
+ .ok_or_else(|| {
+ ProviderBridgeError::invalid("durable provider result size overflow")
+ })
+ })?;
+ if reserved_bytes > MAX_SETTLED_RESULT_BYTES {
+ return Err(ProviderBridgeError::invalid(
+ "durable provider tool result byte limit reached",
+ ));
+ }
+ Ok(())
+}
+
fn deserialize_retained_results<'de, D>(
deserializer: D,
-) -> Result<BTreeMap<String, ToolResult>, D::Error>
+) -> Result<BTreeMap<String, CompletedToolCall>, D::Error>
where
D: Deserializer<'de>,
{
struct RetainedResultsVisitor;
impl<'de> Visitor<'de> for RetainedResultsVisitor {
- type Value = BTreeMap<String, ToolResult>;
+ type Value = BTreeMap<String, CompletedToolCall>;
fn expecting(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
formatter.write_str("a bounded map of retained provider tool results")
@@ -622,14 +1145,14 @@ where
}
let mut results = BTreeMap::new();
let mut retained_bytes = 0usize;
- while let Some((call_id, result)) = map.next_entry::<String, ToolResult>()? {
+ while let Some((call_id, result)) = map.next_entry::<String, CompletedToolCall>()? {
if results.len() >= MAX_SETTLED_CALL_IDS {
return Err(de::Error::custom(
"retained provider tool result count exceeds the durable limit",
));
}
validate_retained_result(&result).map_err(de::Error::custom)?;
- if call_id != result.call_id {
+ if call_id != result.call.call_id || call_id != result.result.call_id {
return Err(de::Error::custom(
"retained provider tool result identity is inconsistent",
));
@@ -658,8 +1181,19 @@ where
deserializer.deserialize_map(RetainedResultsVisitor)
}
-fn is_stable_call_id(value: &str) -> bool {
- !value.is_empty() && value.len() <= 160 && !value.chars().any(char::is_control)
+fn cancelled_tool_result(call: &PendingToolCall, code: &str) -> ToolResult {
+ ToolResult {
+ call_id: call.call_id.clone(),
+ operation_id: call.operation_id.clone(),
+ result: serde_json::json!({
+ "error": {
+ "code": code,
+ "message": "The provider turn stopped before this semantic tool completed",
+ "retryable": false,
+ },
+ }),
+ is_error: true,
+ }
}
pub fn authorized_tool_catalog_digest(
@@ -678,11 +1212,16 @@ pub fn authorized_tool_catalog_digest(
Ok(format!("sha256:{digest:x}"))
}
+pub fn semantic_value_digest(value: &Value) -> String {
+ let digest = Sha256::digest(canonical_json(value).as_bytes());
+ format!("sha256:{digest:x}")
+}
+
fn canonical_json(value: &Value) -> String {
match value {
Value::Null => "null".to_owned(),
Value::Bool(value) => value.to_string(),
- Value::Number(value) => value.to_string(),
+ Value::Number(value) => canonical_json_number(value),
Value::String(value) => {
serde_json::to_string(value).expect("serializing an in-memory JSON string cannot fail")
}
@@ -696,7 +1235,7 @@ fn canonical_json(value: &Value) -> String {
),
Value::Object(object) => {
let mut entries = object.iter().collect::<Vec<_>>();
- entries.sort_by_key(|(key, _)| *key);
+ entries.sort_by(|(left, _), (right, _)| left.encode_utf16().cmp(right.encode_utf16()));
format!(
"{{{}}}",
entries
@@ -714,6 +1253,82 @@ fn canonical_json(value: &Value) -> String {
}
}
+fn canonical_json_number(value: &serde_json::Number) -> String {
+ if value.is_i64() || value.is_u64() {
+ return value.to_string();
+ }
+ let Some(float) = value.as_f64() else {
+ return value.to_string();
+ };
+ if float == 0.0 {
+ return "0".to_owned();
+ }
+
+ // serde_json preserves valid lexical spellings such as `1.0`, whereas
+ // JSON.stringify canonicalizes JavaScript numbers. Normalize the shortest
+ // serde representation to the ECMAScript decimal/exponent thresholds so
+ // the server and runner hash the same schema value.
+ let encoded = value.to_string().to_ascii_lowercase();
+ let (negative, unsigned) = encoded
+ .strip_prefix('-')
+ .map_or((false, encoded.as_str()), |rest| (true, rest));
+ let (coefficient, explicit_exponent) = unsigned
+ .split_once('e')
+ .map_or((unsigned, 0_i32), |(coefficient, exponent)| {
+ (coefficient, exponent.parse::<i32>().unwrap_or(0))
+ });
+ let fraction_digits = coefficient
+ .split_once('.')
+ .map_or(0_i32, |(_, fraction)| fraction.len() as i32);
+ let mut digits = coefficient
+ .bytes()
+ .filter(|byte| *byte != b'.')
+ .map(char::from)
+ .collect::<String>();
+ let mut decimal_position = digits.len() as i32 + explicit_exponent - fraction_digits;
+
+ let leading_zeros = digits.bytes().take_while(|byte| *byte == b'0').count();
+ digits.drain(..leading_zeros);
+ decimal_position -= leading_zeros as i32;
+ while digits.ends_with('0') {
+ digits.pop();
+ }
+ if digits.is_empty() {
+ return "0".to_owned();
+ }
+
+ let body = if (1e-6..1e21).contains(&float.abs()) {
+ if decimal_position <= 0 {
+ format!("0.{}{}", "0".repeat((-decimal_position) as usize), digits)
+ } else if decimal_position >= digits.len() as i32 {
+ format!(
+ "{}{}",
+ digits,
+ "0".repeat((decimal_position - digits.len() as i32) as usize)
+ )
+ } else {
+ let split = decimal_position as usize;
+ format!("{}.{}", &digits[..split], &digits[split..])
+ }
+ } else {
+ let exponent = decimal_position - 1;
+ let coefficient = if digits.len() == 1 {
+ digits
+ } else {
+ format!("{}.{}", &digits[..1], &digits[1..])
+ };
+ format!(
+ "{coefficient}e{}{exponent}",
+ if exponent >= 0 { "+" } else { "" }
+ )
+ };
+ if negative {
+ format!("-{body}")
+ } else {
+ body
+ }
+}
+
fn semantic_response_value(result: &ToolResult) -> Result<Option<&Value>, ProviderBridgeError> {
let Some(envelope) = result.result.as_object() else {
return Ok(Some(&result.result));
@@ -732,9 +1347,11 @@ fn semantic_response_value(result: &ToolResult) -> Result<Option<&Value>, Provid
));
}
if ok {
- envelope.get("result").map(Some).ok_or_else(|| {
- ProviderBridgeError::invalid("successful semantic result omitted result")
- })
+ envelope
+ .get("result")
+ .or_else(|| envelope.get("value"))
+ .map(Some)
+ .ok_or_else(|| ProviderBridgeError::invalid("successful semantic result omitted value"))
} else if envelope.get("denial").is_some() || envelope.get("error").is_some() {
Ok(None)
} else {
@@ -745,6 +1362,10 @@ fn semantic_response_value(result: &ToolResult) -> Result<Option<&Value>, Provid
}
fn validate_operation_id(value: &str) -> Result<(), ProviderBridgeError> {
+ validate_stable_id(value, "tool operation id")
+}
+
+fn validate_stable_id(value: &str, label: &str) -> Result<(), ProviderBridgeError> {
let mut chars = value.chars();
let first = chars
.next()
@@ -755,7 +1376,7 @@ fn validate_operation_id(value: &str) -> Result<(), ProviderBridgeError> {
if first && rest && value.len() <= 160 {
Ok(())
} else {
- Err(ProviderBridgeError::invalid("tool operation id is invalid"))
+ Err(ProviderBridgeError::invalid(format!("{label} is invalid")))
}
}
@@ -774,7 +1395,7 @@ fn bounded_json(
max_bytes: usize,
label: &str,
) -> Result<(), ProviderBridgeError> {
- let bytes = encoded_json_bytes(value, label)?;
+ let bytes = json_size(value, label)?;
if bytes > max_bytes {
return Err(ProviderBridgeError::invalid(format!(
"{label} exceeds the {max_bytes} byte limit"
@@ -783,7 +1404,7 @@ fn bounded_json(
Ok(())
}
-fn encoded_json_bytes(value: &impl Serialize, label: &str) -> Result<usize, ProviderBridgeError> {
+fn json_size(value: &impl Serialize, label: &str) -> Result<usize, ProviderBridgeError> {
serde_json::to_vec(value)
.map(|bytes| bytes.len())
.map_err(|_| ProviderBridgeError::invalid(format!("{label} is not serializable")))
@@ -792,24 +1413,524 @@ fn encoded_json_bytes(value: &impl Serialize, label: &str) -> Result<usize, Prov
#[cfg(test)]
mod tests {
use super::*;
+ use serde_json::json;
#[test]
fn worst_case_result_identity_overhead_fits_the_admission_reserve() {
- let call_id = "\\".repeat(160);
- let result = ToolResult {
+ let call_id = "x".repeat(160);
+ let call = PendingToolCall {
call_id: call_id.clone(),
operation_id: "x".repeat(160),
- result: Value::String("x".repeat(MAX_TOOL_VALUE_BYTES - 2)),
- is_error: false,
+ input: Value::String("x".repeat(MAX_TOOL_VALUE_BYTES - 2)),
+ };
+ let completed = CompletedToolCall {
+ call: call.clone(),
+ result: ToolResult {
+ call_id: call_id.clone(),
+ operation_id: call.operation_id.clone(),
+ result: Value::String("x".repeat(MAX_TOOL_VALUE_BYTES - 2)),
+ is_error: false,
+ },
};
assert_eq!(
- encoded_json_bytes(&result.result, "test result").unwrap(),
+ json_size(&completed.result.result, "test result").unwrap(),
MAX_TOOL_VALUE_BYTES
);
assert!(
- retained_result_entry_bytes(&call_id, &result).unwrap()
- <= MAX_SETTLED_RESULT_ENTRY_BYTES
+ retained_result_entry_bytes(&call_id, &completed).unwrap()
+ <= pending_result_reserve_bytes(&call).unwrap()
+ );
+ }
+
+ #[test]
+ fn exact_completed_results_are_bounded_without_changing_replay() {
+ let operation = AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation.clone()],
+ })
+ .unwrap();
+ bridge
+ .begin_call(
+ "call-0".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect("begin the first exact call");
+ bridge
+ .apply_result(ToolResult {
+ call_id: "call-0".to_owned(),
+ operation_id: "get_task_context".to_owned(),
+ result: json!({"ok": true}),
+ is_error: false,
+ })
+ .expect("complete the first exact call");
+ assert_eq!(
+ bridge
+ .replay_result("call-0", "get_task_context", &json!({}))
+ .unwrap()
+ .unwrap()
+ .result,
+ json!({"ok": true})
+ );
+ for index in 1..MAX_DURABLE_CALL_RECEIPTS {
+ let call_id = format!("call-{index}");
+ bridge.completed.insert(
+ call_id.clone(),
+ CompletedToolCall {
+ call: PendingToolCall {
+ call_id: call_id.clone(),
+ operation_id: "get_task_context".to_owned(),
+ input: json!({}),
+ },
+ result: ToolResult {
+ call_id,
+ operation_id: "get_task_context".to_owned(),
+ result: json!({"ok": true}),
+ is_error: false,
+ },
+ },
+ );
+ }
+ let error = bridge
+ .begin_call(
+ "call-new".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("the exact receipt limit must stop active-turn growth");
+ assert!(error.is_active_turn_receipt_limit());
+
+ assert_eq!(
+ bridge
+ .replay_result("call-0", "get_task_context", &json!({}))
+ .unwrap()
+ .unwrap()
+ .result,
+ json!({"ok": true})
+ );
+ assert!(bridge
+ .apply_result(ToolResult {
+ call_id: "call-0".to_owned(),
+ operation_id: "get_task_context".to_owned(),
+ result: json!({"ok": false}),
+ is_error: false,
+ })
+ .is_err());
+ bridge.validate_retained_value_bytes().unwrap();
+ assert!(serde_json::to_vec_pretty(&bridge).unwrap().len() < 4 * 1024 * 1024);
+ let recovered: ProviderToolBridge =
+ serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();
+ recovered.validate_recovered().unwrap();
+ assert_eq!(
+ recovered
+ .replay_result("call-0", "get_task_context", &json!({}))
+ .unwrap()
+ .unwrap()
+ .result,
+ json!({"ok": true})
+ );
+
+ bridge.settle_turn("provider_turn_terminated").unwrap();
+ assert!(bridge
+ .begin_call(
+ "call-0".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .is_err());
+ bridge.prepare_turn().unwrap();
+ assert!(bridge
+ .replay_result("call-0", "get_task_context", &json!({}))
+ .unwrap()
+ .is_none());
+ assert!(bridge
+ .begin_call(
+ "call-0".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .is_err());
+ bridge
+ .begin_call(
+ "next-turn-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect("a new identity remains admissible after bulky results are released");
+
+ let recovered: ProviderToolBridge =
+ serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();
+ recovered.validate_recovered().unwrap();
+ assert!(recovered.has_completed_call("call-0"));
+ }
+
+ #[test]
+ fn admission_reserves_exact_identity_capacity_for_settlement() {
+ let operation = AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation.clone()],
+ })
+ .unwrap();
+ for index in 0..MAX_SETTLED_CALL_IDS - 1 {
+ let call_id = format!("settled-call-{index}");
+ bridge.settled_call_ids.order.push_back(call_id.clone());
+ bridge.settled_call_ids.members.insert(call_id);
+ }
+
+ bridge
+ .begin_call(
+ "final-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect("the final exact identity slot remains usable");
+ let mut pending_at_capacity = bridge.clone();
+ let error = pending_at_capacity
+ .begin_call(
+ "rejected-while-pending".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("a pending call must reserve its eventual settled identity");
+ assert!(error.is_active_turn_receipt_limit());
+ assert!(pending_at_capacity.durable_run_receipt_limit_reached);
+ assert_eq!(pending_at_capacity.pending.len(), 1);
+ assert!(pending_at_capacity.pending.contains_key("final-call"));
+ assert!(!pending_at_capacity
+ .pending
+ .contains_key("rejected-while-pending"));
+ pending_at_capacity
+ .begin_call(
+ "final-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect("an idempotent retry remains valid at capacity");
+ let cancelled = pending_at_capacity
+ .settle_turn("semantic_tool_turn_receipt_limit")
+ .unwrap();
+ assert_eq!(cancelled.len(), 1);
+ assert_eq!(
+ pending_at_capacity.settled_call_ids.len(),
+ MAX_SETTLED_CALL_IDS
);
+
+ bridge
+ .apply_result(ToolResult {
+ call_id: "final-call".to_owned(),
+ operation_id: "get_task_context".to_owned(),
+ result: json!({"ok": true}),
+ is_error: false,
+ })
+ .expect("complete the call admitted into the final identity slot");
+
+ let mut overcommitted = bridge.clone();
+ overcommitted.pending.insert(
+ "unsettleable-call".to_owned(),
+ PendingToolCall {
+ call_id: "unsettleable-call".to_owned(),
+ operation_id: "get_task_context".to_owned(),
+ input: json!({}),
+ },
+ );
+ assert_eq!(
+ overcommitted.validate_recovered().unwrap_err().to_string(),
+ "recovered provider tool bridge cannot settle its active call identities"
+ );
+ let overcommitted_snapshot = overcommitted.clone();
+ let error = overcommitted
+ .settle_turn("semantic_tool_turn_receipt_limit")
+ .expect_err("defensive settlement must reject an overcommitted recovered state");
+ assert!(error.is_active_turn_receipt_limit());
+ assert_eq!(overcommitted, overcommitted_snapshot);
+
+ let error = bridge
+ .begin_call(
+ "rejected-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("admission must reserve space for every active identity");
+ assert!(error.is_active_turn_receipt_limit());
+ assert!(bridge.durable_run_receipt_limit_reached);
+ assert!(!bridge.pending.contains_key("rejected-call"));
+ assert!(bridge.completed.contains_key("final-call"));
+
+ assert!(bridge
+ .settle_turn("semantic_tool_turn_receipt_limit")
+ .unwrap()
+ .is_empty());
+ assert!(bridge.pending.is_empty());
+ assert!(bridge.completed.is_empty());
+ assert_eq!(bridge.settled_call_ids.len(), MAX_SETTLED_CALL_IDS);
+ assert!(bridge.settled_results.contains_key("final-call"));
+
+ bridge.prepare_turn().unwrap();
+ assert_eq!(bridge.settled_call_ids.len(), MAX_SETTLED_CALL_IDS);
+ assert!(bridge.settled_results.is_empty());
+ assert!(bridge.durable_run_receipt_limit_reached);
+ let error = bridge
+ .begin_call(
+ "next-turn-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("a turn boundary must not reopen a saturated durable run");
+ assert!(error.is_active_turn_receipt_limit());
+
+ let encoded = serde_json::to_string(&bridge).unwrap();
+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();
+ recovered.attach_existing_run().unwrap();
+ assert_eq!(recovered.settled_call_ids.len(), MAX_SETTLED_CALL_IDS);
+ assert!(recovered.durable_run_receipt_limit_reached);
+ let error = recovered
+ .begin_call(
+ "recovered-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("recovery must preserve saturated replay authority");
+ assert!(error.is_active_turn_receipt_limit());
+
+ recovered
+ .attach_run(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation.clone()],
+ })
+ .unwrap();
+ assert!(recovered.settled_call_ids.is_empty());
+ assert!(!recovered.durable_run_receipt_limit_reached);
+ recovered
+ .begin_call(
+ "settled-call-0".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect("a new durable run receives fresh replay authority");
+ }
+
+ #[test]
+ fn legacy_replay_filter_quarantines_the_durable_run() {
+ let operation = AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation.clone()],
+ })
+ .unwrap();
+ bridge.settled_call_filter.words = vec![0; REPLAY_FILTER_WORDS];
+ bridge.settled_call_filter.words[0] = 1;
+
+ let error = bridge
+ .begin_call(
+ "legacy-evicted-or-fresh".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("a legacy probabilistic epoch cannot dispatch more work");
+ assert!(error.is_active_turn_receipt_limit());
+ assert!(bridge.pending.is_empty());
+ assert!(bridge.durable_run_receipt_limit_reached);
+
+ bridge.prepare_turn().unwrap();
+ assert!(!bridge.settled_call_filter.is_empty());
+ assert!(bridge.durable_run_receipt_limit_reached);
+ let error = bridge
+ .begin_call(
+ "legacy-evicted-or-fresh".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("a turn boundary cannot make legacy ambiguity safe");
+ assert!(error.is_active_turn_receipt_limit());
+
+ let encoded = serde_json::to_string(&bridge).unwrap();
+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();
+ recovered.attach_existing_run().unwrap();
+ assert!(!recovered.settled_call_filter.is_empty());
+ assert!(recovered.durable_run_receipt_limit_reached);
+
+ recovered
+ .attach_run(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation],
+ })
+ .unwrap();
+ assert!(recovered.settled_call_filter.is_empty());
+ assert!(!recovered.durable_run_receipt_limit_reached);
+ recovered
+ .begin_call(
+ "legacy-evicted-or-fresh".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect("a new durable run starts a fresh exact receipt ledger");
+ }
+
+ #[test]
+ fn legacy_full_exact_receipt_epoch_is_quarantined_before_dispatch() {
+ let operation = AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation.clone()],
+ })
+ .unwrap();
+ for index in 0..MAX_SETTLED_CALL_IDS {
+ let call_id = format!("legacy-call-{index}");
+ bridge.settled_call_ids.order.push_back(call_id.clone());
+ bridge.settled_call_ids.members.insert(call_id);
+ }
+
+ let error = bridge
+ .begin_call(
+ "fresh-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("a full legacy exact epoch cannot dispatch more work");
+ assert!(error.is_active_turn_receipt_limit());
+ assert!(bridge.pending.is_empty());
+
+ bridge.prepare_turn().unwrap();
+ let error = bridge
+ .begin_call(
+ "fresh-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect_err("a turn boundary must preserve the full durable ledger");
+ assert!(error.is_active_turn_receipt_limit());
+
+ bridge
+ .attach_run(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation],
+ })
+ .unwrap();
+ bridge
+ .begin_call(
+ "fresh-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect("a new durable run resets the full legacy ledger");
+ }
+
+ #[test]
+ fn provider_restart_rotates_only_an_idle_replay_epoch() {
+ let operation = AuthorizedTool {
+ operation_id: "get_task_context".to_owned(),
+ version: 1,
+ description: "Read the active task context.".to_owned(),
+ input_schema: json!({"type": "object"}),
+ response_schema: json!({"type": "object"}),
+ };
+ let mut bridge = ProviderToolBridge::default();
+ bridge
+ .prepare(AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation))
+ .unwrap(),
+ operations: vec![operation],
+ })
+ .unwrap();
+ bridge
+ .begin_call(
+ "old-epoch-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .unwrap();
+ bridge
+ .apply_result(ToolResult {
+ call_id: "old-epoch-call".to_owned(),
+ operation_id: "get_task_context".to_owned(),
+ result: json!({"ok": true}),
+ is_error: false,
+ })
+ .unwrap();
+ bridge.settle_turn("provider_turn_terminated").unwrap();
+ bridge.settled_call_filter.words = vec![0; REPLAY_FILTER_WORDS];
+ bridge.settled_call_filter.words[0] = 1;
+
+ let error = bridge
+ .rollover_replay_epoch_after_provider_restart()
+ .expect_err("retained replay results keep the current epoch authoritative");
+ assert!(error.to_string().contains("receipts are retained"));
+ assert!(bridge
+ .begin_call(
+ "old-epoch-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .is_err());
+
+ bridge.prepare_turn().unwrap();
+ bridge
+ .rollover_replay_epoch_after_provider_restart()
+ .expect("an idle replacement process establishes fresh replay authority");
+ bridge
+ .begin_call(
+ "old-epoch-call".to_owned(),
+ "get_task_context".to_owned(),
+ json!({}),
+ )
+ .expect("the fresh process generation remains live after bounded rotation");
}
}
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs
index 634f8ddda6..801763b8bc 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs
@@ -12,6 +12,29 @@ pub struct NormalizedProviderEvent {
pub payload: Value,
}
+pub(crate) fn normalized_codex_terminal_event_type(
+ method: &str,
+ params: &Value,
+) -> Option<&'static str> {
+ let status = match method {
+ "turn/failed" => "failed",
+ "turn/cancelled" => "cancelled",
+ "turn/interrupted" => "interrupted",
+ "turn/completed" => string(
+ params
+ .pointer("/turn/status")
+ .or_else(|| params.get("status")),
+ ),
+ _ => return None,
+ };
+ Some(match status {
+ "failed" | "error" => "turn.failed",
+ "cancelled" | "canceled" => "turn.cancelled",
+ "interrupted" | "aborted" => "turn.interrupted",
+ _ => "turn.completed",
+ })
+}
+
fn bounded_text(value: &str, max_chars: usize) -> String {
redact_text(value).chars().take(max_chars).collect()
}
@@ -118,18 +141,19 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec<Normali
"providerTurnId": params.pointer("/turn/id").or_else(|| params.get("turnId")).and_then(Value::as_str),
}),
),
- "turn/completed" => {
- let status = string(
- params
- .pointer("/turn/status")
- .or_else(|| params.get("status")),
- );
- let event_type = match status {
- "failed" | "error" => "turn.failed",
- "cancelled" | "canceled" => "turn.cancelled",
- "interrupted" | "aborted" => "turn.interrupted",
- _ => "turn.completed",
+ "turn/completed" | "turn/failed" | "turn/cancelled" | "turn/interrupted" => {
+ let status = match method {
+ "turn/failed" => "failed",
+ "turn/cancelled" => "cancelled",
+ "turn/interrupted" => "interrupted",
+ _ => string(
+ params
+ .pointer("/turn/status")
+ .or_else(|| params.get("status")),
+ ),
};
+ let event_type = normalized_codex_terminal_event_type(method, params)
+ .expect("matched Codex terminal method has a normalized terminal type");
push(
&mut events,
event_type,
@@ -366,6 +390,16 @@ mod tests {
);
assert_eq!(terminal[0].event_type, "turn.failed");
assert_eq!(terminal[0].priority, EventPriority::P0);
+ for (method, expected) in [
+ ("turn/failed", "turn.failed"),
+ ("turn/cancelled", "turn.cancelled"),
+ ("turn/interrupted", "turn.interrupted"),
+ ] {
+ let terminal =
+ normalize_codex_notification(method, &json!({"turnId": "provider-turn"}));
+ assert_eq!(terminal[0].event_type, expected);
+ assert_eq!(terminal[0].priority, EventPriority::P0);
+ }
let usage = normalize_codex_notification(
"thread/tokenUsage/updated",
diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
index 08c3380360..4e8050f1b1 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
@@ -5,9 +5,14 @@ use std::sync::atomic::{AtomicU64, Ordering};
use paperclip_runner_core::codex_provider::{
CodexProvider, CodexProviderConfig, CodexProviderEvent,
};
-use paperclip_runner_core::durable::{Command, CommandExecutor, DurableRunnerError, PolledEvent};
+use paperclip_runner_core::durable::{
+ Command, CommandExecutor, DurableRunnerConfig, DurableRunnerError, PolledEvent,
+};
use paperclip_runner_core::provider_backend::CodexCommandExecutor;
-use paperclip_runner_core::provider_bridge::{AuthorizedTool, ToolResult};
+use paperclip_runner_core::provider_bridge::{
+ authorized_tool_catalog_digest, AuthorizedTool, AuthorizedToolSet, ProviderToolBridge,
+ ToolResult, TOOL_SET_SCHEMA,
+};
use paperclip_runner_core::provider_events::normalize_codex_notification;
use serde_json::{json, Value};
@@ -62,6 +67,36 @@ fn task_context_tool() -> AuthorizedTool {
}
}
+fn task_context_tool_set() -> AuthorizedToolSet {
+ let operations = vec![task_context_tool()];
+ AuthorizedToolSet {
+ schema: TOOL_SET_SCHEMA.to_owned(),
+ schema_version: 1,
+ catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(),
+ operations,
+ }
+}
+
+fn durable_config(directory: &Path) -> DurableRunnerConfig {
+ DurableRunnerConfig {
+ connect_url: "ws://127.0.0.1:3000/runner".to_owned(),
+ state_dir: directory.to_path_buf(),
+ runner_instance_id: "runner-1".to_owned(),
+ environment_lease_id: "lease-1".to_owned(),
+ run_id: "run-1".to_owned(),
+ normalized_session_id: "session-1".to_owned(),
+ turn_id: "turn-1".to_owned(),
+ item_id: "item-1".to_owned(),
+ runner_version: "test-1".to_owned(),
+ runner_digest: format!("sha256:{}", "a".repeat(64)),
+ max_outbox_bytes: 16 * 1024 * 1024,
+ p0_reserve_bytes: 1024 * 1024,
+ max_frame_bytes: 1024 * 1024,
+ reconnect_delay: std::time::Duration::from_millis(1),
+ max_runtime: std::time::Duration::from_secs(5),
+ }
+}
+
fn command(id: &str, sequence: u64, command_type: &str, payload: Value) -> Command {
Command {
schema: "paperclip.prp.command.v1".to_owned(),
@@ -83,6 +118,14 @@ fn call_count(directory: &Path, method: &str) -> usize {
.count()
}
+fn recorded_tool_responses(directory: &Path) -> Vec<String> {
+ fs::read_to_string(directory.join("calls.log"))
+ .unwrap_or_default()
+ .lines()
+ .filter_map(|line| line.strip_prefix("tool-response:").map(str::to_owned))
+ .collect()
+}
+
fn poll_and_ack(
executor: &mut CodexCommandExecutor,
) -> Result<Vec<PolledEvent>, DurableRunnerError> {
@@ -91,6 +134,132 @@ fn poll_and_ack(
Ok(events)
}
+fn wait_for_notification(provider: &mut CodexProvider, expected_method: &str) -> Value {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while std::time::Instant::now() < deadline {
+ match provider.poll().expect("poll provider notification") {
+ Some(CodexProviderEvent::Notification { method, params }) => {
+ if method == expected_method {
+ return params;
+ }
+ }
+ Some(_) => {}
+ None => std::thread::sleep(std::time::Duration::from_millis(1)),
+ }
+ }
+ panic!("did not observe Codex {expected_method} notification before the deadline");
+}
+
+fn wait_for_provider_error(provider: &mut CodexProvider) -> String {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while std::time::Instant::now() < deadline {
+ match provider.poll() {
+ Err(error) => return error.to_string(),
+ Ok(Some(_)) => {}
+ Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)),
+ }
+ }
+ panic!("did not observe the expected Codex provider error before the deadline");
+}
+
+fn wait_for_executor_event(
+ executor: &mut CodexCommandExecutor,
+ expected_event_type: &str,
+) -> PolledEvent {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while std::time::Instant::now() < deadline {
+ let events = poll_and_ack(executor).expect("poll Codex executor event");
+ if let Some(event) = events
+ .into_iter()
+ .find(|event| event.event_type == expected_event_type)
+ {
+ return event;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ panic!("did not observe Codex {expected_event_type} event before the deadline");
+}
+
+fn wait_for_provider_exit(provider: &mut CodexProvider) {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while std::time::Instant::now() < deadline {
+ if matches!(
+ provider.poll().expect("poll terminated provider"),
+ Some(CodexProviderEvent::Exited { .. })
+ ) {
+ return;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ panic!("the provider accepted a reused turn identity but remained live");
+}
+
+fn wait_for_reused_identity_reap(provider: &mut CodexProvider) {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while std::time::Instant::now() < deadline {
+ match provider.poll() {
+ Err(error) => assert!(
+ error.to_string().contains("reused a settled"),
+ "unexpected error while reaping reused provider identity: {error}"
+ ),
+ Ok(Some(CodexProviderEvent::Exited {
+ completed_turn_authoritative,
+ ..
+ })) => {
+ assert!(
+ !completed_turn_authoritative,
+ "accepted identity reuse must revoke prior completion authority"
+ );
+ return;
+ }
+ Ok(Some(_)) => {}
+ Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)),
+ }
+ }
+ panic!("the provider accepted a reused turn identity but was not reaped");
+}
+
+fn wait_for_fake_provider_idle(directory: &Path) {
+ let state_path = directory.join("fake-state.json");
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while std::time::Instant::now() < deadline {
+ let active_turn_id = fs::read(&state_path)
+ .ok()
+ .and_then(|bytes| serde_json::from_slice::<Value>(&bytes).ok())
+ .and_then(|state| state.get("activeTurnId").cloned());
+ if active_turn_id == Some(Value::Null) {
+ return;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ panic!("the fake provider did not persist its idle turn state before the deadline");
+}
+
+fn saturate_provider_tool_receipts(directory: &Path) {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(task_context_tool_set()).unwrap();
+ for index in 0..4_096 {
+ let call_id = format!("retained-call-{index}");
+ bridge
+ .begin_call(call_id.clone(), "get_task_context".into(), json!({}))
+ .unwrap();
+ bridge
+ .apply_result(ToolResult {
+ call_id,
+ operation_id: "get_task_context".into(),
+ result: json!({"ok": true}),
+ is_error: false,
+ })
+ .unwrap();
+ }
+ let state_path = directory.join("codex-provider-state.json");
+ let mut persisted: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read provider state")).unwrap();
+ persisted["toolBridge"] = serde_json::to_value(bridge).unwrap();
+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())
+ .expect("write saturated provider state");
+}
+
#[test]
fn codex_transport_buffers_notifications_while_waiting_for_responses() {
let directory = temporary_directory("buffering");
@@ -505,6 +674,285 @@ fn clean_provider_exit_does_not_refail_a_completed_turn() {
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
+#[test]
+fn durable_backend_reaps_before_rotating_full_provider_identity_epochs() {
+ let directory = temporary_directory("provider-identity-epoch-rollover");
+ let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]);
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ drop(first);
+
+ let state_path = directory.join("codex-provider-state.json");
+ let mut persisted: Value = serde_json::from_slice(
+ &fs::read(&state_path).expect("read provider state before epoch rollover"),
+ )
+ .expect("parse provider state before epoch rollover");
+ let prior_generation = persisted["providerProcessGeneration"]
+ .as_u64()
+ .expect("provider generation is persisted");
+ persisted["settledProviderTurnIds"] = Value::Array(
+ (0..4_096)
+ .map(|index| Value::String(format!("provider-turn-{index}")))
+ .collect(),
+ );
+ persisted["completedTurnAuthoritative"] = Value::Bool(true);
+ persisted["completedTurnProcessGeneration"] = json!(prior_generation);
+ persisted["completedProviderTurnId"] = json!("provider-turn-4095");
+ persisted["toolBridge"]["settledCallIds"] = Value::Array(
+ (0..65_536)
+ .map(|index| Value::String(format!("semantic-call-{index}")))
+ .collect(),
+ );
+ persisted["toolBridge"]["durableRunReceiptLimitReached"] = Value::Bool(true);
+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())
+ .expect("write full provider identity epochs");
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ recovered
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Start after a verified provider epoch rollover."}),
+ ))
+ .expect("roll over the idle provider generation and start fresh work");
+
+ let rolled: Value = serde_json::from_slice(
+ &fs::read(&state_path).expect("read provider state after epoch rollover"),
+ )
+ .expect("parse provider state after epoch rollover");
+ assert!(rolled["providerProcessGeneration"].as_u64().unwrap() > prior_generation);
+ assert_eq!(
+ rolled["settledProviderTurnIds"],
+ json!(["provider-turn-4095"])
+ );
+ assert_eq!(rolled["settledProviderTurnFilter"], json!({"words": []}));
+ assert_eq!(rolled["toolBridge"]["settledCallIds"], json!([]));
+ assert_eq!(
+ rolled["toolBridge"]["settledCallFilter"],
+ json!({"words": []})
+ );
+ assert!(rolled["toolBridge"]["durableRunReceiptLimitReached"].is_null());
+
+ let mut event_types = Vec::new();
+ let semantic_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while std::time::Instant::now() < semantic_deadline {
+ event_types.extend(
+ poll_and_ack(&mut recovered)
+ .expect("poll semantic work in the fresh provider epoch")
+ .into_iter()
+ .map(|event| event.event_type),
+ );
+ if event_types
+ .iter()
+ .any(|event| event == "semantic_tool.input")
+ {
+ break;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ assert!(event_types
+ .iter()
+ .any(|event| event == "semantic_tool.input"));
+
+ recovered.shutdown().expect("stop rolled provider process");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_closes_when_identity_rollover_resumes_unowned_work() {
+ let directory = temporary_directory("provider-identity-rollover-unowned-work");
+ let config = provider_config(&directory, &["--resume-unowned-turn-when-marked"]);
+ let mut first = CodexCommandExecutor::new(&directory);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ drop(first);
+
+ let state_path = directory.join("codex-provider-state.json");
+ let mut persisted: Value = serde_json::from_slice(
+ &fs::read(&state_path).expect("read provider state before unsafe rollover"),
+ )
+ .expect("parse provider state before unsafe rollover");
+ persisted["settledProviderTurnIds"] = Value::Array(
+ (0..4_096)
+ .map(|index| Value::String(format!("provider-turn-{index}")))
+ .collect(),
+ );
+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())
+ .expect("write full provider identity epoch");
+
+ let mut recovered = CodexCommandExecutor::new(&directory);
+ poll_and_ack(&mut recovered).expect("attach an idle provider before rollover");
+ wait_for_fake_provider_idle(&directory);
+ let attached: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read attached provider state"))
+ .expect("parse attached provider state");
+ let attached_generation = attached["providerProcessGeneration"]
+ .as_u64()
+ .expect("attached provider generation is persisted");
+
+ // Race the provider's idle snapshot with work Paperclip never dispatched.
+ // The replacement process observes this turn during thread/resume and must
+ // close the durable run instead of leaving a quarantined session open.
+ fs::write(directory.join("resume-unowned-turn"), b"armed")
+ .expect("arm unowned provider work before rollover");
+ let resumes_before_rollover = call_count(&directory, "thread/resume");
+ let error = recovered
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Never overlap the unowned provider turn."}),
+ ))
+ .expect_err("identity rollover fails closed after resuming unowned work");
+ assert!(error
+ .to_string()
+ .contains("identity epoch rollover failed closed after an invalid accepted identity"));
+ assert_eq!(
+ call_count(&directory, "thread/resume"),
+ resumes_before_rollover + 1,
+ );
+
+ let closed: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read fail-closed provider state"))
+ .expect("parse fail-closed provider state");
+ assert_eq!(closed["lifecycle"], "closed");
+ assert!(closed["activeProviderTurnId"].is_null());
+ assert_eq!(closed["ambiguousTurnStartPending"], false);
+ assert_eq!(closed["completedTurnAuthoritative"], false);
+ assert!(closed["providerProcessGeneration"].as_u64().unwrap() > attached_generation);
+
+ let events = poll_and_ack(&mut recovered).expect("read fail-closed rollover diagnostic");
+ assert!(events.iter().any(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "provider_turn_identity_invalid"
+ && event.payload["paperclipAccepted"] == false
+ && event.payload["providerAccepted"] == true
+ }));
+
+ let resumes_before_retry = call_count(&directory, "thread/resume");
+ assert!(recovered
+ .execute(&command(
+ "turn-retry",
+ 4,
+ "turn.start",
+ json!({"text": "Do not resume the quarantined provider."}),
+ ))
+ .unwrap_err()
+ .to_string()
+ .contains("provider session is closed"));
+ assert_eq!(
+ call_count(&directory, "thread/resume"),
+ resumes_before_retry,
+ "closed rollover state must never resume the unowned provider turn",
+ );
+
+ recovered.shutdown().expect("close fail-closed executor");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn direct_provider_reaps_before_rotating_a_full_turn_identity_epoch() {
+ let directory = temporary_directory("direct-turn-identity-epoch-rollover");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--resume-unowned-turn-when-marked",
+ ],
+ );
+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)
+ .expect("start Codex provider");
+ let initial_process_id = provider.process_id();
+
+ // Keep every identity exact until the process boundary makes it safe to
+ // forget them. The next turn must transparently resume in a new process
+ // generation instead of permanently rejecting this session.
+ for index in 0..4_096 {
+ provider
+ .start_turn(&format!("Complete provider turn {index}."), &config.cwd)
+ .expect("start provider turn before identity rollover");
+ wait_for_notification(&mut provider, "turn/completed");
+ }
+
+ provider
+ .start_turn(
+ "Continue after the exact identity epoch fills.",
+ &config.cwd,
+ )
+ .expect("roll over the provider process and start fresh work");
+ assert_ne!(provider.process_id(), initial_process_id);
+ assert_eq!(call_count(&directory, "thread/resume"), 1);
+ wait_for_notification(&mut provider, "turn/completed");
+ let rolled_process_id = provider.process_id();
+
+ // Fill the next process epoch and force its resume probe to observe an
+ // active turn. Rollover must retain that work instead of dispatching a
+ // concurrent replacement after forgetting the old exact identities. The
+ // prior generation's authoritative completion remains as one tombstone in
+ // this epoch, so 4,094 additional completions fill the remaining slots.
+ for index in 1..4_095 {
+ provider
+ .start_turn(
+ &format!("Complete rolled provider turn {index}."),
+ &config.cwd,
+ )
+ .expect("start provider turn in the rolled identity epoch");
+ wait_for_notification(&mut provider, "turn/completed");
+ }
+ assert_eq!(provider.process_id(), rolled_process_id);
+ // The terminal notification is flushed before the fake provider persists
+ // its idle state. Wait for that write, then arm a one-shot resume race so
+ // only the replacement generation reports unowned active work.
+ wait_for_fake_provider_idle(&directory);
+ fs::write(directory.join("resume-unowned-turn"), b"armed")
+ .expect("arm unowned work for the replacement provider resume");
+ let error = provider
+ .start_turn(
+ "Do not overlap the turn recovered during epoch rollover.",
+ &config.cwd,
+ )
+ .expect_err("a resumed active turn is reaped before replacement work");
+ assert!(error
+ .to_string()
+ .contains("resumed unowned active work; the provider was terminated"));
+ assert_eq!(provider.active_provider_turn_id(), None);
+ assert!(provider
+ .start_turn(
+ "Never admit replacement work after quarantine.",
+ &config.cwd,
+ )
+ .unwrap_err()
+ .to_string()
+ .contains("quarantined after unsafe recovered work"));
+ wait_for_provider_exit(&mut provider);
+
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
#[test]
fn post_completion_observation_does_not_hide_same_or_resumed_process_failure() {
let directory = temporary_directory("completion-then-nonzero-exit");
@@ -650,7 +1098,11 @@ fn rejected_replacement_turn_start_preserves_result_and_exit_authority() {
.start_turn("Reject replacement work.", &config.cwd)
.expect_err("the replacement turn/start returns a definite rejection");
let mut buffered_notification_seen = false;
- let rejected_start_exit = (0..64).find_map(|_| {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ let rejected_start_exit = loop {
+ if std::time::Instant::now() >= deadline {
+ break None;
+ }
match provider
.poll()
.expect("poll exit after rejected replacement start")
@@ -661,21 +1113,23 @@ fn rejected_replacement_turn_start_preserves_result_and_exit_authority() {
== Some("buffered before replacement rejection") =>
{
buffered_notification_seen = true;
- None
}
Some(CodexProviderEvent::Exited {
success,
completed_turn_authoritative,
completion_reconciles_exit,
..
- }) => Some((
- success,
- completed_turn_authoritative,
- completion_reconciles_exit,
- )),
- _ => None,
+ }) => {
+ break Some((
+ success,
+ completed_turn_authoritative,
+ completion_reconciles_exit,
+ ));
+ }
+ _ => {}
}
- });
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ };
assert!(buffered_notification_seen);
assert_eq!(rejected_start_exit, Some((false, true, false)));
@@ -727,7 +1181,11 @@ fn rejected_replacement_turn_start_does_not_hide_contradictory_turn_evidence() {
"unexpected duplicate-start error: {duplicate_error}"
);
let mut contradictory_turn_seen = false;
- let rejected_start_exit = (0..64).find_map(|_| {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ let rejected_start_exit = loop {
+ if std::time::Instant::now() >= deadline {
+ break None;
+ }
match provider
.poll()
.expect("poll exit after contradictory replacement rejection")
@@ -738,21 +1196,23 @@ fn rejected_replacement_turn_start_does_not_hide_contradictory_turn_evidence() {
== Some("provider-turn-contradiction") =>
{
contradictory_turn_seen = true;
- None
}
Some(CodexProviderEvent::Exited {
success,
completed_turn_authoritative,
completion_reconciles_exit,
..
- }) => Some((
- success,
- completed_turn_authoritative,
- completion_reconciles_exit,
- )),
- _ => None,
+ }) => {
+ break Some((
+ success,
+ completed_turn_authoritative,
+ completion_reconciles_exit,
+ ));
+ }
+ _ => {}
}
- });
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ };
assert!(contradictory_turn_seen);
assert_eq!(
provider.active_provider_turn_id(),
@@ -771,7 +1231,6 @@ fn ambiguous_or_dead_replacement_start_preserves_result_not_exit_authority() {
"--fail-after-accepting-second-turn-before-response",
),
("malformed-error", "--malformed-error-second-turn-start"),
- ("missing-turn-id", "--missing-id-second-turn-start"),
] {
let directory = temporary_directory(label);
let config = provider_config(&directory, &[switch]);
@@ -822,6 +1281,63 @@ fn ambiguous_or_dead_replacement_start_preserves_result_not_exit_authority() {
}
}
+#[test]
+fn durable_backend_closes_after_accepted_turn_omits_its_identity() {
+ let directory = temporary_directory("accepted-turn-missing-identity");
+ let config = provider_config(&directory, &["--missing-id-live-turn-start"]);
+ let runner_config = durable_config(&directory);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare Codex provider");
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+
+ let error = executor
+ .execute(&command(
+ "turn-without-id",
+ 3,
+ "turn.start",
+ json!({"text": "Accept work but omit its durable identity."}),
+ ))
+ .expect_err("accepted work without a turn identity must fail closed");
+ assert!(error.to_string().contains("omitted turn.id"));
+
+ let persisted: Value = serde_json::from_slice(
+ &fs::read(directory.join("codex-provider-state.json"))
+ .expect("read fail-closed provider state"),
+ )
+ .expect("parse fail-closed provider state");
+ assert_eq!(persisted["lifecycle"], "closed");
+ assert!(persisted["activeProviderTurnId"].is_null());
+
+ drop(executor);
+ let resumes_before_closed_restore = call_count(&directory, "thread/resume");
+ let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let events = closed
+ .poll_events()
+ .expect("closed invalid-identity state remains readable without resuming Codex");
+ assert!(events.iter().any(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "provider_turn_identity_invalid"
+ && event.payload["providerTurnId"].is_null()
+ }));
+ assert_eq!(
+ call_count(&directory, "thread/resume"),
+ resumes_before_closed_restore,
+ "recovery must not resume provider work accepted without a durable identity"
+ );
+
+ closed.shutdown().expect("close fail-closed executor");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
#[test]
fn ambiguous_replacement_turn_adopts_one_later_completion_identity() {
for (label, switch, omit_started) in [
@@ -940,7 +1456,7 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() {
let config = provider_config(
&directory,
&[
- "--missing-id-second-turn-start",
+ "--malformed-error-second-turn-start",
"--conflicting-ambiguous-second-turn",
],
);
@@ -948,41 +1464,23 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() {
provider
.start_turn("Complete the first turn.", &config.cwd)
.expect("start first provider turn");
- let first_completed = (0..32).any(|_| {
- matches!(
- provider.poll().expect("poll first turn"),
- Some(CodexProviderEvent::Notification { method, .. })
- if method == "turn/completed"
- )
- });
- assert!(
- first_completed,
- "observe the authoritative first completion"
- );
+ wait_for_notification(&mut provider, "turn/completed");
provider
.start_turn("Accept replacement work ambiguously.", &config.cwd)
- .expect_err("the replacement response omits its turn identity");
- let replacement_started = provider
- .poll()
- .expect("poll replacement start")
- .expect("replacement start is available");
- assert!(matches!(
- replacement_started,
- CodexProviderEvent::Notification { method, params }
- if method == "turn/started"
- && params.pointer("/turn/id").and_then(Value::as_str)
- == Some("provider-turn-2")
- ));
+ .expect_err("the replacement response is transport-ambiguous");
+ let replacement_started = wait_for_notification(&mut provider, "turn/started");
+ assert_eq!(
+ replacement_started
+ .pointer("/turn/id")
+ .and_then(Value::as_str),
+ Some("provider-turn-2")
+ );
assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));
- let conflicting_completion = provider
- .poll()
- .expect_err("a second replacement identity must fail closed");
+ let conflicting_completion = wait_for_provider_error(&mut provider);
assert!(
- conflicting_completion
- .to_string()
- .contains("another active turn"),
+ conflicting_completion.contains("another active turn"),
"unexpected conflicting-identity error: {conflicting_completion}"
);
assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));
@@ -990,6 +1488,53 @@ fn ambiguous_replacement_turn_rejects_conflicting_later_identity() {
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
+#[test]
+fn ambiguous_replacement_turn_rejects_an_older_settled_identity() {
+ let directory = temporary_directory("ambiguous-older-settled-turn-identity");
+ let config = provider_config(&directory, &["--ambiguous-older-reused-turn"]);
+ let mut provider = CodexProvider::start(&config, None).expect("start Codex provider");
+ for message in ["Complete turn one.", "Complete turn two."] {
+ provider
+ .start_turn(message, &config.cwd)
+ .expect("start completed provider turn");
+ wait_for_notification(&mut provider, "turn/completed");
+ }
+ provider
+ .start_turn("Ambiguously reuse the first turn identity.", &config.cwd)
+ .expect_err("the replacement response is transport-ambiguous");
+ let error = wait_for_provider_error(&mut provider);
+ assert!(
+ error.contains("reused a settled provider turn identity"),
+ "unexpected older-identity error: {error}"
+ );
+ assert_eq!(provider.active_provider_turn_id(), None);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ let completed_turn_authoritative = (0..)
+ .take_while(|_| std::time::Instant::now() < deadline)
+ .find_map(|_| {
+ match provider
+ .poll()
+ .expect("poll provider after rejected identity")
+ {
+ Some(CodexProviderEvent::Exited {
+ completed_turn_authoritative,
+ ..
+ }) => Some(completed_turn_authoritative),
+ Some(_) => None,
+ None => {
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ None
+ }
+ }
+ });
+ assert!(
+ completed_turn_authoritative == Some(false),
+ "accepted work with an older settled identity must terminate the provider and revoke completion authority"
+ );
+
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
#[test]
fn clean_exit_after_ambiguous_replacement_start_fails_the_durable_session() {
let directory = temporary_directory("durable-clean-exit-after-ambiguous-turn-start");
@@ -1161,7 +1706,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() {
let config = provider_config(
&directory,
&[
- "--missing-id-second-turn-start",
+ "--malformed-error-second-turn-start",
"--hold-ambiguous-second-turn-after-item",
],
);
@@ -1185,15 +1730,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() {
json!({"text": "Complete the first turn."}),
))
.expect("start first provider turn");
- for _ in 0..32 {
- if poll_and_ack(&mut executor)
- .expect("poll first turn")
- .iter()
- .any(|event| event.event_type == "turn.completed")
- {
- break;
- }
- }
+ wait_for_executor_event(&mut executor, "turn.completed");
executor
.execute(&command(
@@ -1202,7 +1739,7 @@ fn replacement_item_is_not_persisted_before_ambiguous_turn_identity() {
"turn.start",
json!({"text": "Emit replacement output before terminal authority."}),
))
- .expect_err("replacement response omits its identity");
+ .expect_err("replacement response is transport-ambiguous");
assert!(
poll_and_ack(&mut executor)
.expect("defer identity-less replacement output")
@@ -1252,7 +1789,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() {
let config = provider_config(
&directory,
&[
- "--missing-id-second-turn-start",
+ "--malformed-error-second-turn-start",
"--complete-ambiguous-second-turn-before-response",
"--omit-ambiguous-turn-started",
],
@@ -1277,15 +1814,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() {
json!({"text": "Complete the first turn."}),
))
.expect("start first provider turn");
- for _ in 0..32 {
- if poll_and_ack(&mut executor)
- .expect("poll first turn")
- .iter()
- .any(|event| event.event_type == "turn.completed")
- {
- break;
- }
- }
+ wait_for_executor_event(&mut executor, "turn.completed");
executor
.execute(&command(
@@ -1294,7 +1823,7 @@ fn completed_ambiguous_replacement_fails_closed_after_process_loss() {
"turn.start",
json!({"text": "Complete replacement work before returning an invalid response."}),
))
- .expect_err("replacement response omits its identity");
+ .expect_err("replacement response is transport-ambiguous");
executor.shutdown().expect("stop first provider process");
drop(executor);
@@ -1335,7 +1864,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() {
let config = provider_config(
&directory,
&[
- "--missing-id-second-turn-start",
+ "--malformed-error-second-turn-start",
"--complete-ambiguous-second-turn",
],
);
@@ -1360,19 +1889,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() {
))
.expect("start first provider turn");
- let mut first_events = Vec::new();
- for _ in 0..32 {
- first_events.extend(
- poll_and_ack(&mut executor)
- .expect("poll first turn")
- .into_iter()
- .map(|event| event.event_type),
- );
- if first_events.iter().any(|event| event == "turn.completed") {
- break;
- }
- }
- assert!(first_events.iter().any(|event| event == "turn.completed"));
+ wait_for_executor_event(&mut executor, "turn.completed");
executor
.execute(&command(
@@ -1381,10 +1898,11 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() {
"turn.start",
json!({"text": "Complete replacement work after the malformed response."}),
))
- .expect_err("accepted replacement response omits its turn identity");
+ .expect_err("accepted replacement response is transport-ambiguous");
let mut replacement_events = Vec::new();
- for _ in 0..64 {
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while std::time::Instant::now() < deadline {
replacement_events.extend(
poll_and_ack(&mut executor)
.expect("poll accepted replacement evidence")
@@ -1397,6 +1915,7 @@ fn ambiguous_replacement_completion_replaces_durable_turn_authority() {
{
break;
}
+ std::thread::sleep(std::time::Duration::from_millis(1));
}
assert!(replacement_events
.iter()
@@ -1479,81 +1998,128 @@ fn codex_rejects_a_tool_call_that_was_not_advertised() {
}
#[test]
-fn codex_resume_advertises_the_same_authorized_tools() {
- let directory = temporary_directory("dynamic-tool-resume");
- let config = provider_config(&directory, &["--require-dynamic-tool"]);
- let mut provider =
- CodexProvider::start_with_tools(&config, [task_context_tool()], Some("codex-thread-1"))
- .expect("resume Codex with the run-scoped tool set");
- assert_eq!(provider.thread_id(), "codex-thread-1");
- provider.shutdown().expect("stop provider");
+fn codex_rejects_delayed_calls_after_every_terminal_notification() {
+ let directory = temporary_directory("delayed-tool-after-failure");
+ let config = provider_config(&directory, &["--delayed-tool-after-failed-turn"]);
+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)
+ .expect("start Codex with an authorized tool");
+ provider
+ .start_turn("Fail before invoking a tool.", &config.cwd)
+ .expect("start provider turn");
+
+ wait_for_notification(&mut provider, "turn/failed");
+ assert_eq!(provider.active_provider_turn_id(), None);
+ let warning = wait_for_notification(&mut provider, "warning");
+ assert!(warning["message"]
+ .as_str()
+ .is_some_and(|message| message.contains("after the Codex turn terminated")));
+
+ let _ = provider.shutdown();
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
#[test]
-fn durable_backend_resumes_the_active_thread_without_restarting_the_turn() {
- let directory = temporary_directory("resume");
- let config = provider_config(&directory, &["--hold-turn"]);
- let mut first = CodexCommandExecutor::new(&directory);
- first
- .execute(&command(
- "prepare",
- 1,
- "run.prepare",
- json!({"provider": config}),
- ))
- .expect("prepare Codex provider");
- first
- .execute(&command("open", 2, "session.open", json!({})))
- .expect("open Codex session");
- first
- .execute(&command(
- "turn",
- 3,
- "turn.start",
- json!({"text": "Hold this turn for recovery."}),
- ))
- .expect("start held provider turn");
- assert_eq!(call_count(&directory, "turn/start"), 1);
- first.shutdown().expect("stop first provider process");
- drop(first);
+fn codex_rejects_a_delayed_prior_turn_call_while_the_next_turn_is_active() {
+ let directory = temporary_directory("delayed-tool-after-next-turn-start");
+ let config = provider_config(&directory, &["--delayed-tool-after-next-turn-start"]);
+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)
+ .expect("start Codex with an authorized tool");
+ provider
+ .start_turn("Complete the first turn.", &config.cwd)
+ .expect("start first provider turn");
+ wait_for_notification(&mut provider, "turn/completed");
- let mut recovered = CodexCommandExecutor::new(&directory);
- let snapshot = recovered
- .execute(&command("snapshot", 4, "session.snapshot", json!({})))
- .expect("restore provider session");
- assert_eq!(snapshot.result["status"], "turn_active");
- assert_eq!(snapshot.result["activeProviderTurnId"], "provider-turn-1");
- assert_eq!(call_count(&directory, "turn/start"), 1);
- assert_eq!(call_count(&directory, "thread/resume"), 1);
- assert_eq!(call_count(&directory, "thread/read"), 1);
+ provider
+ .start_turn("Keep the second turn active.", &config.cwd)
+ .expect("start second provider turn");
+ assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));
+ let warning = wait_for_notification(&mut provider, "warning");
+ assert!(warning["message"]
+ .as_str()
+ .is_some_and(|message| message.contains("after the Codex turn terminated")));
+ assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-2"));
- recovered
- .execute(&command("interrupt", 5, "turn.interrupt", json!({})))
- .expect("interrupt recovered provider turn");
- let mut terminal_seen = false;
- for _ in 0..16 {
- let events = poll_and_ack(&mut recovered).expect("poll interrupted turn");
- terminal_seen |= events
- .iter()
- .any(|event| event.event_type == "turn.interrupted");
- if terminal_seen {
- break;
- }
+ let _ = provider.shutdown();
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn codex_rejects_a_two_turn_old_call_while_fresh_work_is_active() {
+ let directory = temporary_directory("delayed-tool-after-third-turn-start");
+ let config = provider_config(&directory, &["--delayed-tool-after-third-turn-start"]);
+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)
+ .expect("start Codex with an authorized tool");
+
+ for message in ["Complete turn one.", "Complete turn two."] {
+ provider
+ .start_turn(message, &config.cwd)
+ .expect("start completed provider turn");
+ wait_for_notification(&mut provider, "turn/completed");
}
- assert!(terminal_seen);
- recovered
- .shutdown()
- .expect("stop recovered provider process");
+ provider
+ .start_turn("Keep turn three active.", &config.cwd)
+ .expect("start third provider turn");
+ assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-3"));
+
+ let warning = wait_for_notification(&mut provider, "warning");
+ assert_eq!(warning["providerMethod"], "item/tool/call");
+ assert_eq!(provider.active_provider_turn_id(), Some("provider-turn-3"));
+
+ let _ = provider.shutdown();
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
#[test]
-fn provider_exit_preserves_and_reconciles_the_active_turn() {
- let directory = temporary_directory("exit-active-turn");
- let config = provider_config(&directory, &["--exit-after-turn-start"]);
- let mut executor = CodexCommandExecutor::new(&directory);
- executor
+fn codex_fails_closed_when_a_provider_reuses_a_settled_turn_id() {
+ let directory = temporary_directory("tool-after-reused-turn-start");
+ let config = provider_config(&directory, &["--tool-after-reused-turn-start"]);
+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)
+ .expect("start Codex with an authorized tool");
+ provider
+ .start_turn("Complete the first turn.", &config.cwd)
+ .expect("start first provider turn");
+ wait_for_notification(&mut provider, "turn/completed");
+
+ let error = provider
+ .start_turn("Reuse the settled provider identity.", &config.cwd)
+ .expect_err("reject a provider turn with a reused identity");
+ assert!(error.to_string().contains("reused a settled"));
+ assert_eq!(provider.active_provider_turn_id(), None);
+ wait_for_reused_identity_reap(&mut provider);
+
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn codex_fails_closed_when_a_provider_reuses_an_older_settled_turn_id() {
+ let directory = temporary_directory("tool-after-older-reused-turn-start");
+ let config = provider_config(&directory, &["--tool-after-older-reused-turn-start"]);
+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)
+ .expect("start Codex with an authorized tool");
+ for message in ["Complete turn one.", "Complete turn two."] {
+ provider
+ .start_turn(message, &config.cwd)
+ .expect("start completed provider turn");
+ wait_for_notification(&mut provider, "turn/completed");
+ }
+
+ let error = provider
+ .start_turn("Reuse the older settled provider identity.", &config.cwd)
+ .expect_err("reject a provider turn with an older reused identity");
+ assert!(error.to_string().contains("reused a settled"));
+ assert_eq!(provider.active_provider_turn_id(), None);
+ wait_for_reused_identity_reap(&mut provider);
+
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_rejects_an_older_provider_turn_id_after_restart() {
+ let directory = temporary_directory("durable-older-reused-turn-after-restart");
+ let config = provider_config(&directory, &[]);
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
.execute(&command(
"prepare",
1,
@@ -1561,65 +2127,82 @@ fn provider_exit_preserves_and_reconciles_the_active_turn() {
json!({"provider": config}),
))
.expect("prepare Codex provider");
- executor
+ first
.execute(&command("open", 2, "session.open", json!({})))
.expect("open Codex session");
- executor
+
+ for (sequence, message) in [(3, "Complete turn one."), (4, "Complete turn two.")] {
+ first
+ .execute(&command(
+ &format!("turn-{sequence}"),
+ sequence,
+ "turn.start",
+ json!({"text": message}),
+ ))
+ .expect("start completed provider turn");
+ let mut completed = false;
+ for _ in 0..32 {
+ completed |= poll_and_ack(&mut first)
+ .expect("poll completed provider turn")
+ .iter()
+ .any(|event| event.event_type == "turn.completed");
+ if completed {
+ break;
+ }
+ }
+ assert!(completed);
+ }
+ first.shutdown().expect("stop first provider process");
+ drop(first);
+
+ // The fake provider's process-local counter restarts at provider-turn-1.
+ // Durable state must still remember that older identity, not only the most
+ // recent provider-turn-2 completion authority.
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let error = recovered
.execute(&command(
- "turn",
- 3,
+ "turn-reused",
+ 5,
"turn.start",
- json!({"text": "Keep the native turn active while the provider exits."}),
+ json!({"text": "Do not accept an older provider turn identity."}),
))
- .expect("start provider turn");
+ .expect_err("reject a provider turn identity retained before restart");
+ assert!(error.to_string().contains("reused a settled"));
- let mut provider_exit_seen = false;
- for _ in 0..32 {
- provider_exit_seen |= poll_and_ack(&mut executor)
- .expect("poll provider exit")
- .iter()
- .any(|event| event.event_type == "session.failed");
- if provider_exit_seen {
- break;
- }
- }
- assert!(provider_exit_seen);
let persisted: Value = serde_json::from_slice(
&fs::read(directory.join("codex-provider-state.json"))
- .expect("read provider state after exit"),
+ .expect("read fail-closed provider state"),
)
- .expect("parse provider state after exit");
- assert_eq!(persisted["lifecycle"], "provider_exited");
- assert_eq!(persisted["activeProviderTurnId"], "provider-turn-1");
+ .expect("parse fail-closed provider state");
+ assert_eq!(persisted["lifecycle"], "closed");
+ assert!(persisted["activeProviderTurnId"].is_null());
- let interrupted = executor
- .execute(&command("interrupt", 4, "turn.interrupt", json!({})))
- .expect("interrupt reconciled provider turn");
- assert_eq!(interrupted.result["status"], "interrupt_requested");
- assert_eq!(call_count(&directory, "thread/resume"), 1);
- assert_eq!(call_count(&directory, "thread/read"), 1);
- assert_eq!(call_count(&directory, "turn/interrupt"), 1);
+ drop(recovered);
+ let resumes_before_closed_restore = call_count(&directory, "thread/resume");
+ let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let events = closed
+ .poll_events()
+ .expect("closed reused-identity state remains readable without resuming Codex");
+ assert!(events.iter().any(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "provider_turn_identity_reused"
+ }));
+ assert_eq!(
+ call_count(&directory, "thread/resume"),
+ resumes_before_closed_restore,
+ "recovery must not resume provider work accepted under a reused identity"
+ );
- let mut terminal_seen = false;
- for _ in 0..32 {
- terminal_seen |= poll_and_ack(&mut executor)
- .expect("poll reconciled interruption")
- .iter()
- .any(|event| event.event_type == "turn.interrupted");
- if terminal_seen {
- break;
- }
- }
- assert!(terminal_seen);
- executor.shutdown().expect("stop resumed provider process");
+ closed.shutdown().expect("close fail-closed executor");
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
#[test]
-fn unacknowledged_provider_events_survive_executor_restart() {
- let directory = temporary_directory("pending-event-recovery");
- let config = provider_config(&directory, &["--emit-question"]);
- let mut first = CodexCommandExecutor::new(&directory);
+fn durable_recovery_closes_a_provider_that_reopens_a_settled_turn() {
+ let directory = temporary_directory("durable-settled-turn-active-on-recovery");
+ let config = provider_config(&directory, &[]);
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
first
.execute(&command(
"prepare",
@@ -1636,23 +2219,1685 @@ fn unacknowledged_provider_events_survive_executor_restart() {
"turn",
3,
"turn.start",
- json!({"text": "Emit a durable question."}),
+ json!({"text": "Complete the durable turn."}),
))
.expect("start provider turn");
-
- let mut retained = None;
+ let mut completed = false;
for _ in 0..32 {
- let events = first.poll_events().expect("poll provider events");
- if events
+ completed |= poll_and_ack(&mut first)
+ .expect("poll completed provider turn")
.iter()
- .any(|event| event.event_type == "runtime_request.created")
- {
- retained = Some(events);
+ .any(|event| event.event_type == "turn.completed");
+ if completed {
break;
}
- first
- .acknowledge_events(events.len())
- .expect("acknowledge events before the question");
+ }
+ assert!(completed);
+ first.shutdown().expect("stop first provider process");
+ drop(first);
+
+ // Contradict the durable terminal ledger with a resumed provider snapshot
+ // that reports the exact settled identity as active again.
+ fs::write(
+ directory.join("fake-state.json"),
+ serde_json::to_vec_pretty(&json!({
+ "threadId": "codex-thread-1",
+ "activeTurnId": "provider-turn-1",
+ }))
+ .unwrap(),
+ )
+ .expect("write contradictory fake provider state");
+
+ let resumes_before_recovery = call_count(&directory, "thread/resume");
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let events = recovered
+ .poll_events()
+ .expect("fail-closed recovery remains observable");
+ assert!(events.iter().any(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "provider_turn_identity_reused"
+ && event.payload["providerTurnId"] == "provider-turn-1"
+ }));
+ assert_eq!(
+ call_count(&directory, "thread/resume"),
+ resumes_before_recovery + 1,
+ );
+ let persisted: Value = serde_json::from_slice(
+ &fs::read(directory.join("codex-provider-state.json"))
+ .expect("read fail-closed provider state"),
+ )
+ .expect("parse fail-closed provider state");
+ assert_eq!(persisted["lifecycle"], "closed");
+ assert!(persisted["activeProviderTurnId"].is_null());
+ assert_eq!(persisted["completedTurnAuthoritative"], false);
+
+ drop(recovered);
+ let resumes_before_closed_restore = call_count(&directory, "thread/resume");
+ let mut closed = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ closed
+ .poll_events()
+ .expect("closed recovery state remains readable");
+ assert_eq!(
+ call_count(&directory, "thread/resume"),
+ resumes_before_closed_restore,
+ "closed recovery must not resume the contradictory provider turn again",
+ );
+
+ closed.shutdown().expect("close fail-closed executor");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn codex_rejects_a_two_turn_old_call_while_idle() {
+ let directory = temporary_directory("delayed-tool-after-second-turn-completion");
+ let config = provider_config(&directory, &["--delayed-tool-after-second-turn-completion"]);
+ let mut provider = CodexProvider::start_with_tools(&config, [task_context_tool()], None)
+ .expect("start Codex with an authorized tool");
+
+ for message in ["Complete turn one.", "Complete turn two."] {
+ provider
+ .start_turn(message, &config.cwd)
+ .expect("start completed provider turn");
+ wait_for_notification(&mut provider, "turn/completed");
+ }
+ assert_eq!(provider.active_provider_turn_id(), None);
+
+ let warning = wait_for_notification(&mut provider, "warning");
+ assert_eq!(warning["providerMethod"], "item/tool/call");
+ assert_eq!(provider.active_provider_turn_id(), None);
+
+ let _ = provider.shutdown();
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_delivers_a_terminal_before_rejecting_a_delayed_tool_call() {
+ let directory = temporary_directory("durable-delayed-tool-after-terminal");
+ let config = provider_config(&directory, &["--delayed-tool-after-failed-turn"]);
+ let runner_config = durable_config(&directory);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ "completionContract": {
+ "revision": "sha256:delayed-tool-contract",
+ "criterionIds": ["criterion_delayed_tool"]
+ },
+ }),
+ ))
+ .unwrap();
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .unwrap();
+ executor
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Fail before invoking a delayed tool."}),
+ ))
+ .unwrap();
+
+ let mut observed = Vec::new();
+ for _ in 0..32 {
+ observed.extend(poll_and_ack(&mut executor).expect("poll durable provider events"));
+ if observed
+ .iter()
+ .any(|event| event.event_type == "provider.notice.recorded")
+ {
+ break;
+ }
+ }
+ let terminal = observed
+ .iter()
+ .position(|event| event.event_type == "run.terminal")
+ .expect("the durable terminal is delivered");
+ let rejection = observed
+ .iter()
+ .position(|event| event.event_type == "provider.notice.recorded")
+ .expect("the delayed request is rejected non-fatally");
+ assert!(terminal < rejection);
+
+ executor.shutdown().unwrap();
+ fs::remove_dir_all(directory).unwrap();
+}
+
+#[test]
+fn codex_rejects_a_runtime_response_after_its_turn_terminates() {
+ let directory = temporary_directory("delayed-question-response");
+ let config = provider_config(&directory, &["--question-before-failed-turn"]);
+ let mut provider = CodexProvider::start(&config, None).expect("start Codex provider");
+ provider
+ .start_turn("Ask and then fail.", &config.cwd)
+ .expect("start provider turn");
+
+ let mut request_id = None;
+ let mut terminal_seen = false;
+ for _ in 0..16 {
+ match provider.poll().expect("poll question and terminal") {
+ Some(CodexProviderEvent::RuntimeRequest {
+ request_id: observed,
+ ..
+ }) => request_id = Some(observed),
+ Some(CodexProviderEvent::Notification { method, .. }) if method == "turn/completed" => {
+ terminal_seen = true;
+ }
+ _ => {}
+ }
+ if request_id.is_some() && terminal_seen {
+ break;
+ }
+ }
+ let request_id = request_id.expect("observe the runtime request before termination");
+ assert!(terminal_seen);
+ let error = provider
+ .resolve_runtime_request(
+ &request_id,
+ &json!({
+ "schema": "paperclip.question_response.v1",
+ "answers": {"environment": {"selectedOptionIds": ["option-1"]}}
+ }),
+ )
+ .expect_err("terminal requests must not remain resolvable");
+ assert!(error.to_string().contains("no pending Codex request"));
+
+ let _ = provider.shutdown();
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn reused_provider_question_ids_get_unique_controller_identities() {
+ let directory = temporary_directory("reused-question-id");
+ let config = provider_config(&directory, &["--emit-question", "--reuse-question-id"]);
+ let mut provider = CodexProvider::start(&config, None).expect("start Codex provider");
+ provider
+ .start_turn("Ask twice with one provider request id.", &config.cwd)
+ .expect("start provider turn");
+
+ let first_request_id = (0..16)
+ .find_map(|_| match provider.poll().expect("poll first question") {
+ Some(CodexProviderEvent::RuntimeRequest { request_id, .. }) => Some(request_id),
+ _ => None,
+ })
+ .expect("observe first runtime request");
+ let response = json!({
+ "schema": "paperclip.question_response.v1",
+ "answers": {"environment": {"selectedOptionIds": ["option-1"]}}
+ });
+ provider
+ .resolve_runtime_request(&first_request_id, &response)
+ .expect("resolve first runtime request");
+ let second_request_id = (0..16)
+ .find_map(|_| match provider.poll().expect("poll second question") {
+ Some(CodexProviderEvent::RuntimeRequest { request_id, .. }) => Some(request_id),
+ _ => None,
+ })
+ .expect("observe second runtime request");
+
+ assert_ne!(first_request_id, second_request_id);
+ let stale = provider
+ .resolve_runtime_request(&first_request_id, &response)
+ .expect_err("the first controller identity cannot resolve the second question");
+ assert!(stale.to_string().contains("no pending Codex request"));
+ provider
+ .resolve_runtime_request(&second_request_id, &response)
+ .expect("resolve second runtime request");
+
+ let _ = provider.shutdown();
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn codex_resume_advertises_the_same_authorized_tools() {
+ let directory = temporary_directory("dynamic-tool-resume");
+ let config = provider_config(&directory, &["--require-dynamic-tool"]);
+ let mut provider =
+ CodexProvider::start_with_tools(&config, [task_context_tool()], Some("codex-thread-1"))
+ .expect("resume Codex with the run-scoped tool set");
+ assert_eq!(provider.thread_id(), "codex-thread-1");
+ provider.shutdown().expect("stop provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_routes_a_semantic_tool_result_back_to_codex() {
+ let directory = temporary_directory("durable-dynamic-tool");
+ let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]);
+ let runner_config = durable_config(&directory);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare the durable Codex tool set");
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open the Codex session");
+ executor
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Inspect the durable fake task."}),
+ ))
+ .expect("start the Codex turn");
+
+ let mut semantic_input = None;
+ for _ in 0..32 {
+ let events = poll_and_ack(&mut executor).expect("poll semantic input");
+ semantic_input = events
+ .iter()
+ .find(|event| event.event_type == "semantic_tool.input")
+ .cloned()
+ .or(semantic_input);
+ if semantic_input.is_some() {
+ break;
+ }
+ }
+ let semantic_input = semantic_input.expect("durable semantic input is emitted");
+ assert_eq!(
+ semantic_input.payload["semantic_tool"]["correlation"]["runId"],
+ "run-1"
+ );
+ assert_eq!(
+ semantic_input.payload["semantic_tool"]["operationId"],
+ "get_task_context"
+ );
+
+ let delivered = executor
+ .execute(&command(
+ "tool-result",
+ 4,
+ "semantic_tool.result",
+ json!({
+ "callId": "semantic-call-1",
+ "operationId": "get_task_context",
+ "result": {"ok": true, "task": {"id": "task-1"}},
+ "isError": false,
+ }),
+ ))
+ .expect("deliver the durable semantic result");
+ assert_eq!(delivered.result["status"], "delivered");
+
+ let mut result_seen = false;
+ let mut terminal_seen = false;
+ for _ in 0..32 {
+ let events = poll_and_ack(&mut executor).expect("poll result and completion");
+ result_seen |= events
+ .iter()
+ .any(|event| event.event_type == "semantic_tool.result");
+ terminal_seen |= events
+ .iter()
+ .any(|event| event.event_type == "turn.completed");
+ if result_seen && terminal_seen {
+ break;
+ }
+ }
+ assert!(result_seen);
+ assert!(terminal_seen);
+ executor.shutdown().expect("stop provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_replays_completed_results_without_mutating_the_event_queue() {
+ let directory = temporary_directory("durable-completed-tool-replay");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--emit-tool-call",
+ "--hold-turn",
+ "--replay-completed-tool-call-count",
+ "4",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare the durable Codex tool set");
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open the Codex session");
+ executor
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Replay the completed fake tool call."}),
+ ))
+ .expect("start the Codex turn");
+
+ let mut input_seen = false;
+ for _ in 0..32 {
+ input_seen |= poll_and_ack(&mut executor)
+ .expect("poll semantic input")
+ .iter()
+ .any(|event| event.event_type == "semantic_tool.input");
+ if input_seen {
+ break;
+ }
+ }
+ assert!(input_seen, "durable semantic input is emitted");
+
+ executor
+ .execute(&command(
+ "tool-result",
+ 4,
+ "semantic_tool.result",
+ json!({
+ "callId": "semantic-call-1",
+ "operationId": "get_task_context",
+ "result": {"ok": true, "task": {"id": "task-1"}},
+ "isError": false,
+ }),
+ ))
+ .expect("deliver the original durable semantic result");
+ let result_events = poll_and_ack(&mut executor).expect("acknowledge semantic result");
+ assert_eq!(
+ result_events
+ .iter()
+ .filter(|event| event.event_type == "semantic_tool.result")
+ .count(),
+ 1
+ );
+
+ let state_path = directory.join("codex-provider-state.json");
+ let state_before_replays = fs::read(&state_path).expect("read state before exact replays");
+ let expected_response = r#"{"ok":true,"task":{"id":"task-1"}}"#;
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ while recorded_tool_responses(&directory).len() < 5 && std::time::Instant::now() < deadline {
+ let events = poll_and_ack(&mut executor).expect("service completed tool replay");
+ assert!(
+ events.iter().all(|event| {
+ event.event_type != "semantic_tool.input"
+ && event.event_type != "semantic_tool.reconciled"
+ && event.event_type != "semantic_tool.result"
+ }),
+ "an exact completed replay must not add semantic reconciliation events"
+ );
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+
+ let responses = recorded_tool_responses(&directory);
+ assert_eq!(
+ responses.len(),
+ 5,
+ "the original result and four replays return"
+ );
+ assert!(responses
+ .iter()
+ .all(|response| response == expected_response));
+ assert_eq!(
+ fs::read(&state_path).expect("read state after exact replays"),
+ state_before_replays,
+ "exact completed replays must not consume durable event capacity"
+ );
+
+ executor.shutdown().expect("stop provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_replays_pending_tool_calls_without_mutating_the_event_queue() {
+ let directory = temporary_directory("durable-tool-recovery");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--emit-tool-call",
+ "--emit-tool-call-on-resume",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare the recoverable tool bridge");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open the first provider");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Hold the tool call across recovery."}),
+ ))
+ .expect("start the first turn");
+ let mut input_seen = false;
+ for _ in 0..32 {
+ input_seen |= poll_and_ack(&mut first)
+ .expect("poll first tool input")
+ .iter()
+ .any(|event| event.event_type == "semantic_tool.input");
+ if input_seen {
+ break;
+ }
+ }
+ assert!(input_seen);
+ drop(first);
+
+ let state_path = directory.join("codex-provider-state.json");
+ let mut recovered = None;
+ for replay in 0..4 {
+ let before: Value = serde_json::from_slice(
+ &fs::read(&state_path).expect("read state before provider recovery"),
+ )
+ .expect("parse state before provider recovery");
+ let resume_count = call_count(&directory, "thread/resume");
+ let mut next = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ for _ in 0..32 {
+ let events = poll_and_ack(&mut next).expect("poll exact pending replay");
+ assert!(events.iter().all(|event| {
+ event.event_type != "semantic_tool.input"
+ && event.event_type != "semantic_tool.reconciled"
+ && event.event_type != "semantic_tool.result"
+ }));
+ if call_count(&directory, "thread/resume") > resume_count {
+ break;
+ }
+ }
+ assert!(
+ call_count(&directory, "thread/resume") > resume_count,
+ "recovery {replay} resumes the active provider turn"
+ );
+ for _ in 0..4 {
+ let events = poll_and_ack(&mut next).expect("poll exact pending replay");
+ assert!(events.iter().all(|event| {
+ event.event_type != "semantic_tool.input"
+ && event.event_type != "semantic_tool.reconciled"
+ && event.event_type != "semantic_tool.result"
+ }));
+ }
+ let after: Value = serde_json::from_slice(
+ &fs::read(&state_path).expect("read state after pending replay"),
+ )
+ .expect("parse state after pending replay");
+ assert_eq!(
+ after["pendingEvents"], before["pendingEvents"],
+ "exact pending replay {replay} must not append pending events"
+ );
+ assert_eq!(
+ after["queuedEvents"], before["queuedEvents"],
+ "exact pending replay {replay} must not append queued events"
+ );
+ assert_eq!(
+ after["nextProviderEventSeq"], before["nextProviderEventSeq"],
+ "exact pending replay {replay} must not consume durable event capacity"
+ );
+ recovered = Some(next);
+ if replay < 3 {
+ drop(recovered.take());
+ }
+ }
+ let mut recovered = recovered.expect("retain the final recovered provider");
+ recovered
+ .execute(&command(
+ "tool-result",
+ 4,
+ "semantic_tool.result",
+ json!({
+ "callId": "semantic-call-1",
+ "operationId": "get_task_context",
+ "result": {"ok": true, "task": {"id": "task-1"}},
+ "isError": false,
+ }),
+ ))
+ .expect("complete the replayed tool call");
+ let events = poll_and_ack(&mut recovered).expect("poll the completed pending call");
+ assert_eq!(
+ events
+ .iter()
+ .filter(|event| event.event_type == "semantic_tool.result")
+ .count(),
+ 1
+ );
+ recovered.shutdown().expect("stop recovered provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_settles_pending_tools_when_recovery_finds_the_turn_ended() {
+ let directory = temporary_directory("durable-tool-ended-offline");
+ let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]);
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ "completionContract": {
+ "revision": "sha256:offline-recovery-contract",
+ "criterionIds": ["criterion_offline_recovery"]
+ },
+ }),
+ ))
+ .expect("prepare the recoverable tool bridge");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open the first provider");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "End while the runner is offline."}),
+ ))
+ .expect("start the first turn");
+ let mut input_seen = false;
+ for _ in 0..32 {
+ input_seen |= poll_and_ack(&mut first)
+ .expect("poll first tool input")
+ .iter()
+ .any(|event| event.event_type == "semantic_tool.input");
+ if input_seen {
+ break;
+ }
+ }
+ assert!(input_seen);
+ drop(first);
+
+ fs::write(
+ directory.join("fake-state.json"),
+ serde_json::to_vec_pretty(&json!({
+ "threadId": "codex-thread-1",
+ "activeTurnId": null,
+ }))
+ .unwrap(),
+ )
+ .expect("record that the provider turn ended while offline");
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ recovered
+ .execute(&command("snapshot", 4, "session.snapshot", json!({})))
+ .expect("restore the provider session");
+ let mut observed = Vec::new();
+ for _ in 0..32 {
+ observed.extend(
+ poll_and_ack(&mut recovered)
+ .expect("poll recovered settlement")
+ .into_iter()
+ .map(|event| event.event_type),
+ );
+ if observed.iter().any(|event| event == "run.terminal") {
+ break;
+ }
+ }
+ let semantic_result = observed
+ .iter()
+ .position(|event| event == "semantic_tool.result")
+ .expect("recovery settles the pending semantic tool");
+ let reconciled = observed
+ .iter()
+ .position(|event| event == "session.reconciled")
+ .expect("recovery emits a reconciliation event");
+ let terminal = observed
+ .iter()
+ .position(|event| event == "run.terminal")
+ .expect("offline turn recovery terminates the run");
+ assert!(semantic_result < reconciled);
+ assert!(reconciled < terminal);
+ assert!(recovered
+ .execute(&command(
+ "late-result",
+ 5,
+ "semantic_tool.result",
+ json!({
+ "callId": "semantic-call-1",
+ "operationId": "get_task_context",
+ "result": {"ok": true},
+ "isError": false,
+ }),
+ ))
+ .is_err());
+
+ recovered.shutdown().expect("stop recovered provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_rejects_tool_catalog_drift_during_attach() {
+ let directory = temporary_directory("durable-tool-attach-drift");
+ let config = provider_config(&directory, &[]);
+ let runner_config = durable_config(&directory);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare the durable tool catalog");
+
+ let mut changed = task_context_tool_set();
+ changed.operations[0].description = "Changed after recovery.".to_owned();
+ changed.catalog_digest = authorized_tool_catalog_digest(&changed.operations).unwrap();
+ let error = executor
+ .execute(&command(
+ "attach",
+ 2,
+ "run.attach",
+ json!({"authorizedTools": changed}),
+ ))
+ .expect_err("attach must reject tool catalog drift");
+ assert!(error.to_string().contains("tool contract changed"));
+
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn durable_backend_settles_tools_before_a_natural_terminal_event() {
+ let directory = temporary_directory("durable-tool-terminal");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--emit-tool-call",
+ "--finish-turn-with-pending-tool",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .unwrap();
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .unwrap();
+ executor
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Terminate with a pending tool."}),
+ ))
+ .unwrap();
+
+ let mut observed = Vec::new();
+ for _ in 0..32 {
+ let events = poll_and_ack(&mut executor).unwrap();
+ observed.extend(events.into_iter().map(|event| event.event_type));
+ if observed.iter().any(|event| event == "turn.completed") {
+ break;
+ }
+ }
+ let semantic_result = observed
+ .iter()
+ .position(|event| event == "semantic_tool.result")
+ .expect("terminal settlement emits a failed semantic result");
+ let terminal = observed
+ .iter()
+ .position(|event| event == "turn.completed")
+ .expect("provider terminal event is emitted");
+ assert!(semantic_result < terminal);
+ assert!(executor
+ .execute(&command(
+ "late-result",
+ 4,
+ "semantic_tool.result",
+ json!({
+ "callId": "semantic-call-1",
+ "operationId": "get_task_context",
+ "result": {"ok": true},
+ "isError": false,
+ }),
+ ))
+ .is_err());
+
+ executor.shutdown().unwrap();
+ fs::remove_dir_all(directory).unwrap();
+}
+
+#[test]
+fn durable_backend_resumes_the_active_thread_without_restarting_the_turn() {
+ let directory = temporary_directory("resume");
+ let config = provider_config(&directory, &["--hold-turn"]);
+ let mut first = CodexCommandExecutor::new(&directory);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Hold this turn for recovery."}),
+ ))
+ .expect("start held provider turn");
+ assert_eq!(call_count(&directory, "turn/start"), 1);
+ first.shutdown().expect("stop first provider process");
+ drop(first);
+
+ let mut recovered = CodexCommandExecutor::new(&directory);
+ let snapshot = recovered
+ .execute(&command("snapshot", 4, "session.snapshot", json!({})))
+ .expect("restore provider session");
+ assert_eq!(snapshot.result["status"], "turn_active");
+ assert_eq!(snapshot.result["activeProviderTurnId"], "provider-turn-1");
+ assert_eq!(call_count(&directory, "turn/start"), 1);
+ assert_eq!(call_count(&directory, "thread/resume"), 1);
+ assert_eq!(call_count(&directory, "thread/read"), 1);
+
+ recovered
+ .execute(&command("interrupt", 5, "turn.interrupt", json!({})))
+ .expect("interrupt recovered provider turn");
+ let mut terminal_seen = false;
+ for _ in 0..16 {
+ let events = poll_and_ack(&mut recovered).expect("poll interrupted turn");
+ terminal_seen |= events
+ .iter()
+ .any(|event| event.event_type == "turn.interrupted");
+ if terminal_seen {
+ break;
+ }
+ }
+ assert!(terminal_seen);
+ recovered
+ .shutdown()
+ .expect("stop recovered provider process");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn legacy_full_active_epoch_is_closed_on_recovery() {
+ let directory = temporary_directory("legacy-full-active-recovery");
+ let config = provider_config(&directory, &["--hold-turn"]);
+ let mut first = CodexCommandExecutor::new(&directory);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Hold the legacy full-epoch turn."}),
+ ))
+ .expect("start held provider turn");
+ first.shutdown().expect("stop first provider process");
+ drop(first);
+
+ let state_path = directory.join("codex-provider-state.json");
+ let mut persisted: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read active provider state"))
+ .expect("parse active provider state");
+ let prior_generation = persisted["providerProcessGeneration"]
+ .as_u64()
+ .expect("provider generation is persisted");
+ persisted["settledProviderTurnIds"] = Value::Array(
+ (0..4_096)
+ .map(|index| Value::String(format!("legacy-provider-turn-{index}")))
+ .collect(),
+ );
+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())
+ .expect("write legacy full active state");
+
+ let mut recovered = CodexCommandExecutor::new(&directory);
+ let events = recovered
+ .poll_events()
+ .expect("legacy full-epoch recovery remains observable");
+ assert!(events.iter().any(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "legacy_provider_turn_epoch_ambiguous"
+ }));
+ let closed: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read closed legacy full state"))
+ .expect("parse closed legacy full state");
+ assert_eq!(closed["lifecycle"], "closed");
+ assert!(closed["activeProviderTurnId"].is_null());
+ assert_eq!(closed["ambiguousTurnStartPending"], false);
+ assert_eq!(closed["completedTurnAuthoritative"], false);
+ assert!(closed["settledProviderTurnFilter"]["words"]
+ .as_array()
+ .expect("legacy filter shape remains valid")
+ .is_empty());
+ assert_eq!(
+ closed["settledProviderTurnIds"].as_array().unwrap().len(),
+ 4_096
+ );
+ assert!(closed["providerProcessGeneration"].as_u64().unwrap() > prior_generation);
+ assert!(recovered
+ .execute(&command(
+ "replacement-turn",
+ 4,
+ "turn.start",
+ json!({"text": "Do not reopen closed legacy work."}),
+ ))
+ .expect_err("closed legacy full-epoch state rejects replacement work")
+ .to_string()
+ .contains("closed"));
+ assert_eq!(call_count(&directory, "turn/start"), 1);
+
+ recovered.shutdown().expect("close recovered executor");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn legacy_filtered_ambiguous_epoch_is_closed_on_recovery() {
+ let directory = temporary_directory("legacy-filtered-ambiguous-recovery");
+ let config = provider_config(&directory, &[]);
+ let mut first = CodexCommandExecutor::new(&directory);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first.shutdown().expect("stop first provider process");
+ drop(first);
+
+ let state_path = directory.join("codex-provider-state.json");
+ let mut persisted: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read idle provider state"))
+ .expect("parse idle provider state");
+ let mut legacy_words = vec![0_u64; 32_768];
+ legacy_words[0] = 1;
+ persisted["settledProviderTurnFilter"] = json!({"words": legacy_words});
+ persisted["ambiguousTurnStartPending"] = Value::Bool(true);
+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())
+ .expect("write legacy-filtered ambiguous state");
+
+ let mut recovered = CodexCommandExecutor::new(&directory);
+ let events = recovered
+ .poll_events()
+ .expect("legacy ambiguous recovery remains observable");
+ let diagnostic = events
+ .iter()
+ .find(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "legacy_provider_turn_epoch_ambiguous"
+ })
+ .expect("legacy ambiguous recovery emits a terminal diagnostic");
+ assert_eq!(diagnostic.payload["ambiguousStartPending"], true);
+ assert_eq!(diagnostic.payload["providerReportedActive"], false);
+ let closed: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read closed legacy ambiguous state"))
+ .expect("parse closed legacy ambiguous state");
+ assert_eq!(closed["lifecycle"], "closed");
+ assert!(closed["activeProviderTurnId"].is_null());
+ assert_eq!(closed["ambiguousTurnStartPending"], false);
+ assert!(!closed["settledProviderTurnFilter"]["words"]
+ .as_array()
+ .expect("legacy filter remains durable evidence")
+ .is_empty());
+ assert_eq!(call_count(&directory, "turn/start"), 0);
+
+ recovered.shutdown().expect("close recovered executor");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn legacy_full_ended_epoch_reconciles_before_idle_rollover() {
+ let directory = temporary_directory("legacy-full-ended-recovery");
+ let config = provider_config(&directory, &["--hold-turn"]);
+ let mut first = CodexCommandExecutor::new(&directory);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Let this legacy turn end while runnerd is away."}),
+ ))
+ .expect("start held provider turn");
+ first.shutdown().expect("stop first provider process");
+ drop(first);
+
+ let state_path = directory.join("codex-provider-state.json");
+ let mut persisted: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read active provider state"))
+ .expect("parse active provider state");
+ persisted["settledProviderTurnIds"] = Value::Array(
+ (0..4_096)
+ .map(|index| Value::String(format!("legacy-provider-turn-{index}")))
+ .collect(),
+ );
+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())
+ .expect("write full legacy provider epoch");
+ fs::write(
+ directory.join("fake-state.json"),
+ serde_json::to_vec_pretty(&json!({"threadId": "codex-thread-1"})).unwrap(),
+ )
+ .expect("mark the provider turn ended while runnerd was away");
+
+ let mut recovered = CodexCommandExecutor::new(&directory);
+ let events = recovered
+ .poll_events()
+ .expect("reconcile an ended turn from a full legacy epoch");
+ assert!(events.iter().any(|event| event.event_type == "turn.failed"));
+ let reconciled: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read reconciled legacy state"))
+ .expect("parse reconciled legacy state");
+ assert_eq!(reconciled["lifecycle"], "session_open");
+ assert!(reconciled["activeProviderTurnId"].is_null());
+ assert_eq!(
+ reconciled["settledProviderTurnIds"]
+ .as_array()
+ .unwrap()
+ .len(),
+ 4_096
+ );
+
+ recovered
+ .execute(&command(
+ "replacement-turn",
+ 4,
+ "turn.start",
+ json!({"text": "Start only after the idle epoch rolls over."}),
+ ))
+ .expect("roll over the idle full epoch and start replacement work");
+ let rolled: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read rolled legacy state"))
+ .expect("parse rolled legacy state");
+ assert!(rolled["settledProviderTurnIds"]
+ .as_array()
+ .expect("rolled exact identities remain an array")
+ .is_empty());
+ assert!(rolled["settledProviderTurnFilter"]["words"]
+ .as_array()
+ .expect("rolled filter remains valid")
+ .is_empty());
+ assert_eq!(call_count(&directory, "turn/start"), 2);
+
+ recovered.shutdown().expect("stop rolled provider process");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn codex_resume_rejects_an_oversized_active_turn_identity() {
+ let directory = temporary_directory("resume-oversized-active-turn-id");
+ let config = provider_config(&directory, &[]);
+ fs::write(
+ directory.join("fake-state.json"),
+ serde_json::to_vec_pretty(&json!({
+ "threadId": "codex-thread-1",
+ "activeTurnId": "x".repeat(241),
+ }))
+ .unwrap(),
+ )
+ .expect("write fake provider state with an oversized active identity");
+
+ let error = CodexProvider::start(&config, Some("codex-thread-1"))
+ .err()
+ .expect("reject an oversized recovered provider turn identity");
+ assert!(error.to_string().contains("invalid turn.id"));
+
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn provider_exit_preserves_and_reconciles_the_active_turn() {
+ let directory = temporary_directory("exit-active-turn");
+ let config = provider_config(&directory, &["--exit-after-turn-start"]);
+ let mut executor = CodexCommandExecutor::new(&directory);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare Codex provider");
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ executor
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Keep the native turn active while the provider exits."}),
+ ))
+ .expect("start provider turn");
+
+ let mut provider_exit_seen = false;
+ for _ in 0..32 {
+ provider_exit_seen |= poll_and_ack(&mut executor)
+ .expect("poll provider exit")
+ .iter()
+ .any(|event| event.event_type == "session.failed");
+ if provider_exit_seen {
+ break;
+ }
+ }
+ assert!(provider_exit_seen);
+ let persisted: Value = serde_json::from_slice(
+ &fs::read(directory.join("codex-provider-state.json"))
+ .expect("read provider state after exit"),
+ )
+ .expect("parse provider state after exit");
+ assert_eq!(persisted["lifecycle"], "provider_exited");
+ assert_eq!(persisted["activeProviderTurnId"], "provider-turn-1");
+
+ let interrupted = executor
+ .execute(&command("interrupt", 4, "turn.interrupt", json!({})))
+ .expect("interrupt reconciled provider turn");
+ assert_eq!(interrupted.result["status"], "interrupt_requested");
+ assert_eq!(call_count(&directory, "thread/resume"), 1);
+ assert_eq!(call_count(&directory, "thread/read"), 1);
+ assert_eq!(call_count(&directory, "turn/interrupt"), 1);
+
+ let mut terminal_seen = false;
+ for _ in 0..32 {
+ terminal_seen |= poll_and_ack(&mut executor)
+ .expect("poll reconciled interruption")
+ .iter()
+ .any(|event| event.event_type == "turn.interrupted");
+ if terminal_seen {
+ break;
+ }
+ }
+ assert!(terminal_seen);
+ executor.shutdown().expect("stop resumed provider process");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn receipt_limit_rejects_the_call_and_keeps_polling_when_interrupt_fails() {
+ let directory = temporary_directory("receipt-limit-interrupt-failure");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--hold-turn",
+ "--emit-tool-call-on-resume",
+ "--fail-first-interrupt",
+ "--accept-interrupt-without-terminal-once",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Hold this saturated turn for recovery."}),
+ ))
+ .expect("start held provider turn");
+ drop(first);
+
+ saturate_provider_tool_receipts(&directory);
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let events = poll_and_ack(&mut recovered)
+ .expect("a failed first interrupt must not terminate durable provider polling");
+ assert!(events.iter().any(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "semantic_tool_turn_receipt_limit"
+ }));
+ assert_eq!(call_count(&directory, "tool-response:failure"), 1);
+ assert_eq!(call_count(&directory, "turn/interrupt"), 1);
+
+ let mut settled = Vec::new();
+ for _ in 0..4 {
+ settled.extend(
+ poll_and_ack(&mut recovered)
+ .expect("polling must autonomously retry the durable receipt-limit interrupt"),
+ );
+ if settled
+ .iter()
+ .any(|event| event.event_type == "turn.interrupted")
+ {
+ break;
+ }
+ }
+ assert!(
+ settled
+ .iter()
+ .any(|event| event.event_type == "turn.interrupted"),
+ "the retry must settle the receipt-exhausted turn"
+ );
+ assert_eq!(call_count(&directory, "turn/interrupt"), 3);
+
+ recovered.shutdown().expect("stop recovered provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn receipt_limit_retry_preserves_a_turn_settled_during_provider_recovery() {
+ let directory = temporary_directory("receipt-limit-recovered-settlement");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--hold-turn",
+ "--emit-tool-call-on-resume",
+ "--fail-first-interrupt",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Recover a provider-settled receipt-limited turn."}),
+ ))
+ .expect("start held provider turn");
+ drop(first);
+
+ saturate_provider_tool_receipts(&directory);
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let events = poll_and_ack(&mut recovered)
+ .expect("the first failed interruption must leave a durable retry");
+ assert!(events.iter().any(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "semantic_tool_turn_receipt_limit"
+ }));
+ assert_eq!(call_count(&directory, "turn/interrupt"), 1);
+
+ // Lose the live transport while retaining its durable interruption retry,
+ // then model the provider settling the turn before runnerd reconnects.
+ recovered
+ .shutdown()
+ .expect("stop the provider before retry recovery");
+ let fake_state_path = directory.join("fake-state.json");
+ let mut fake_state: Value =
+ serde_json::from_slice(&fs::read(&fake_state_path).expect("read fake provider state"))
+ .expect("parse fake provider state");
+ fake_state["activeTurnId"] = Value::Null;
+ fs::write(
+ &fake_state_path,
+ serde_json::to_vec_pretty(&fake_state).unwrap(),
+ )
+ .expect("settle the fake provider turn before recovery");
+
+ let events = poll_and_ack(&mut recovered)
+ .expect("an already-settled retry must preserve the recovered terminal state");
+ assert!(events.iter().any(|event| {
+ event.event_type == "session.reconciled" && event.payload["activeProviderTurnId"].is_null()
+ }));
+ assert_eq!(
+ call_count(&directory, "turn/interrupt"),
+ 1,
+ "an already-settled recovered turn must not receive another interrupt RPC"
+ );
+
+ let persisted: Value = serde_json::from_slice(
+ &fs::read(directory.join("codex-provider-state.json"))
+ .expect("read reconciled provider state"),
+ )
+ .expect("parse reconciled provider state");
+ assert!(persisted["activeProviderTurnId"].is_null());
+ assert_eq!(persisted["receiptLimitInterruptPending"], false);
+ assert_eq!(persisted["receiptLimitInterruptAccepted"], false);
+ assert_eq!(persisted["receiptLimitInterruptAttempts"], 0);
+ assert!(persisted["receiptLimitInterruptDeadlineUnixMs"].is_null());
+
+ recovered.shutdown().expect("stop recovered provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn receipt_limit_accepts_a_terminal_after_the_initial_interrupt_deadline() {
+ let directory = temporary_directory("receipt-limit-delayed-terminal");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--hold-turn",
+ "--emit-tool-call-on-resume",
+ "--interrupt-terminal-delay-ms",
+ "2100",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Accept a delayed authoritative interruption."}),
+ ))
+ .expect("start held provider turn");
+ drop(first);
+ saturate_provider_tool_receipts(&directory);
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let mut emitted = Vec::new();
+ for _ in 0..4_096 {
+ emitted.extend(
+ poll_and_ack(&mut recovered)
+ .expect("a delayed accepted interrupt must remain durably pollable"),
+ );
+ if emitted
+ .iter()
+ .any(|event| event.event_type == "turn.interrupted")
+ {
+ break;
+ }
+ }
+ assert!(
+ emitted
+ .iter()
+ .any(|event| event.event_type == "turn.interrupted"),
+ "the delayed provider terminal must remain authoritative after two seconds"
+ );
+ assert!(!emitted.iter().any(|event| {
+ event.event_type == "turn.interrupted"
+ && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_deadline"
+ }));
+ assert!(
+ !emitted.iter().any(|event| {
+ event.event_type == "turn.failed"
+ && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_unconfirmed"
+ }),
+ "fast provider polls must not replace a delayed terminal with fallback failure"
+ );
+
+ recovered.shutdown().expect("stop recovered provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn receipt_limit_polls_an_authoritative_terminal_with_unacknowledged_events() {
+ let directory = temporary_directory("receipt-limit-terminal-with-unacked-events");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--hold-turn",
+ "--emit-tool-call-on-resume",
+ "--accept-interrupt-without-terminal-once",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Retain the receipt-limit diagnostic until terminal polling."}),
+ ))
+ .expect("start held provider turn");
+ drop(first);
+ saturate_provider_tool_receipts(&directory);
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let pending = recovered
+ .poll_events()
+ .expect("begin the durable receipt-limit stop");
+ assert!(pending.iter().any(|event| {
+ event.event_type == "harness.diagnostic"
+ && event.payload["code"] == "semantic_tool_turn_receipt_limit"
+ }));
+ assert!(!pending
+ .iter()
+ .any(|event| event.event_type == "turn.interrupted"));
+
+ let terminal = recovered
+ .poll_events()
+ .expect("poll the provider terminal before old events are acknowledged");
+ assert!(terminal.iter().any(|event| {
+ event.event_type == "turn.interrupted" && event.payload.get("code").is_none()
+ }));
+ assert!(!terminal.iter().any(|event| {
+ event.event_type == "turn.interrupted"
+ && event.payload["code"] == "semantic_tool_turn_receipt_limit_interrupt_deadline"
+ }));
+ recovered
+ .acknowledge_events(terminal.len())
+ .expect("acknowledge the diagnostic and authoritative terminal together");
+
+ recovered.shutdown().expect("stop recovered provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn receipt_limit_polling_bounds_and_rejects_runtime_request_floods() {
+ let directory = temporary_directory("receipt-limit-runtime-request-flood");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--hold-turn",
+ "--emit-tool-call-on-resume",
+ "--accept-interrupt-without-terminal",
+ "--flood-runtime-requests-on-interrupt",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Bound runtime requests while stopping this turn."}),
+ ))
+ .expect("start held provider turn");
+ drop(first);
+ saturate_provider_tool_receipts(&directory);
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
+ let mut rejected_at_capacity = false;
+ while !rejected_at_capacity && std::time::Instant::now() < deadline {
+ let events = recovered
+ .poll_events()
+ .expect("runtime-request cleanup remains bounded across repeated polls");
+ assert!(events.len() <= 128);
+ rejected_at_capacity = events.iter().any(|event| {
+ event.event_type == "provider.notice.recorded"
+ && event.payload["summary"]
+ == "rejected a Codex runtime request at the bounded pending-input limit"
+ });
+ recovered
+ .acknowledge_events(events.len())
+ .expect("advance the bounded durable event prefix");
+ if events.is_empty() {
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ }
+ assert!(
+ rejected_at_capacity,
+ "production records that requests above the pending count/byte envelope were rejected"
+ );
+
+ recovered.shutdown().expect("stop recovered provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn receipt_limit_synthesizes_interrupted_after_an_accepted_terminal_deadline() {
+ let directory = temporary_directory("receipt-limit-missing-terminal");
+ let config = provider_config(
+ &directory,
+ &[
+ "--require-dynamic-tool",
+ "--hold-turn",
+ "--emit-tool-call-on-resume",
+ "--accept-interrupt-without-terminal",
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({
+ "provider": config,
+ "authorizedTools": task_context_tool_set(),
+ }),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Bound a missing interrupt terminal."}),
+ ))
+ .expect("start held provider turn");
+ drop(first);
+ saturate_provider_tool_receipts(&directory);
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let mut emitted = Vec::new();
+ for _ in 0..8 {
+ emitted.extend(
+ poll_and_ack(&mut recovered)
+ .expect("receipt-limit fallback must remain durably pollable"),
+ );
+ if call_count(&directory, "turn/interrupt") == 3 {
+ break;
+ }
+ }
+ assert!(!emitted
+ .iter()
+ .any(|event| event.event_type == "turn.failed"));
+ recovered
+ .shutdown()
+ .expect("pause the provider before expiring its durable deadline");
+ drop(recovered);
+
+ let state_path = directory.join("codex-provider-state.json");
+ let mut persisted: Value =
+ serde_json::from_slice(&fs::read(&state_path).expect("read pending receipt-limit state"))
+ .expect("parse pending receipt-limit state");
+ persisted["receiptLimitInterruptDeadlineUnixMs"] = json!(1);
+ fs::write(&state_path, serde_json::to_vec_pretty(&persisted).unwrap())
+ .expect("expire the durable receipt-limit deadline");
+
+ let mut recovered = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ for _ in 0..4 {
+ emitted.extend(
+ poll_and_ack(&mut recovered)
+ .expect("expired receipt-limit fallback must remain durably pollable"),
+ );
+ if emitted
+ .iter()
+ .any(|event| event.event_type == "turn.interrupted")
+ {
+ break;
+ }
+ }
+ let interrupted = emitted
+ .iter()
+ .find(|event| event.event_type == "turn.interrupted")
+ .expect("the bounded accepted path must synthesize an interrupted turn");
+ assert_eq!(
+ interrupted.payload["code"],
+ "semantic_tool_turn_receipt_limit_interrupt_deadline"
+ );
+ assert_eq!(interrupted.payload["interruptAccepted"], true);
+ assert_eq!(interrupted.payload["providerTerminalObserved"], false);
+ assert!(!emitted
+ .iter()
+ .any(|event| event.event_type == "turn.failed"));
+ assert_eq!(call_count(&directory, "turn/interrupt"), 3);
+
+ let persisted: Value = serde_json::from_slice(
+ &fs::read(directory.join("codex-provider-state.json"))
+ .expect("read bounded receipt-limit state"),
+ )
+ .expect("parse bounded receipt-limit state");
+ assert_eq!(persisted["lifecycle"], "provider_exited");
+ assert!(persisted["activeProviderTurnId"].is_null());
+ assert_eq!(persisted["receiptLimitInterruptPending"], false);
+ assert_eq!(persisted["receiptLimitInterruptAttempts"], 0);
+ assert!(persisted["receiptLimitInterruptDeadlineUnixMs"].is_null());
+
+ recovered
+ .shutdown()
+ .expect("bounded fallback stopped provider");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
+#[test]
+fn unacknowledged_provider_events_survive_executor_restart() {
+ let directory = temporary_directory("pending-event-recovery");
+ let config = provider_config(&directory, &["--emit-question"]);
+ let mut first = CodexCommandExecutor::new(&directory);
+ first
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare Codex provider");
+ first
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open Codex session");
+ first
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Emit a durable question."}),
+ ))
+ .expect("start provider turn");
+
+ let mut retained = None;
+ for _ in 0..32 {
+ let events = first.poll_events().expect("poll provider events");
+ if events
+ .iter()
+ .any(|event| event.event_type == "runtime_request.created")
+ {
+ retained = Some(events);
+ break;
+ }
+ first
+ .acknowledge_events(events.len())
+ .expect("acknowledge events before the question");
}
let retained = retained.expect("observe a durable runtime request");
first.shutdown().expect("stop first provider process");
@@ -1672,6 +3917,70 @@ fn unacknowledged_provider_events_survive_executor_restart() {
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
+#[test]
+fn durable_backend_rejects_a_runtime_response_after_terminal_settlement() {
+ let directory = temporary_directory("durable-delayed-question-response");
+ let config = provider_config(&directory, &["--question-before-failed-turn"]);
+ let mut executor = CodexCommandExecutor::new(&directory);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider": config}),
+ ))
+ .expect("prepare provider");
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .expect("open provider session");
+ executor
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text": "Ask and then fail."}),
+ ))
+ .expect("start provider turn");
+
+ let mut request_id = None;
+ let mut terminal_seen = false;
+ for _ in 0..16 {
+ for event in poll_and_ack(&mut executor).expect("poll question and terminal") {
+ if event.event_type == "runtime_request.created" {
+ request_id = event
+ .payload
+ .pointer("/request/requestId")
+ .and_then(Value::as_str)
+ .map(str::to_owned);
+ }
+ terminal_seen |= event.event_type == "turn.failed";
+ }
+ if request_id.is_some() && terminal_seen {
+ break;
+ }
+ }
+ let request_id = request_id.expect("observe the durable runtime request id");
+ assert!(terminal_seen);
+ let error = executor
+ .execute(&command(
+ "resolve",
+ 4,
+ "request.resolve",
+ json!({
+ "requestId": request_id,
+ "response": {
+ "schema": "paperclip.question_response.v1",
+ "answers": {"environment": {"selectedOptionIds": ["option-1"]}}
+ }
+ }),
+ ))
+ .expect_err("terminal runtime requests must fail closed");
+ assert!(error.to_string().contains("outside an active turn"));
+
+ executor.shutdown().expect("stop provider process");
+ fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
+}
+
#[test]
fn structured_question_round_trips_through_the_normalized_backend() {
let directory = temporary_directory("questions");
@@ -1700,6 +4009,7 @@ fn structured_question_round_trips_through_the_normalized_backend() {
assert_eq!(started.events[0].0, "turn.accepted");
let mut question_set = None;
+ let mut request_id = None;
let mut provider_started_events = 0;
for _ in 0..16 {
for event in poll_and_ack(&mut executor).expect("poll question") {
@@ -1710,6 +4020,11 @@ fn structured_question_round_trips_through_the_normalized_backend() {
"paperclip.runtime_request.v2"
);
question_set = event.payload.pointer("/request/input").cloned();
+ request_id = event
+ .payload
+ .pointer("/request/requestId")
+ .and_then(Value::as_str)
+ .map(str::to_owned);
}
}
if question_set.is_some() {
@@ -1717,6 +4032,7 @@ fn structured_question_round_trips_through_the_normalized_backend() {
}
}
let question_set = question_set.expect("normalized question set is emitted");
+ let request_id = request_id.expect("normalized request id is emitted");
assert_eq!(provider_started_events, 1);
assert_eq!(question_set["schema"], "paperclip.question_set.v1");
assert_eq!(
@@ -1730,7 +4046,7 @@ fn structured_question_round_trips_through_the_normalized_backend() {
4,
"request.resolve",
json!({
- "requestId": "runtime-request-1",
+ "requestId": request_id,
"response": {
"schema": "paperclip.question_response.v1",
"answers": {"environment": {"selectedOptionIds": ["option-1"]}}
diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs
index 17303b3bf9..2504f383ed 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/tests/provider_bridge.rs
@@ -73,6 +73,13 @@ fn rejects_unknown_tools_and_conflicting_duplicate_results() {
};
bridge.apply_result(result.clone()).unwrap();
bridge.apply_result(result).unwrap();
+ assert!(bridge
+ .replay_result("call-1", "get_task_context", &json!({}))
+ .unwrap()
+ .is_some());
+ assert!(bridge
+ .replay_result("call-1", "get_task_context", &json!({"changed": true}))
+ .is_err());
assert!(bridge
.apply_result(ToolResult {
call_id: "call-1".to_owned(),
@@ -108,6 +115,29 @@ fn catalog_digest_matches_the_typescript_canonical_json_contract() {
);
}
+#[test]
+fn catalog_digest_normalizes_json_numbers_like_javascript() {
+ let operation = AuthorizedTool {
+ operation_id: "get_task_context".into(),
+ version: 1,
+ description: "Read the active task context.".into(),
+ input_schema: json!({
+ "type": "object",
+ "properties": {
+ "limit": { "type": "number", "default": 1.0 },
+ "epsilon": { "type": "number", "default": 1e-6 },
+ },
+ }),
+ response_schema: json!({ "type": "object" }),
+ };
+ let digest = authorized_tool_catalog_digest(&[operation]).unwrap();
+
+ assert_eq!(
+ digest,
+ "sha256:1c93693d9b5b48b46c83cd1c11d1ea329774f1b9b0ae741197cb2b8e992c4b8d"
+ );
+}
+
#[test]
fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() {
let mut set = tools("sha256:catalog-a");
@@ -120,7 +150,7 @@ fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() {
let mut bridge = ProviderToolBridge::default();
set.catalog_digest = digest('a');
set.catalog_digest = authorized_tool_catalog_digest(&set.operations).unwrap();
- bridge.prepare(set).unwrap();
+ bridge.prepare(set.clone()).unwrap();
bridge
.begin_call("call-1".into(), "get_task_context".into(), json!({}))
.unwrap();
@@ -139,6 +169,25 @@ fn validates_the_operation_value_inside_a_semantic_dispatch_envelope() {
})
.unwrap();
assert_eq!(bridge.pending_calls().count(), 0);
+
+ let mut second = ProviderToolBridge::default();
+ second.prepare(set).unwrap();
+ second
+ .begin_call("call-2".into(), "get_task_context".into(), json!({}))
+ .unwrap();
+ second
+ .apply_result(ToolResult {
+ call_id: "call-2".into(),
+ operation_id: "get_task_context".into(),
+ result: json!({
+ "ok": true,
+ "operationId": "get_task_context",
+ "callId": "call-2",
+ "value": { "value": "accepted" }
+ }),
+ is_error: false,
+ })
+ .unwrap();
}
#[test]
@@ -159,6 +208,24 @@ fn rejects_noncanonical_digests_and_oversized_contract_values() {
json!({ "value": "x".repeat(1024 * 1024) }),
)
.is_err());
+
+ let retained = json!({"value": "x".repeat(700 * 1024)});
+ for index in 0..5 {
+ bridge
+ .begin_call(
+ format!("call-{index}"),
+ "get_task_context".into(),
+ retained.clone(),
+ )
+ .unwrap();
+ }
+ assert!(bridge
+ .begin_call(
+ "call-over-aggregate-limit".into(),
+ "get_task_context".into(),
+ retained,
+ )
+ .is_err());
}
#[test]
@@ -203,12 +270,214 @@ fn recovery_preserves_completed_call_replay_identities() {
let encoded = serde_json::to_string(&bridge).unwrap();
let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();
+ recovered.validate_recovered().unwrap();
recovered.attach_existing_run().unwrap();
assert!(recovered
.begin_call("call-1".into(), "get_task_context".into(), json!({}))
.is_err());
}
+#[test]
+fn recovered_bridge_rejects_tampered_authorization_state() {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(tools("computed")).unwrap();
+ let mut encoded = serde_json::to_value(&bridge).unwrap();
+ encoded["authorized"]["get_task_context"]["description"] = json!("Tampered");
+ let recovered: ProviderToolBridge = serde_json::from_value(encoded).unwrap();
+ assert!(recovered.validate_recovered().is_err());
+}
+
+#[test]
+fn cancellation_completes_pending_calls_and_rejects_late_results() {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(tools("computed")).unwrap();
+ bridge
+ .begin_call("call-1".into(), "get_task_context".into(), json!({}))
+ .unwrap();
+ let cancelled = bridge
+ .cancel_pending_calls("provider_turn_stopped")
+ .unwrap();
+ assert_eq!(cancelled.len(), 1);
+ assert!(cancelled[0].is_error);
+ assert_eq!(bridge.pending_calls().count(), 0);
+ assert!(bridge
+ .apply_result(ToolResult {
+ call_id: "call-1".into(),
+ operation_id: "get_task_context".into(),
+ result: json!({"ok": true}),
+ is_error: false,
+ })
+ .is_err());
+}
+
+#[test]
+fn turn_settlement_releases_value_capacity_without_reusing_call_ids() {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(tools("computed")).unwrap();
+ bridge
+ .begin_call("call-1".into(), "get_task_context".into(), json!({}))
+ .unwrap();
+ bridge
+ .apply_result(ToolResult {
+ call_id: "call-1".into(),
+ operation_id: "get_task_context".into(),
+ result: json!({"ok": true}),
+ is_error: false,
+ })
+ .unwrap();
+
+ assert!(bridge
+ .settle_turn("provider_turn_terminated")
+ .unwrap()
+ .is_empty());
+ assert!(bridge
+ .begin_call("call-1".into(), "get_task_context".into(), json!({}))
+ .is_err());
+ bridge
+ .begin_call("call-2".into(), "get_task_context".into(), json!({}))
+ .expect("a new turn can use a fresh call id after releasing exact values");
+}
+
+#[test]
+fn completed_receipts_are_exact_until_the_controlled_turn_limit() {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(tools("computed")).unwrap();
+ for index in 0..4_096 {
+ let call_id = format!("call-{index}");
+ bridge
+ .begin_call(call_id.clone(), "get_task_context".into(), json!({}))
+ .expect("a completed call must release concurrent capacity");
+ bridge
+ .apply_result(ToolResult {
+ call_id,
+ operation_id: "get_task_context".into(),
+ result: json!({"ok": true}),
+ is_error: false,
+ })
+ .unwrap();
+ }
+
+ let error = bridge
+ .begin_call(
+ "call-after-limit".into(),
+ "get_task_context".into(),
+ json!({}),
+ )
+ .expect_err("the bounded exact receipt ledger must stop the active turn");
+ assert!(error.is_active_turn_receipt_limit());
+ assert!(bridge
+ .replay_result("call-0", "get_task_context", &json!({}))
+ .unwrap()
+ .is_some());
+ assert!(bridge
+ .replay_result("call-4095", "get_task_context", &json!({}))
+ .unwrap()
+ .is_some());
+
+ let recovered: ProviderToolBridge =
+ serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();
+ recovered.validate_recovered().unwrap();
+}
+
+#[test]
+fn turn_settlement_cannot_be_blocked_by_completed_value_pressure() {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(tools("computed")).unwrap();
+ let large = json!({"value": "x".repeat(700 * 1024)});
+ for index in 0..5 {
+ bridge
+ .begin_call(
+ format!("call-{index}"),
+ "get_task_context".into(),
+ large.clone(),
+ )
+ .unwrap();
+ }
+ for index in 0..4 {
+ bridge
+ .apply_result(ToolResult {
+ call_id: format!("call-{index}"),
+ operation_id: "get_task_context".into(),
+ result: large.clone(),
+ is_error: false,
+ })
+ .unwrap();
+ }
+
+ let settled = bridge.settle_turn("provider_turn_terminated").unwrap();
+ assert_eq!(settled.len(), 1);
+ assert!(settled[0].is_error);
+ assert!(bridge
+ .begin_call("call-0".into(), "get_task_context".into(), json!({}))
+ .is_err());
+ bridge
+ .begin_call(
+ "call-after-settlement".into(),
+ "get_task_context".into(),
+ json!({}),
+ )
+ .expect("settlement releases prior turn value retention");
+}
+
+#[test]
+fn recovered_turn_preserves_exact_results_at_the_value_boundary() {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(tools("computed")).unwrap();
+ let large = json!({"value": "x".repeat(700 * 1024)});
+ for index in 0..5 {
+ bridge
+ .begin_call(
+ format!("call-{index}"),
+ "get_task_context".into(),
+ large.clone(),
+ )
+ .unwrap();
+ }
+ for index in 0..5 {
+ bridge
+ .apply_result(ToolResult {
+ call_id: format!("call-{index}"),
+ operation_id: "get_task_context".into(),
+ result: large.clone(),
+ is_error: false,
+ })
+ .unwrap();
+ }
+
+ let error = bridge
+ .begin_call("call-next".into(), "get_task_context".into(), large.clone())
+ .expect_err("exact replay values must not be discarded for later work");
+ assert!(error.is_active_turn_receipt_limit());
+ assert_eq!(
+ bridge
+ .replay_result("call-0", "get_task_context", &large)
+ .unwrap()
+ .unwrap()
+ .result,
+ large,
+ );
+ bridge
+ .apply_result(ToolResult {
+ call_id: "call-0".into(),
+ operation_id: "get_task_context".into(),
+ result: large.clone(),
+ is_error: false,
+ })
+ .expect("a matching exact result receipt remains idempotent");
+ assert!(bridge
+ .apply_result(ToolResult {
+ call_id: "call-0".into(),
+ operation_id: "get_task_context".into(),
+ result: json!({"value": "changed"}),
+ is_error: false,
+ })
+ .is_err());
+
+ let recovered: ProviderToolBridge =
+ serde_json::from_str(&serde_json::to_string(&bridge).unwrap()).unwrap();
+ recovered.validate_recovered().unwrap();
+}
+
#[test]
fn recovery_preserves_pending_calls_for_the_existing_run() {
let mut bridge = ProviderToolBridge::default();
@@ -252,7 +521,7 @@ fn recovery_rejects_nonempty_state_without_a_catalog_digest() {
let error = recovered
.attach_existing_run()
.expect_err("nonempty recovered state must remain bound to a catalog digest");
- assert!(error.to_string().contains("omit the catalog digest"));
+ assert!(error.to_string().contains("omitted its catalog identity"));
}
#[test]
@@ -269,8 +538,8 @@ fn recovery_rejects_tampered_authorization_catalog_bindings() {
let mut recovered: ProviderToolBridge = serde_json::from_value(changed_contract).unwrap();
let error = recovered
.attach_existing_run()
- .expect_err("recovery must recompute the catalog digest");
- assert!(error.to_string().contains("catalog digest"));
+ .expect_err("recovery must reconstruct the authorized catalog projection");
+ assert!(error.to_string().contains("changed its authorized catalog"));
let mut changed_map_key = encoded;
let authorized = changed_map_key["authorized"].as_object_mut().unwrap();
@@ -280,7 +549,7 @@ fn recovery_rejects_tampered_authorization_catalog_bindings() {
let error = recovered
.attach_existing_run()
.expect_err("recovery must bind map keys to declared operation identities");
- assert!(error.to_string().contains("identities are inconsistent"));
+ assert!(error.to_string().contains("changed its authorized catalog"));
}
#[test]
@@ -326,18 +595,21 @@ fn recovery_rejects_tampered_retained_result_contracts() {
let completed = serde_json::to_value(&bridge).unwrap();
let mut unauthorized = completed.clone();
- unauthorized["completed"]["call-1"]["operationId"] = json!("delete_company");
- let mut recovered: ProviderToolBridge = serde_json::from_value(unauthorized).unwrap();
- assert!(recovered.attach_existing_run().is_err());
+ unauthorized["completed"]["call-1"]["result"]["operationId"] = json!("delete_company");
+ let error = serde_json::from_value::<ProviderToolBridge>(unauthorized)
+ .expect_err("durable decoding must reject mismatched call and result identities");
+ assert!(error
+ .to_string()
+ .contains("retained provider tool receipt identity is inconsistent"));
let mut invalid_output = completed;
- invalid_output["completed"]["call-1"]["result"] = json!(["not", "an", "object"]);
+ invalid_output["completed"]["call-1"]["result"]["result"] = json!(["not", "an", "object"]);
let mut recovered: ProviderToolBridge = serde_json::from_value(invalid_output).unwrap();
assert!(recovered.attach_existing_run().is_err());
- bridge.settle_turn().unwrap();
+ bridge.settle_turn("provider_turn_terminated").unwrap();
let mut invalid_settled_output = serde_json::to_value(&bridge).unwrap();
- invalid_settled_output["settledResults"]["call-1"]["result"] = json!("invalid");
+ invalid_settled_output["settledResults"]["call-1"]["result"]["result"] = json!("invalid");
let mut recovered: ProviderToolBridge = serde_json::from_value(invalid_settled_output).unwrap();
assert!(recovered.attach_existing_run().is_err());
}
@@ -389,7 +661,7 @@ fn settles_completed_receipts_before_the_next_turn() {
assert!(bridge
.begin_call("call-next".into(), "get_task_context".into(), json!({}))
.is_err());
- bridge.settle_turn().unwrap();
+ bridge.settle_turn("provider_turn_terminated").unwrap();
assert!(bridge
.begin_call("call-next".into(), "get_task_context".into(), json!({}))
.is_ok());
@@ -410,7 +682,7 @@ fn settlement_preserves_call_ids_for_the_durable_run() {
is_error: false,
})
.unwrap();
- bridge.settle_turn().unwrap();
+ bridge.settle_turn("provider_turn_terminated").unwrap();
let replay = ToolResult {
call_id: "call-1".into(),
@@ -456,7 +728,7 @@ fn settlement_preserves_call_ids_for_the_durable_run() {
}
#[test]
-fn reserves_identity_capacity_before_accepting_a_call() {
+fn exact_identity_overflow_saturates_the_durable_run() {
let mut bridge = ProviderToolBridge::default();
bridge.prepare(tools("computed")).unwrap();
@@ -472,27 +744,147 @@ fn reserves_identity_capacity_before_accepting_a_call() {
bridge
.begin_call("last-call".into(), "get_task_context".into(), json!({}))
.unwrap();
- bridge
- .apply_result(ToolResult {
- call_id: "last-call".into(),
- operation_id: "get_task_context".into(),
- result: json!({"ok": true}),
- is_error: false,
- })
- .unwrap();
- bridge.settle_turn().unwrap();
- assert!(bridge
+ let overflow = bridge
+ .begin_call("overflow".into(), "get_task_context".into(), json!({}))
+ .expect_err("the pending call reserves the final exact identity slot");
+ assert!(overflow.is_active_turn_receipt_limit());
+
+ let encoded = serde_json::to_string(&bridge).unwrap();
+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();
+ recovered.attach_existing_run().unwrap();
+ assert!(recovered.durable_run_receipt_limit_reached());
+ recovered
+ .settle_turn("provider_turn_terminated")
+ .expect("the controlled turn stop retains replay protection");
+ assert!(recovered.durable_run_receipt_limit_reached());
+ assert!(recovered.has_completed_call("settled-0"));
+ assert!(recovered.has_completed_call("last-call"));
+ let stopped_turn_receipt = recovered
+ .replay_result("last-call", "get_task_context", &json!({}))
+ .unwrap()
+ .expect("the call admitted before exhaustion retains an exact terminal receipt");
+ assert!(stopped_turn_receipt.is_error);
+ assert_eq!(
+ stopped_turn_receipt.result["error"]["code"],
+ "provider_turn_terminated"
+ );
+ assert!(recovered
+ .begin_call("settled-0".into(), "get_task_context".into(), json!({}))
+ .is_err());
+ assert!(recovered
.begin_call("overflow".into(), "get_task_context".into(), json!({}))
.is_err());
- assert!(bridge.settle_turn().is_ok());
+ recovered.prepare_turn().unwrap();
+ assert!(recovered
+ .replay_result("last-call", "get_task_context", &json!({}))
+ .unwrap()
+ .is_none());
+ assert!(recovered.has_completed_call("last-call"));
+ assert!(recovered.has_completed_call("settled-0"));
+ assert!(recovered.durable_run_receipt_limit_reached());
+ let saturation = recovered
+ .begin_call("next-call".into(), "get_task_context".into(), json!({}))
+ .expect_err("a turn boundary must not reopen the saturated durable run");
+ assert!(saturation.is_active_turn_receipt_limit());
+
+ let encoded = serde_json::to_string(&recovered).unwrap();
+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();
+ recovered.attach_existing_run().unwrap();
+ assert!(recovered.has_completed_call("settled-0"));
+ assert!(recovered.has_completed_call("last-call"));
+ assert!(recovered.durable_run_receipt_limit_reached());
+
+ recovered.attach_run(tools("computed")).unwrap();
+ assert!(!recovered.durable_run_receipt_limit_reached());
+ recovered
+ .begin_call("overflow".into(), "get_task_context".into(), json!({}))
+ .expect("a new durable run receives a fresh tool-call identity ledger");
+}
+
+#[test]
+fn settled_result_byte_exhaustion_recovers_after_turn_cleanup() {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(tools("computed")).unwrap();
+ let large_result = json!({"value": "x".repeat(750 * 1024)});
+
+ for index in 0..10 {
+ let call_id = format!("large-settled-{index}");
+ bridge
+ .begin_call(call_id.clone(), "get_task_context".into(), json!({}))
+ .unwrap();
+ bridge
+ .apply_result(ToolResult {
+ call_id,
+ operation_id: "get_task_context".into(),
+ result: large_result.clone(),
+ is_error: false,
+ })
+ .unwrap();
+ }
+ bridge.settle_turn("provider_turn_terminated").unwrap();
+
+ let error = bridge
+ .begin_call(
+ "over-byte-limit".into(),
+ "get_task_context".into(),
+ json!({}),
+ )
+ .expect_err("settled byte exhaustion must stop the durable run");
+ assert!(error.is_active_turn_receipt_limit());
+
+ let encoded = serde_json::to_string(&bridge).unwrap();
+ let mut recovered: ProviderToolBridge = serde_json::from_str(&encoded).unwrap();
+ recovered.attach_existing_run().unwrap();
+ assert!(recovered.durable_run_receipt_limit_reached());
+ assert_eq!(
+ recovered
+ .replay_result("large-settled-0", "get_task_context", &json!({}))
+ .unwrap()
+ .unwrap()
+ .result,
+ large_result
+ );
+ assert!(recovered
+ .begin_call(
+ "over-byte-limit".into(),
+ "get_task_context".into(),
+ json!({})
+ )
+ .is_err());
+ recovered.prepare_turn().unwrap();
+ assert!(recovered
+ .replay_result("large-settled-0", "get_task_context", &json!({}))
+ .unwrap()
+ .is_none());
+ assert!(recovered.has_completed_call("large-settled-0"));
+ assert!(!recovered.durable_run_receipt_limit_reached());
+ recovered
+ .begin_call(
+ "after-turn-boundary".into(),
+ "get_task_context".into(),
+ json!({}),
+ )
+ .expect("releasing bulky results clears transient byte pressure");
+ recovered
+ .settle_turn("provider_turn_terminated")
+ .expect("the admitted next-turn call remains settleable");
+
+ recovered.attach_run(tools("computed")).unwrap();
+ recovered
+ .begin_call(
+ "over-byte-limit".into(),
+ "get_task_context".into(),
+ json!({}),
+ )
+ .expect("a new durable run resets the settled result budget");
}
#[test]
fn reserves_settled_result_bytes_before_accepting_a_call() {
let mut bridge = ProviderToolBridge::default();
bridge.prepare(tools("computed")).unwrap();
- let large_result = json!({"value": "x".repeat(900 * 1024)});
+ let large_result = json!({"value": "x".repeat(700 * 1024)});
let mut completed = 0;
for index in 0..20 {
@@ -523,7 +915,7 @@ fn reserves_settled_result_bytes_before_accepting_a_call() {
)
.is_err());
bridge
- .settle_turn()
+ .settle_turn("provider_turn_terminated")
.expect("settlement cannot strand results whose bytes were reserved at admission");
assert_eq!(
bridge
@@ -534,7 +926,7 @@ fn reserves_settled_result_bytes_before_accepting_a_call() {
is_error: false,
})
.unwrap(),
- json!({"value": "x".repeat(900 * 1024)})
+ json!({"value": "x".repeat(700 * 1024)})
);
}
@@ -544,15 +936,22 @@ fn recovery_rejects_an_oversized_settled_result_envelope() {
bridge.prepare(tools("computed")).unwrap();
let mut encoded = serde_json::to_value(&bridge).unwrap();
let settled = encoded["settledResults"].as_object_mut().unwrap();
- for index in 0..10 {
+ for index in 0..12 {
let call_id = format!("recovered-large-{index}");
settled.insert(
call_id.clone(),
json!({
- "callId": call_id,
- "operationId": "get_task_context",
- "result": {"value": "x".repeat(900 * 1024)},
- "isError": false
+ "call": {
+ "callId": call_id,
+ "operationId": "get_task_context",
+ "input": {}
+ },
+ "result": {
+ "callId": call_id,
+ "operationId": "get_task_context",
+ "result": {"value": "x".repeat(700 * 1024)},
+ "isError": false
+ }
}),
);
}
@@ -568,18 +967,28 @@ fn recovery_rejects_state_without_room_for_a_pending_result() {
bridge.prepare(tools("computed")).unwrap();
let mut encoded = serde_json::to_value(&bridge).unwrap();
let settled = encoded["settledResults"].as_object_mut().unwrap();
- for index in 0..8 {
+ let mut settled_ids = Vec::new();
+ for index in 0..11 {
let call_id = format!("recovered-large-{index}");
+ settled_ids.push(json!(call_id));
settled.insert(
call_id.clone(),
json!({
- "callId": call_id,
- "operationId": "get_task_context",
- "result": {"value": "x".repeat(900 * 1024)},
- "isError": false
+ "call": {
+ "callId": call_id,
+ "operationId": "get_task_context",
+ "input": {}
+ },
+ "result": {
+ "callId": call_id,
+ "operationId": "get_task_context",
+ "result": {"value": "x".repeat(700 * 1024)},
+ "isError": false
+ }
}),
);
}
+ encoded["settledCallIds"] = serde_json::Value::Array(settled_ids);
encoded["pending"]["pending-call"] = json!({
"callId": "pending-call",
"operationId": "get_task_context",
@@ -594,13 +1003,62 @@ fn recovery_rejects_state_without_room_for_a_pending_result() {
}
#[test]
-fn refuses_to_settle_receipts_while_calls_are_pending() {
+fn settles_pending_receipts_with_explicit_terminal_results() {
let mut bridge = ProviderToolBridge::default();
bridge.prepare(tools("computed")).unwrap();
bridge
.begin_call("call-1".into(), "get_task_context".into(), json!({}))
.unwrap();
- assert!(bridge.settle_turn().is_err());
- assert_eq!(bridge.pending_calls().count(), 1);
+ let settled = bridge.settle_turn("provider_turn_terminated").unwrap();
+ assert_eq!(settled.len(), 1);
+ assert_eq!(settled[0].call_id, "call-1");
+ assert!(settled[0].is_error);
+ assert_eq!(bridge.pending_calls().count(), 0);
+}
+
+#[test]
+fn full_identity_ledger_fails_closed_across_turn_and_recovery() {
+ let mut bridge = ProviderToolBridge::default();
+ bridge.prepare(tools("computed")).unwrap();
+ let mut encoded = serde_json::to_value(&bridge).unwrap();
+ encoded["settledCallIds"] = serde_json::Value::Array(
+ (0..65_536)
+ .map(|index| json!(format!("settled-{index:05}")))
+ .collect(),
+ );
+ let mut recovered: ProviderToolBridge = serde_json::from_value(encoded).unwrap();
+ recovered.validate_recovered().unwrap();
+ let saturation = recovered
+ .begin_call("current-call".into(), "get_task_context".into(), json!({}))
+ .expect_err("a full exact ledger must stop fresh work");
+ assert!(saturation.is_active_turn_receipt_limit());
+ assert!(recovered.durable_run_receipt_limit_reached());
+ assert!(recovered.has_completed_call("settled-65535"));
+ assert!(recovered.has_completed_call("settled-00000"));
+ assert!(!recovered.has_completed_call("current-call"));
+ assert!(recovered
+ .begin_call("settled-00000".into(), "get_task_context".into(), json!({}))
+ .is_err());
+ recovered.prepare_turn().unwrap();
+ let fresh = recovered
+ .begin_call("fresh-call".into(), "get_task_context".into(), json!({}))
+ .expect_err("a turn boundary must preserve durable-run saturation");
+ assert!(fresh.is_active_turn_receipt_limit());
+
+ let round_trip = serde_json::to_value(&recovered).unwrap();
+ assert_eq!(
+ round_trip["settledCallIds"].as_array().unwrap().len(),
+ 65_536
+ );
+ let mut recovered_again: ProviderToolBridge = serde_json::from_value(round_trip).unwrap();
+ recovered_again.attach_existing_run().unwrap();
+ assert!(recovered_again.durable_run_receipt_limit_reached());
+ assert!(recovered_again.has_completed_call("settled-00000"));
+
+ recovered_again.attach_run(tools("computed")).unwrap();
+ assert!(!recovered_again.durable_run_receipt_limit_reached());
+ recovered_again
+ .begin_call("current-call".into(), "get_task_context".into(), json!({}))
+ .expect("only a new durable run resets replay authority");
}