Lightweight coding agent that runs in your terminal
Rust
Derive report prompt hashes from persisted rollouts (#36271)
Felicia Chenf0c30e528a54
diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.tomlindex f9cc95b843..3824d15ccc 100644--- a/codex-rs/app-server/Cargo.toml+++ b/codex-rs/app-server/Cargo.toml@@ -91,6 +91,7 @@ clap = { workspace = true, features = ["derive"] }futures = { workspace = true }serde = { workspace = true, features = ["derive"] }serde_json = { workspace = true }+sha2 = { workspace = true }tempfile = { workspace = true }thiserror = { workspace = true }time = { workspace = true }@@ -135,7 +136,6 @@ rmcp = { workspace = true, default-features = false, features = ["transport-streamable-http-server",] }serial_test = { workspace = true }-sha2 = { workspace = true }shlex = { workspace = true }tar = { workspace = true }test-case = "3.3.1"diff --git a/codex-rs/app-server/src/request_processors/feedback_processor.rs b/codex-rs/app-server/src/request_processors/feedback_processor.rsindex 5fbb39e603..216f3b8b43 100644--- a/codex-rs/app-server/src/request_processors/feedback_processor.rs+++ b/codex-rs/app-server/src/request_processors/feedback_processor.rs@@ -7,6 +7,8 @@ use codex_feedback::CODEX_APPS_TOOLS_CACHE_ATTACHMENT_FILENAME;#[cfg(target_os = "windows")]use codex_feedback::WINDOWS_SANDBOX_LOG_ATTACHMENT_FILENAME;use codex_rollout::RolloutRecorder;+use sha2::Digest;+use sha2::Sha256;const MAX_FEEDBACK_TREE_THREADS: usize = 8;@@ -76,21 +78,22 @@ impl FeedbackRequestProcessor {None => None,};- if let Some(conversation_id) = conversation_id+ let auth = self.auth_manager.auth_cached();+ let turn_metadata = if let Some(conversation_id) = conversation_id&& let Some(rollout_path) = self.resolve_rollout_path(conversation_id, self.state_db.as_ref()).await- && let Some((model, reasoning_effort)) = feedback_model_and_effort_from_rollout(+ {+ feedback_turn_metadata_from_rollout(&rollout_path,upload_tags.get("turn_id").map(String::as_str),).await- {- upload_tags.insert("model".to_string(), model);- upload_tags.insert("effort".to_string(), format!("{reasoning_effort:?}"));- }+ } else {+ None+ };+ apply_feedback_turn_metadata(&mut upload_tags, turn_metadata);- let auth = self.auth_manager.auth_cached();if let Some(chatgpt_user_id) = auth.as_ref().and_then(codex_login::CodexAuth::get_chatgpt_user_id)@@ -300,24 +303,71 @@ impl FeedbackRequestProcessor {}}-async fn feedback_model_and_effort_from_rollout(+#[derive(Debug, PartialEq)]+struct FeedbackTurnMetadata {+ model: String,+ effort: Option<ReasoningEffort>,+ prompt_hash: Option<String>,+}++fn apply_feedback_turn_metadata(+ upload_tags: &mut BTreeMap<String, String>,+ turn_metadata: Option<FeedbackTurnMetadata>,+) {+ // These are reserved tags derived from the persisted rollout rather than+ // accepted from the feedback request.+ upload_tags.remove("prompt_hash");+ upload_tags.remove("prompt_version");++ if let Some(FeedbackTurnMetadata {+ model,+ effort,+ prompt_hash,+ }) = turn_metadata+ {+ upload_tags.insert("model".to_string(), model);+ upload_tags.insert("effort".to_string(), format!("{effort:?}"));+ if let Some(prompt_hash) = prompt_hash {+ upload_tags.insert("prompt_hash".to_string(), prompt_hash);+ }+ }+}++async fn feedback_turn_metadata_from_rollout(rollout_path: &Path,turn_id: Option<&str>,-) -> Option<(String, Option<ReasoningEffort>)> {+) -> Option<FeedbackTurnMetadata> {let (items, _, _) = RolloutRecorder::load_rollout_items(rollout_path).await.ok()?;+ let prompt_hash = items.iter().find_map(|item| match item {+ RolloutItem::SessionMeta(meta) => meta+ .meta+ .base_instructions+ .as_ref()+ .map(|prompt| normalized_prompt_hash(&prompt.text)),+ _ => None,+ });items.into_iter().rev().find_map(|item| match item {RolloutItem::TurnContext(context)if turn_id.is_none() || context.turn_id.as_deref() == turn_id =>{- Some((context.model, context.effort))+ Some(FeedbackTurnMetadata {+ model: context.model,+ effort: context.effort,+ prompt_hash: prompt_hash.clone(),+ })}_ => None,})}+fn normalized_prompt_hash(prompt: &str) -> String {+ let normalized_prompt = prompt.split_whitespace().collect::<Vec<_>>().join(" ");+ format!("{:x}", Sha256::digest(normalized_prompt.as_bytes()))+}+fn tool_cache_feedback_attachments(codex_home: &Path,chatgpt_base_url: &str,@@ -385,52 +435,179 @@ mod tests {use codex_protocol::protocol::RolloutLine;use codex_protocol::protocol::TurnContextItem;use pretty_assertions::assert_eq;+ #[test]+ fn feedback_tags_drop_unverified_client_prompt_tags() {+ let mut upload_tags = BTreeMap::from([+ ("custom".to_string(), "preserved".to_string()),+ (+ "prompt_hash".to_string(),+ "unverified-client-hash".to_string(),+ ),+ ("prompt_version".to_string(), "client-prompt-v1".to_string()),+ ]);++ apply_feedback_turn_metadata(&mut upload_tags, /*turn_metadata*/ None);++ assert_eq!(+ upload_tags,+ BTreeMap::from([("custom".to_string(), "preserved".to_string())])+ );+ }++ #[test]+ fn feedback_tags_drop_client_prompt_hash_when_rollout_has_no_hash() {+ let mut upload_tags = BTreeMap::from([(+ "prompt_hash".to_string(),+ "unverified-client-hash".to_string(),+ )]);++ apply_feedback_turn_metadata(+ &mut upload_tags,+ Some(FeedbackTurnMetadata {+ model: "reported-model".to_string(),+ effort: Some(ReasoningEffort::High),+ prompt_hash: None,+ }),+ );++ assert_eq!(+ upload_tags,+ BTreeMap::from([+ ("effort".to_string(), "Some(High)".to_string()),+ ("model".to_string(), "reported-model".to_string()),+ ])+ );+ }++ #[test]+ fn feedback_tags_replace_client_prompt_hash_with_rollout_hash() {+ let mut upload_tags = BTreeMap::from([(+ "prompt_hash".to_string(),+ "unverified-client-hash".to_string(),+ )]);++ apply_feedback_turn_metadata(+ &mut upload_tags,+ Some(FeedbackTurnMetadata {+ model: "reported-model".to_string(),+ effort: Some(ReasoningEffort::High),+ prompt_hash: Some("rollout-prompt-hash".to_string()),+ }),+ );++ assert_eq!(+ upload_tags,+ BTreeMap::from([+ ("effort".to_string(), "Some(High)".to_string()),+ ("model".to_string(), "reported-model".to_string()),+ ("prompt_hash".to_string(), "rollout-prompt-hash".to_string(),),+ ])+ );+ }++ #[tokio::test]+ async fn feedback_tags_do_not_trust_the_prompt_version_from_the_reported_rollout() {+ let (_tempdir, rollout_path) =+ feedback_rollout(&[("turn-1", "synthetic-model", Some(ReasoningEffort::High))]);+ let mut upload_tags = BTreeMap::from([(+ "prompt_version".to_string(),+ "unverified-client-prompt".to_string(),+ )]);++ let turn_metadata =+ feedback_turn_metadata_from_rollout(&rollout_path, Some("turn-1")).await;+ apply_feedback_turn_metadata(&mut upload_tags, turn_metadata);++ assert_eq!(+ upload_tags,+ BTreeMap::from([+ ("effort".to_string(), "Some(High)".to_string()),+ ("model".to_string(), "synthetic-model".to_string()),+ (+ "prompt_hash".to_string(),+ normalized_prompt_hash("actual developer prompt"),+ ),+ ])+ );+ }#[tokio::test]- async fn feedback_model_and_effort_use_the_reported_turn() {+ async fn feedback_metadata_uses_the_reported_turn() {let (_tempdir, rollout_path) = feedback_rollout(&[("turn-1", "reported-model", Some(ReasoningEffort::High)),("turn-2", "newer-model", Some(ReasoningEffort::Ultra)),]);assert_eq!(- feedback_model_and_effort_from_rollout(&rollout_path, Some("turn-1")).await,- Some(("reported-model".to_string(), Some(ReasoningEffort::High)))+ feedback_turn_metadata_from_rollout(&rollout_path, Some("turn-1")).await,+ Some(FeedbackTurnMetadata {+ model: "reported-model".to_string(),+ effort: Some(ReasoningEffort::High),+ prompt_hash: Some(normalized_prompt_hash("actual developer prompt")),+ }));}#[tokio::test]- async fn feedback_model_and_effort_use_the_latest_turn_when_no_turn_is_reported() {+ async fn feedback_metadata_uses_the_latest_turn_when_no_turn_is_reported() {let (_tempdir, rollout_path) = feedback_rollout(&[("turn-1", "older-model", Some(ReasoningEffort::High)),("turn-2", "latest-model", Some(ReasoningEffort::Ultra)),]);assert_eq!(- feedback_model_and_effort_from_rollout(&rollout_path, /*turn_id*/ None).await,- Some(("latest-model".to_string(), Some(ReasoningEffort::Ultra)))+ feedback_turn_metadata_from_rollout(&rollout_path, /*turn_id*/ None).await,+ Some(FeedbackTurnMetadata {+ model: "latest-model".to_string(),+ effort: Some(ReasoningEffort::Ultra),+ prompt_hash: Some(normalized_prompt_hash("actual developer prompt")),+ }));}#[tokio::test]- async fn feedback_model_and_effort_do_not_substitute_a_different_turn() {+ async fn feedback_metadata_does_not_substitute_a_different_turn() {let (_tempdir, rollout_path) =feedback_rollout(&[("turn-1", "different-model", Some(ReasoningEffort::High))]);assert_eq!(- feedback_model_and_effort_from_rollout(&rollout_path, Some("missing-turn")).await,+ feedback_turn_metadata_from_rollout(&rollout_path, Some("missing-turn")).await,None);}#[tokio::test]- async fn feedback_model_and_effort_preserve_unspecified_effort() {+ async fn feedback_metadata_preserves_unspecified_effort_and_prompt_hash() {let (_tempdir, rollout_path) =feedback_rollout(&[("turn-1", "reported-model", /*effort*/ None)]);assert_eq!(- feedback_model_and_effort_from_rollout(&rollout_path, Some("turn-1")).await,- Some(("reported-model".to_string(), None))+ feedback_turn_metadata_from_rollout(&rollout_path, Some("turn-1")).await,+ Some(FeedbackTurnMetadata {+ model: "reported-model".to_string(),+ effort: None,+ prompt_hash: Some(normalized_prompt_hash("actual developer prompt")),+ })+ );+ }++ #[tokio::test]+ async fn feedback_hashes_the_actual_developer_prompt_from_session_metadata() {+ let (_tempdir, rollout_path) = feedback_rollout(&[("turn-1", "reported-model", None)]);++ assert_eq!(+ feedback_turn_metadata_from_rollout(&rollout_path, Some("turn-1"))+ .await+ .and_then(|metadata| metadata.prompt_hash),+ Some(normalized_prompt_hash("actual developer prompt")),+ );+ }++ #[test]+ fn prompt_hash_normalizes_whitespace() {+ assert_eq!(+ normalized_prompt_hash("actual developer\r\nprompt\t"),+ "9ae77301cc2a30e729c28661b7a0f9490c80a72e7d23277e7e74f0ac81779541");}@@ -445,6 +622,9 @@ mod tests {item: RolloutItem::SessionMeta(SessionMetaLine {meta: codex_protocol::protocol::SessionMeta {cwd: tempdir.path().to_path_buf(),+ base_instructions: Some(codex_protocol::models::BaseInstructions {+ text: "actual developer prompt".to_string(),+ }),..Default::default()},git: None,