Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
agent: Fix a UTF-8 boundary panic in the agent streaming parser (#58432)
Anthony Eid19b7625a6735
diff --git a/Cargo.lock b/Cargo.lockindex 1bf582f9221..a3e46c9eb17 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -271,6 +271,7 @@ dependencies = ["pretty_assertions","project","prompt_store",+ "proptest","quick-xml 0.38.3","rand 0.9.4","regex",diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.tomlindex d1f9877af3e..4e9647be103 100644--- a/crates/agent/Cargo.toml+++ b/crates/agent/Cargo.toml@@ -99,6 +99,7 @@ language_model = { workspace = true, "features" = ["test-support"] }lsp = { workspace = true, "features" = ["test-support"] }pretty_assertions.workspace = trueproject = { workspace = true, "features" = ["test-support"] }+proptest.workspace = truerand.workspace = truereqwest_client.workspace = truesettings = { workspace = true, "features" = ["test-support"] }diff --git a/crates/agent/src/tools/edit_session/streaming_parser.rs b/crates/agent/src/tools/edit_session/streaming_parser.rsindex 71dbc2c9bba..b210dd76e98 100644--- a/crates/agent/src/tools/edit_session/streaming_parser.rs+++ b/crates/agent/src/tools/edit_session/streaming_parser.rs@@ -169,8 +169,9 @@ impl StreamingParser {let mut events = SmallVec::new();let safe_end = safe_emit_end(content);- if safe_end > self.content_emitted_len {- let chunk = content[self.content_emitted_len..safe_end].to_string();+ let safe_start = find_char_boundary(content, self.content_emitted_len);+ if safe_end > safe_start {+ let chunk = content[safe_start..safe_end].to_string();self.content_emitted_len = safe_end;events.push(WriteEvent::ContentChunk { chunk });}@@ -228,7 +229,7 @@ impl StreamingParser {}if !state.old_text_done {- let start = state.old_text_emitted_len.min(edit.old_text.len());+ let start = find_char_boundary(&edit.old_text, state.old_text_emitted_len);let chunk = normalize_done_chunk(edit.old_text[start..].to_string());state.old_text_done = true;state.old_text_emitted_len = edit.old_text.len();@@ -240,7 +241,7 @@ impl StreamingParser {}if !state.new_text_done {- let start = state.new_text_emitted_len.min(edit.new_text.len());+ let start = find_char_boundary(&edit.new_text, state.new_text_emitted_len);let chunk = normalize_done_chunk(edit.new_text[start..].to_string());state.new_text_done = true;state.new_text_emitted_len = edit.new_text.len();@@ -259,7 +260,7 @@ impl StreamingParser {pub fn finalize_content(&mut self, content: &str) -> SmallVec<[WriteEvent; 1]> {let mut events = SmallVec::new();- let start = self.content_emitted_len.min(content.len());+ let start = find_char_boundary(content, self.content_emitted_len);if content.len() > start {let chunk = content[start..].to_string();self.content_emitted_len = content.len();@@ -313,7 +314,7 @@ impl StreamingParser {if !state.old_text_done {let old_text = old_text.unwrap_or_default();- let start = state.old_text_emitted_len.min(old_text.len());+ let start = find_char_boundary(old_text, state.old_text_emitted_len);state.old_text_done = true;state.old_text_emitted_len = old_text.len();events.push(EditEvent::OldTextChunk {@@ -325,7 +326,7 @@ impl StreamingParser {if !state.new_text_done {let new_text = new_text.unwrap_or_default();- let start = state.new_text_emitted_len.min(new_text.len());+ let start = find_char_boundary(new_text, state.new_text_emitted_len);state.new_text_done = true;state.new_text_emitted_len = new_text.len();state.buffer_new_text_until_old_text_done = false;@@ -396,6 +397,44 @@ fn normalize_done_chunk(mut chunk: String) -> String {#[cfg(test)]mod tests {use super::*;+ use proptest::prelude::*;++ fn emitted_len_inside_multibyte_char() -> impl Strategy<Value = (String, String)> {+ (1usize..8, prop::sample::select(&["。", "—", "é", "🦀"])).prop_map(+ |(emitted_len, multibyte_char)| {+ let first = "a".repeat(emitted_len);+ let second = format!("{}{}", "a".repeat(emitted_len - 1), multibyte_char);+ (first, second)+ },+ )+ }++ fn boundary_sensitive_text() -> impl Strategy<Value = String> {+ prop_oneof![+ emitted_len_inside_multibyte_char().prop_map(|(first, _)| first),+ emitted_len_inside_multibyte_char().prop_map(|(_, second)| second),+ prop::sample::select(&[+ "",+ "a",+ "ab",+ "ab\\",+ "a。",+ "a—",+ "hello,\\",+ "hello,\n",+ "hello,\nworld",+ ])+ .prop_map(ToString::to_string),+ ]+ }++ fn partial_edit() -> impl Strategy<Value = PartialEdit> {+ (+ prop::option::of(boundary_sensitive_text()),+ prop::option::of(boundary_sensitive_text()),+ )+ .prop_map(|(old_text, new_text)| PartialEdit { old_text, new_text })+ }#[test]fn test_first_edit_with_new_text_in_first_chunk_is_held_until_finalize() {@@ -761,6 +800,30 @@ mod tests {);}+ proptest! {+ #[test]+ fn test_content_finalize_does_not_panic_when_emitted_len_lands_inside_multibyte_char(+ pair in emitted_len_inside_multibyte_char()+ ) {+ let (first, second) = pair;+ let mut parser = StreamingParser::default();++ parser.push_content(&first);+ parser.finalize_content(&second);+ }++ #[test]+ fn test_push_edits_does_not_panic_on_boundary_sensitive_sequences(+ partials in prop::collection::vec(prop::collection::vec(partial_edit(), 0..4), 1..12)+ ) {+ let mut parser = StreamingParser::default();++ for edits in partials {+ parser.push_edits(&edits);+ }+ }+ }+#[test]fn test_no_partials_direct_finalize() {let mut parser = StreamingParser::default();