Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

agent: Fix a UTF-8 boundary panic in the agent streaming parser (#58432)

Anthony Eid19b7625a6735
diff --git a/Cargo.lock b/Cargo.lock
index 1bf582f9221..a3e46c9eb17 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -271,6 +271,7 @@ dependencies = [
"pretty_assertions",
"project",
"prompt_store",
+ "proptest",
"quick-xml 0.38.3",
"rand 0.9.4",
"regex",
diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml
index d1f9877af3e..4e9647be103 100644
--- a/crates/agent/Cargo.toml
+++ b/crates/agent/Cargo.toml
@@ -99,6 +99,7 @@ language_model = { workspace = true, "features" = ["test-support"] }
lsp = { workspace = true, "features" = ["test-support"] }
pretty_assertions.workspace = true
project = { workspace = true, "features" = ["test-support"] }
+proptest.workspace = true
rand.workspace = true
reqwest_client.workspace = true
settings = { workspace = true, "features" = ["test-support"] }
diff --git a/crates/agent/src/tools/edit_session/streaming_parser.rs b/crates/agent/src/tools/edit_session/streaming_parser.rs
index 71dbc2c9bba..b210dd76e98 100644
--- a/crates/agent/src/tools/edit_session/streaming_parser.rs
+++ b/crates/agent/src/tools/edit_session/streaming_parser.rs
@@ -169,8 +169,9 @@ impl StreamingParser {
let mut events = SmallVec::new();
let safe_end = safe_emit_end(content);
- if safe_end > self.content_emitted_len {
- let chunk = content[self.content_emitted_len..safe_end].to_string();
+ let safe_start = find_char_boundary(content, self.content_emitted_len);
+ if safe_end > safe_start {
+ let chunk = content[safe_start..safe_end].to_string();
self.content_emitted_len = safe_end;
events.push(WriteEvent::ContentChunk { chunk });
}
@@ -228,7 +229,7 @@ impl StreamingParser {
}
if !state.old_text_done {
- let start = state.old_text_emitted_len.min(edit.old_text.len());
+ let start = find_char_boundary(&edit.old_text, state.old_text_emitted_len);
let chunk = normalize_done_chunk(edit.old_text[start..].to_string());
state.old_text_done = true;
state.old_text_emitted_len = edit.old_text.len();
@@ -240,7 +241,7 @@ impl StreamingParser {
}
if !state.new_text_done {
- let start = state.new_text_emitted_len.min(edit.new_text.len());
+ let start = find_char_boundary(&edit.new_text, state.new_text_emitted_len);
let chunk = normalize_done_chunk(edit.new_text[start..].to_string());
state.new_text_done = true;
state.new_text_emitted_len = edit.new_text.len();
@@ -259,7 +260,7 @@ impl StreamingParser {
pub fn finalize_content(&mut self, content: &str) -> SmallVec<[WriteEvent; 1]> {
let mut events = SmallVec::new();
- let start = self.content_emitted_len.min(content.len());
+ let start = find_char_boundary(content, self.content_emitted_len);
if content.len() > start {
let chunk = content[start..].to_string();
self.content_emitted_len = content.len();
@@ -313,7 +314,7 @@ impl StreamingParser {
if !state.old_text_done {
let old_text = old_text.unwrap_or_default();
- let start = state.old_text_emitted_len.min(old_text.len());
+ let start = find_char_boundary(old_text, state.old_text_emitted_len);
state.old_text_done = true;
state.old_text_emitted_len = old_text.len();
events.push(EditEvent::OldTextChunk {
@@ -325,7 +326,7 @@ impl StreamingParser {
if !state.new_text_done {
let new_text = new_text.unwrap_or_default();
- let start = state.new_text_emitted_len.min(new_text.len());
+ let start = find_char_boundary(new_text, state.new_text_emitted_len);
state.new_text_done = true;
state.new_text_emitted_len = new_text.len();
state.buffer_new_text_until_old_text_done = false;
@@ -396,6 +397,44 @@ fn normalize_done_chunk(mut chunk: String) -> String {
#[cfg(test)]
mod tests {
use super::*;
+ use proptest::prelude::*;
+
+ fn emitted_len_inside_multibyte_char() -> impl Strategy<Value = (String, String)> {
+ (1usize..8, prop::sample::select(&["。", "—", "é", "🦀"])).prop_map(
+ |(emitted_len, multibyte_char)| {
+ let first = "a".repeat(emitted_len);
+ let second = format!("{}{}", "a".repeat(emitted_len - 1), multibyte_char);
+ (first, second)
+ },
+ )
+ }
+
+ fn boundary_sensitive_text() -> impl Strategy<Value = String> {
+ prop_oneof![
+ emitted_len_inside_multibyte_char().prop_map(|(first, _)| first),
+ emitted_len_inside_multibyte_char().prop_map(|(_, second)| second),
+ prop::sample::select(&[
+ "",
+ "a",
+ "ab",
+ "ab\\",
+ "a。",
+ "a—",
+ "hello,\\",
+ "hello,\n",
+ "hello,\nworld",
+ ])
+ .prop_map(ToString::to_string),
+ ]
+ }
+
+ fn partial_edit() -> impl Strategy<Value = PartialEdit> {
+ (
+ prop::option::of(boundary_sensitive_text()),
+ prop::option::of(boundary_sensitive_text()),
+ )
+ .prop_map(|(old_text, new_text)| PartialEdit { old_text, new_text })
+ }
#[test]
fn test_first_edit_with_new_text_in_first_chunk_is_held_until_finalize() {
@@ -761,6 +800,30 @@ mod tests {
);
}
+ proptest! {
+ #[test]
+ fn test_content_finalize_does_not_panic_when_emitted_len_lands_inside_multibyte_char(
+ pair in emitted_len_inside_multibyte_char()
+ ) {
+ let (first, second) = pair;
+ let mut parser = StreamingParser::default();
+
+ parser.push_content(&first);
+ parser.finalize_content(&second);
+ }
+
+ #[test]
+ fn test_push_edits_does_not_panic_on_boundary_sensitive_sequences(
+ partials in prop::collection::vec(prop::collection::vec(partial_edit(), 0..4), 1..12)
+ ) {
+ let mut parser = StreamingParser::default();
+
+ for edits in partials {
+ parser.push_edits(&edits);
+ }
+ }
+ }
+
#[test]
fn test_no_partials_direct_finalize() {
let mut parser = StreamingParser::default();