Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Make opening large files use less peak memory (#62748)
Ali00c0e96e7690
diff --git a/crates/languages/src/json.rs b/crates/languages/src/json.rsindex d96be549ac0..3d5f8ea4668 100644--- a/crates/languages/src/json.rs+++ b/crates/languages/src/json.rs@@ -69,11 +69,12 @@ impl ContextProvider for JsonTaskProvider {.await.ok()?;let path = cx.update(|cx| file.abs_path(cx)).as_path().into();+ let contents_text = contents.text.to_string();let task_templates = if is_package_json {let package_json = serde_json_lenient::from_str::<HashMap<String, serde_json_lenient::Value>,- >(&contents.text)+ >(&contents_text).ok()?;let package_json = PackageJsonData::new(path, package_json);let command = package_json.package_manager.unwrap_or("npm").to_owned();@@ -100,7 +101,7 @@ impl ContextProvider for JsonTaskProvider {}]).collect()} else if is_composer_json {- serde_json_lenient::Value::from_str(&contents.text)+ serde_json_lenient::Value::from_str(&contents_text).ok()?.get("scripts")?.as_object()?diff --git a/crates/project/src/buffer_store.rs b/crates/project/src/buffer_store.rsindex ff49a39f761..4f01061aa9d 100644--- a/crates/project/src/buffer_store.rs+++ b/crates/project/src/buffer_store.rs@@ -696,7 +696,12 @@ impl LocalBufferStore {let buffer_id = BufferId::from(reservation.entity_id().as_non_zero_u64());let text_buffer = cx.background_spawn(async move {- text::Buffer::new(ReplicaId::LOCAL, buffer_id, loaded.text)+ text::Buffer::new_normalized(+ ReplicaId::LOCAL,+ buffer_id,+ loaded.line_ending,+ loaded.text,+ )}).await;cx.insert_entity(reservation, |_| {diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rsindex 2c744ef4a9c..fc8270a61fb 100644--- a/crates/project/tests/integration/project_tests.rs+++ b/crates/project/tests/integration/project_tests.rs@@ -6105,7 +6105,8 @@ async fn test_rename_file_to_new_directory(cx: &mut gpui::TestAppContext) {}).await.unwrap()- .text,+ .text+ .to_string(),expected_contents,"Moved file's contents should be preserved");@@ -6152,7 +6153,8 @@ async fn test_rename_file_to_new_directory(cx: &mut gpui::TestAppContext) {}).await.unwrap()- .text,+ .text+ .to_string(),expected_contents,"Moved file's contents should be preserved");diff --git a/crates/vim/src/command.rs b/crates/vim/src/command.rsindex f5d4fe52668..a80a6fd9f01 100644--- a/crates/vim/src/command.rs+++ b/crates/vim/src/command.rs@@ -732,13 +732,11 @@ pub fn register(editor: &mut Editor, cx: &mut Context<Vim>) {cx.spawn_in(window, async move |editor, cx| {if let Some(task) = task {- text.push_str(- &task- .await- .log_err()- .map(|loaded_file| loaded_file.text)- .unwrap_or_default(),- );+ if let Some(loaded_file) = task.await.log_err() {+ for chunk in loaded_file.text.chunks() {+ text.push_str(chunk);+ }+ }}if !text.is_empty() && !is_end_of_file {diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rsindex 4a8a726a15d..02e3b2c38cb 100644--- a/crates/worktree/src/worktree.rs+++ b/crates/worktree/src/worktree.rs@@ -59,6 +59,7 @@ use std::{ffi::OsStr,fmt,future::Future,+ io::Read,mem::{self},ops::{Deref, DerefMut, Range},path::{Path, PathBuf},@@ -108,7 +109,8 @@ pub enum CreatedEntry {#[derive(Debug)]pub struct LoadedFile {pub file: Arc<File>,- pub text: String,+ pub text: Rope,+ pub line_ending: LineEnding,pub encoding: &'static Encoding,pub has_bom: bool,pub is_writable: bool,@@ -1715,7 +1717,8 @@ impl LocalWorktree {{anyhow::bail!("File is too large to load");}- let (text, encoding, has_bom) = decode_file_text(fs.as_ref(), &abs_path).await?;+ let (text, line_ending, encoding, has_bom) =+ decode_file_text_to_rope(fs.as_ref(), &abs_path).await?;let is_writable = metadata.is_some_and(|metadata| metadata.is_writable);let worktree = this.upgrade().context("worktree was dropped")?;@@ -1748,6 +1751,7 @@ impl LocalWorktree {Ok(LoadedFile {file,text,+ line_ending,encoding,has_bom,is_writable,@@ -7173,24 +7177,13 @@ impl fs::Watcher for NullWatcher {}}-pub async fn decode_file_text(- fs: &dyn Fs,- abs_path: &Path,-) -> Result<(String, &'static Encoding, bool)> {- let mut file = fs- .open_sync(&abs_path)- .await- .with_context(|| format!("opening file {abs_path:?}"))?;-- // First, read the beginning of the file to determine its kind and encoding.- // We do not want to load an entire large blob into memory only to discard it.- let mut file_first_bytes = Vec::with_capacity(FILE_ANALYSIS_BYTES);+/// Reads the beginning of `file` to determine its kind and encoding, returning+/// the bytes consumed and whether the file ended within them.+fn read_file_header(file: &mut dyn Read, abs_path: &Path) -> Result<(Vec<u8>, bool)> {+ let mut header = Vec::with_capacity(FILE_ANALYSIS_BYTES);let mut buf = [0u8; FILE_ANALYSIS_BYTES];let mut reached_eof = false;- loop {- if file_first_bytes.len() >= FILE_ANALYSIS_BYTES {- break;- }+ while header.len() < FILE_ANALYSIS_BYTES {let n = file.read(&mut buf).with_context(|| format!("reading bytes of the file {abs_path:?}"))?;@@ -7198,8 +7191,21 @@ pub async fn decode_file_text(reached_eof = true;break;}- file_first_bytes.extend_from_slice(&buf[..n]);+ header.extend_from_slice(&buf[..n]);}+ Ok((header, reached_eof))+}++pub async fn decode_file_text(+ fs: &dyn Fs,+ abs_path: &Path,+) -> Result<(String, &'static Encoding, bool)> {+ let mut file = fs+ .open_sync(&abs_path)+ .await+ .with_context(|| format!("opening file {abs_path:?}"))?;++ let (file_first_bytes, reached_eof) = read_file_header(&mut *file, abs_path)?;let (_, byte_content) = decode_byte_header(&file_first_bytes);anyhow::ensure!(byte_content != ByteContent::Binary,@@ -7224,6 +7230,156 @@ pub async fn decode_file_text(Ok((decoded.text, decoded.encoding, decoded.has_bom))}+const STREAM_BLOCK_BYTES: usize = 1024 * 1024;+/// Reads and decodes a file straight into a [`Rope`].+/// The returned rope has already had its line endings normalized, the+/// [`LineEnding`] detected before normalizing is returned alongside it.+pub async fn decode_file_text_to_rope(+ fs: &dyn Fs,+ abs_path: &Path,+) -> Result<(Rope, LineEnding, &'static Encoding, bool)> {+ let mut file = fs+ .open_sync(abs_path)+ .await+ .with_context(|| format!("opening file {abs_path:?}"))?;++ let (prefix, reached_eof) = read_file_header(&mut *file, abs_path)?;+ let (bom_encoding, byte_content) = decode_byte_header(&prefix);+ anyhow::ensure!(+ byte_content != ByteContent::Binary,+ "Binary files are not supported"+ );++ // Only BOM-less, non-UTF-16 files are candidates for streaming: everything+ // else needs the whole byte buffer in hand to decode or to detect encoding.+ if bom_encoding.is_none()+ && byte_content == ByteContent::Unknown+ && let Some((rope, line_ending)) =+ stream_utf8_into_rope(&mut *file, prefix, reached_eof, abs_path)?+ {+ return Ok((rope, line_ending, encoding_rs::UTF_8, false));+ }++ // Not plain UTF-8 after all. Re-read the file and decode it all at once.+ let (mut text, encoding, has_bom) = decode_file_text(fs, abs_path).await?;+ let line_ending = LineEnding::detect(&text);+ LineEnding::normalize(&mut text);+ Ok((Rope::from(text), line_ending, encoding, has_bom))+}++/// Streams a presumed-UTF-8 file into a [`Rope`], normalizing line endings as it+/// goes.+///+/// Returns `None` if the file turns out not to be plain UTF-8, in which case the+/// caller re-reads it and decodes it the slow way. `prefix` is the portion of+/// the file already consumed from `file` for encoding detection.+fn stream_utf8_into_rope(+ file: &mut dyn Read,+ prefix: Vec<u8>,+ reached_eof: bool,+ abs_path: &Path,+) -> Result<Option<(Rope, LineEnding)>> {+ let mut rope = Rope::new();+ let mut line_ending = None;+ let mut scratch = String::new();+ let mut pending = prefix;+ let mut buf = vec![0u8; STREAM_BLOCK_BYTES];+ let mut eof = reached_eof;++ loop {+ // Fill a whole block before decoding, so that each `Rope::push` gets a+ // slice large enough to build its chunks in parallel.+ while !eof && pending.len() < STREAM_BLOCK_BYTES {+ let n = file+ .read(&mut buf)+ .with_context(|| format!("reading bytes of the file {abs_path:?}"))?;+ if n == 0 {+ eof = true;+ } else {+ pending.extend_from_slice(&buf[..n]);+ }+ }++ // Decode as much of `pending` as forms complete UTF-8.+ let valid_len = match std::str::from_utf8(&pending) {+ Ok(_) => pending.len(),+ // A multi-byte character straddling the block boundary; the rest of+ // it arrives with the next block.+ Err(e) if e.error_len().is_none() && !eof => e.valid_up_to(),+ // Genuinely not UTF-8, or truncated at EOF: fall back.+ Err(_) => return Ok(None),+ };++ // Hold back a trailing carriage return until we know what follows it.+ let emit_len = if !eof && valid_len > 0 && pending[valid_len - 1] == b'\r' {+ valid_len - 1+ } else {+ valid_len+ };++ let text = std::str::from_utf8(&pending[..emit_len])+ .expect("a prefix of validated UTF-8 is itself valid UTF-8");++ // ISO-2022-JP and friends are valid UTF-8 but carry escape sequences, so+ // they need the full-file encoding detector rather than this fast path.+ if text.contains('\x1b') {+ return Ok(None);+ }++ // `LineEnding::detect` only inspects the first 1000 bytes, so the first+ // block gives the same answer the whole file would.+ if line_ending.is_none() && !text.is_empty() {+ line_ending = Some(LineEnding::detect(text));+ }++ push_normalized(&mut rope, text, &mut scratch);+ pending.drain(..emit_len);++ if eof {+ break;+ }+ }++ // At EOF everything should have been consumed. Anything left over is a+ // truncated multi-byte sequence, which means this is not valid UTF-8.+ if !pending.is_empty() {+ return Ok(None);+ }++ Ok(Some((rope, line_ending.unwrap_or_default())))+}++/// Appends `text` to `rope`, rewriting CRLF and lone CR as LF, matching+/// [`LineEnding::normalize`]. `scratch` is reused across calls.+fn push_normalized(rope: &mut Rope, text: &str, scratch: &mut String) {+ if !text.contains('\r') {+ rope.push(text);+ return;+ }++ scratch.clear();+ scratch.reserve(text.len());+ let bytes = text.as_bytes();+ let mut start = 0;+ let mut ix = 0;+ while ix < bytes.len() {+ if bytes[ix] == b'\r' {+ scratch.push_str(&text[start..ix]);+ scratch.push('\n');+ ix += if bytes.get(ix + 1) == Some(&b'\n') {+ 2+ } else {+ 1+ };+ start = ix;+ } else {+ ix += 1;+ }+ }+ scratch.push_str(&text[start..]);+ rope.push(scratch);+}+pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {return (Some(encoding), ByteContent::Unknown);@@ -7235,6 +7391,63 @@ pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContmod tests {use super::*;+ /// Streams `bytes` the way `decode_file_text_to_rope` would, returning the+ /// decoded text and detected line ending, or `None` if the fast path bailed.+ fn stream(bytes: &[u8]) -> Option<(String, LineEnding)> {+ let mut reader = std::io::Cursor::new(bytes.to_vec());+ stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"))+ .unwrap()+ .map(|(rope, line_ending)| (rope.to_string(), line_ending))+ }++ #[test]+ fn test_stream_utf8_normalizes_line_endings() {+ let crlf = "one\r\ntwo\r\nthree\r\n".repeat(40);+ let (text, line_ending) = stream(crlf.as_bytes()).unwrap();+ assert_eq!(text, crlf.replace("\r\n", "\n"));+ assert_eq!(line_ending, LineEnding::Windows);++ let cr = "one\rtwo\rthree\r".repeat(40);+ assert_eq!(stream(cr.as_bytes()).unwrap().0, cr.replace('\r', "\n"));+ }++ #[test]+ fn test_stream_utf8_block_boundaries() {+ // A carriage return landing on the last byte of a block, with and+ // without its newline arriving in the next one.+ for (suffix, expected) in [("\r\ntail\n", "\ntail\n"), ("\rtail", "\ntail")] {+ let filler = "a".repeat(STREAM_BLOCK_BYTES - 1);+ let source = format!("{filler}{suffix}");+ assert_eq!(+ stream(source.as_bytes()).unwrap().0,+ format!("{filler}{expected}"),+ "suffix = {suffix:?}"+ );+ }++ // Multi-byte characters straddling the boundary at every split point.+ for ch in ['\u{20ac}', '\u{1f600}'] {+ for split in 1..=ch.len_utf8() {+ let filler = "a".repeat(STREAM_BLOCK_BYTES - split);+ let source = format!("{filler}{ch}tail");+ assert_eq!(+ stream(source.as_bytes()).unwrap().0,+ source,+ "ch = {ch:?}, split = {split}"+ );+ }+ }+ }++ #[test]+ fn test_stream_utf8_falls_back_on_non_utf8() {+ // Each of these must bail so the caller re-reads and decodes the slow+ // way, rather than silently mangling the file.+ assert_eq!(stream(b"hello \xff\xfeA"), None, "invalid utf-8");+ assert_eq!(stream(b"hello \xe2\x82"), None, "truncated at eof");+ assert_eq!(stream(b"plain \x1b$B text"), None, "iso-2022 escape");+ }+/// reproduction of issue #50785fn build_pcm16_wav_bytes() -> Vec<u8> {let header: Vec<u8> = vec![diff --git a/crates/worktree/tests/integration/worktree_tests.rs b/crates/worktree/tests/integration/worktree_tests.rsindex 62ce80e4f2b..6ca65a10b72 100644--- a/crates/worktree/tests/integration/worktree_tests.rs+++ b/crates/worktree/tests/integration/worktree_tests.rs@@ -5593,7 +5593,8 @@ async fn test_load_file_encoding(cx: &mut TestAppContext) {}let loaded = loaded.unwrap();assert_eq!(- loaded.text, case.expected_text,+ loaded.text.to_string(),+ case.expected_text,"Encoding mismatch for file: {}",case.name);