Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Make opening large files use less peak memory (#62748)

Ali00c0e96e7690
diff --git a/crates/languages/src/json.rs b/crates/languages/src/json.rs
index d96be549ac0..3d5f8ea4668 100644
--- a/crates/languages/src/json.rs
+++ b/crates/languages/src/json.rs
@@ -69,11 +69,12 @@ impl ContextProvider for JsonTaskProvider {
.await
.ok()?;
let path = cx.update(|cx| file.abs_path(cx)).as_path().into();
+ let contents_text = contents.text.to_string();
let task_templates = if is_package_json {
let package_json = serde_json_lenient::from_str::<
HashMap<String, serde_json_lenient::Value>,
- >(&contents.text)
+ >(&contents_text)
.ok()?;
let package_json = PackageJsonData::new(path, package_json);
let command = package_json.package_manager.unwrap_or("npm").to_owned();
@@ -100,7 +101,7 @@ impl ContextProvider for JsonTaskProvider {
}])
.collect()
} else if is_composer_json {
- serde_json_lenient::Value::from_str(&contents.text)
+ serde_json_lenient::Value::from_str(&contents_text)
.ok()?
.get("scripts")?
.as_object()?
diff --git a/crates/project/src/buffer_store.rs b/crates/project/src/buffer_store.rs
index ff49a39f761..4f01061aa9d 100644
--- a/crates/project/src/buffer_store.rs
+++ b/crates/project/src/buffer_store.rs
@@ -696,7 +696,12 @@ impl LocalBufferStore {
let buffer_id = BufferId::from(reservation.entity_id().as_non_zero_u64());
let text_buffer = cx
.background_spawn(async move {
- text::Buffer::new(ReplicaId::LOCAL, buffer_id, loaded.text)
+ text::Buffer::new_normalized(
+ ReplicaId::LOCAL,
+ buffer_id,
+ loaded.line_ending,
+ loaded.text,
+ )
})
.await;
cx.insert_entity(reservation, |_| {
diff --git a/crates/project/tests/integration/project_tests.rs b/crates/project/tests/integration/project_tests.rs
index 2c744ef4a9c..fc8270a61fb 100644
--- a/crates/project/tests/integration/project_tests.rs
+++ b/crates/project/tests/integration/project_tests.rs
@@ -6105,7 +6105,8 @@ async fn test_rename_file_to_new_directory(cx: &mut gpui::TestAppContext) {
})
.await
.unwrap()
- .text,
+ .text
+ .to_string(),
expected_contents,
"Moved file's contents should be preserved"
);
@@ -6152,7 +6153,8 @@ async fn test_rename_file_to_new_directory(cx: &mut gpui::TestAppContext) {
})
.await
.unwrap()
- .text,
+ .text
+ .to_string(),
expected_contents,
"Moved file's contents should be preserved"
);
diff --git a/crates/vim/src/command.rs b/crates/vim/src/command.rs
index f5d4fe52668..a80a6fd9f01 100644
--- a/crates/vim/src/command.rs
+++ b/crates/vim/src/command.rs
@@ -732,13 +732,11 @@ pub fn register(editor: &mut Editor, cx: &mut Context<Vim>) {
cx.spawn_in(window, async move |editor, cx| {
if let Some(task) = task {
- text.push_str(
- &task
- .await
- .log_err()
- .map(|loaded_file| loaded_file.text)
- .unwrap_or_default(),
- );
+ if let Some(loaded_file) = task.await.log_err() {
+ for chunk in loaded_file.text.chunks() {
+ text.push_str(chunk);
+ }
+ }
}
if !text.is_empty() && !is_end_of_file {
diff --git a/crates/worktree/src/worktree.rs b/crates/worktree/src/worktree.rs
index 4a8a726a15d..02e3b2c38cb 100644
--- a/crates/worktree/src/worktree.rs
+++ b/crates/worktree/src/worktree.rs
@@ -59,6 +59,7 @@ use std::{
ffi::OsStr,
fmt,
future::Future,
+ io::Read,
mem::{self},
ops::{Deref, DerefMut, Range},
path::{Path, PathBuf},
@@ -108,7 +109,8 @@ pub enum CreatedEntry {
#[derive(Debug)]
pub struct LoadedFile {
pub file: Arc<File>,
- pub text: String,
+ pub text: Rope,
+ pub line_ending: LineEnding,
pub encoding: &'static Encoding,
pub has_bom: bool,
pub is_writable: bool,
@@ -1715,7 +1717,8 @@ impl LocalWorktree {
{
anyhow::bail!("File is too large to load");
}
- let (text, encoding, has_bom) = decode_file_text(fs.as_ref(), &abs_path).await?;
+ let (text, line_ending, encoding, has_bom) =
+ decode_file_text_to_rope(fs.as_ref(), &abs_path).await?;
let is_writable = metadata.is_some_and(|metadata| metadata.is_writable);
let worktree = this.upgrade().context("worktree was dropped")?;
@@ -1748,6 +1751,7 @@ impl LocalWorktree {
Ok(LoadedFile {
file,
text,
+ line_ending,
encoding,
has_bom,
is_writable,
@@ -7173,24 +7177,13 @@ impl fs::Watcher for NullWatcher {
}
}
-pub async fn decode_file_text(
- fs: &dyn Fs,
- abs_path: &Path,
-) -> Result<(String, &'static Encoding, bool)> {
- let mut file = fs
- .open_sync(&abs_path)
- .await
- .with_context(|| format!("opening file {abs_path:?}"))?;
-
- // First, read the beginning of the file to determine its kind and encoding.
- // We do not want to load an entire large blob into memory only to discard it.
- let mut file_first_bytes = Vec::with_capacity(FILE_ANALYSIS_BYTES);
+/// Reads the beginning of `file` to determine its kind and encoding, returning
+/// the bytes consumed and whether the file ended within them.
+fn read_file_header(file: &mut dyn Read, abs_path: &Path) -> Result<(Vec<u8>, bool)> {
+ let mut header = Vec::with_capacity(FILE_ANALYSIS_BYTES);
let mut buf = [0u8; FILE_ANALYSIS_BYTES];
let mut reached_eof = false;
- loop {
- if file_first_bytes.len() >= FILE_ANALYSIS_BYTES {
- break;
- }
+ while header.len() < FILE_ANALYSIS_BYTES {
let n = file
.read(&mut buf)
.with_context(|| format!("reading bytes of the file {abs_path:?}"))?;
@@ -7198,8 +7191,21 @@ pub async fn decode_file_text(
reached_eof = true;
break;
}
- file_first_bytes.extend_from_slice(&buf[..n]);
+ header.extend_from_slice(&buf[..n]);
}
+ Ok((header, reached_eof))
+}
+
+pub async fn decode_file_text(
+ fs: &dyn Fs,
+ abs_path: &Path,
+) -> Result<(String, &'static Encoding, bool)> {
+ let mut file = fs
+ .open_sync(&abs_path)
+ .await
+ .with_context(|| format!("opening file {abs_path:?}"))?;
+
+ let (file_first_bytes, reached_eof) = read_file_header(&mut *file, abs_path)?;
let (_, byte_content) = decode_byte_header(&file_first_bytes);
anyhow::ensure!(
byte_content != ByteContent::Binary,
@@ -7224,6 +7230,156 @@ pub async fn decode_file_text(
Ok((decoded.text, decoded.encoding, decoded.has_bom))
}
+const STREAM_BLOCK_BYTES: usize = 1024 * 1024;
+/// Reads and decodes a file straight into a [`Rope`].
+/// The returned rope has already had its line endings normalized, the
+/// [`LineEnding`] detected before normalizing is returned alongside it.
+pub async fn decode_file_text_to_rope(
+ fs: &dyn Fs,
+ abs_path: &Path,
+) -> Result<(Rope, LineEnding, &'static Encoding, bool)> {
+ let mut file = fs
+ .open_sync(abs_path)
+ .await
+ .with_context(|| format!("opening file {abs_path:?}"))?;
+
+ let (prefix, reached_eof) = read_file_header(&mut *file, abs_path)?;
+ let (bom_encoding, byte_content) = decode_byte_header(&prefix);
+ anyhow::ensure!(
+ byte_content != ByteContent::Binary,
+ "Binary files are not supported"
+ );
+
+ // Only BOM-less, non-UTF-16 files are candidates for streaming: everything
+ // else needs the whole byte buffer in hand to decode or to detect encoding.
+ if bom_encoding.is_none()
+ && byte_content == ByteContent::Unknown
+ && let Some((rope, line_ending)) =
+ stream_utf8_into_rope(&mut *file, prefix, reached_eof, abs_path)?
+ {
+ return Ok((rope, line_ending, encoding_rs::UTF_8, false));
+ }
+
+ // Not plain UTF-8 after all. Re-read the file and decode it all at once.
+ let (mut text, encoding, has_bom) = decode_file_text(fs, abs_path).await?;
+ let line_ending = LineEnding::detect(&text);
+ LineEnding::normalize(&mut text);
+ Ok((Rope::from(text), line_ending, encoding, has_bom))
+}
+
+/// Streams a presumed-UTF-8 file into a [`Rope`], normalizing line endings as it
+/// goes.
+///
+/// Returns `None` if the file turns out not to be plain UTF-8, in which case the
+/// caller re-reads it and decodes it the slow way. `prefix` is the portion of
+/// the file already consumed from `file` for encoding detection.
+fn stream_utf8_into_rope(
+ file: &mut dyn Read,
+ prefix: Vec<u8>,
+ reached_eof: bool,
+ abs_path: &Path,
+) -> Result<Option<(Rope, LineEnding)>> {
+ let mut rope = Rope::new();
+ let mut line_ending = None;
+ let mut scratch = String::new();
+ let mut pending = prefix;
+ let mut buf = vec![0u8; STREAM_BLOCK_BYTES];
+ let mut eof = reached_eof;
+
+ loop {
+ // Fill a whole block before decoding, so that each `Rope::push` gets a
+ // slice large enough to build its chunks in parallel.
+ while !eof && pending.len() < STREAM_BLOCK_BYTES {
+ let n = file
+ .read(&mut buf)
+ .with_context(|| format!("reading bytes of the file {abs_path:?}"))?;
+ if n == 0 {
+ eof = true;
+ } else {
+ pending.extend_from_slice(&buf[..n]);
+ }
+ }
+
+ // Decode as much of `pending` as forms complete UTF-8.
+ let valid_len = match std::str::from_utf8(&pending) {
+ Ok(_) => pending.len(),
+ // A multi-byte character straddling the block boundary; the rest of
+ // it arrives with the next block.
+ Err(e) if e.error_len().is_none() && !eof => e.valid_up_to(),
+ // Genuinely not UTF-8, or truncated at EOF: fall back.
+ Err(_) => return Ok(None),
+ };
+
+ // Hold back a trailing carriage return until we know what follows it.
+ let emit_len = if !eof && valid_len > 0 && pending[valid_len - 1] == b'\r' {
+ valid_len - 1
+ } else {
+ valid_len
+ };
+
+ let text = std::str::from_utf8(&pending[..emit_len])
+ .expect("a prefix of validated UTF-8 is itself valid UTF-8");
+
+ // ISO-2022-JP and friends are valid UTF-8 but carry escape sequences, so
+ // they need the full-file encoding detector rather than this fast path.
+ if text.contains('\x1b') {
+ return Ok(None);
+ }
+
+ // `LineEnding::detect` only inspects the first 1000 bytes, so the first
+ // block gives the same answer the whole file would.
+ if line_ending.is_none() && !text.is_empty() {
+ line_ending = Some(LineEnding::detect(text));
+ }
+
+ push_normalized(&mut rope, text, &mut scratch);
+ pending.drain(..emit_len);
+
+ if eof {
+ break;
+ }
+ }
+
+ // At EOF everything should have been consumed. Anything left over is a
+ // truncated multi-byte sequence, which means this is not valid UTF-8.
+ if !pending.is_empty() {
+ return Ok(None);
+ }
+
+ Ok(Some((rope, line_ending.unwrap_or_default())))
+}
+
+/// Appends `text` to `rope`, rewriting CRLF and lone CR as LF, matching
+/// [`LineEnding::normalize`]. `scratch` is reused across calls.
+fn push_normalized(rope: &mut Rope, text: &str, scratch: &mut String) {
+ if !text.contains('\r') {
+ rope.push(text);
+ return;
+ }
+
+ scratch.clear();
+ scratch.reserve(text.len());
+ let bytes = text.as_bytes();
+ let mut start = 0;
+ let mut ix = 0;
+ while ix < bytes.len() {
+ if bytes[ix] == b'\r' {
+ scratch.push_str(&text[start..ix]);
+ scratch.push('\n');
+ ix += if bytes.get(ix + 1) == Some(&b'\n') {
+ 2
+ } else {
+ 1
+ };
+ start = ix;
+ } else {
+ ix += 1;
+ }
+ }
+ scratch.push_str(&text[start..]);
+ rope.push(scratch);
+}
+
pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteContent) {
if let Some((encoding, _bom_len)) = Encoding::for_bom(prefix) {
return (Some(encoding), ByteContent::Unknown);
@@ -7235,6 +7391,63 @@ pub fn decode_byte_header(prefix: &[u8]) -> (Option<&'static Encoding>, ByteCont
mod tests {
use super::*;
+ /// Streams `bytes` the way `decode_file_text_to_rope` would, returning the
+ /// decoded text and detected line ending, or `None` if the fast path bailed.
+ fn stream(bytes: &[u8]) -> Option<(String, LineEnding)> {
+ let mut reader = std::io::Cursor::new(bytes.to_vec());
+ stream_utf8_into_rope(&mut reader, Vec::new(), false, Path::new("test"))
+ .unwrap()
+ .map(|(rope, line_ending)| (rope.to_string(), line_ending))
+ }
+
+ #[test]
+ fn test_stream_utf8_normalizes_line_endings() {
+ let crlf = "one\r\ntwo\r\nthree\r\n".repeat(40);
+ let (text, line_ending) = stream(crlf.as_bytes()).unwrap();
+ assert_eq!(text, crlf.replace("\r\n", "\n"));
+ assert_eq!(line_ending, LineEnding::Windows);
+
+ let cr = "one\rtwo\rthree\r".repeat(40);
+ assert_eq!(stream(cr.as_bytes()).unwrap().0, cr.replace('\r', "\n"));
+ }
+
+ #[test]
+ fn test_stream_utf8_block_boundaries() {
+ // A carriage return landing on the last byte of a block, with and
+ // without its newline arriving in the next one.
+ for (suffix, expected) in [("\r\ntail\n", "\ntail\n"), ("\rtail", "\ntail")] {
+ let filler = "a".repeat(STREAM_BLOCK_BYTES - 1);
+ let source = format!("{filler}{suffix}");
+ assert_eq!(
+ stream(source.as_bytes()).unwrap().0,
+ format!("{filler}{expected}"),
+ "suffix = {suffix:?}"
+ );
+ }
+
+ // Multi-byte characters straddling the boundary at every split point.
+ for ch in ['\u{20ac}', '\u{1f600}'] {
+ for split in 1..=ch.len_utf8() {
+ let filler = "a".repeat(STREAM_BLOCK_BYTES - split);
+ let source = format!("{filler}{ch}tail");
+ assert_eq!(
+ stream(source.as_bytes()).unwrap().0,
+ source,
+ "ch = {ch:?}, split = {split}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn test_stream_utf8_falls_back_on_non_utf8() {
+ // Each of these must bail so the caller re-reads and decodes the slow
+ // way, rather than silently mangling the file.
+ assert_eq!(stream(b"hello \xff\xfeA"), None, "invalid utf-8");
+ assert_eq!(stream(b"hello \xe2\x82"), None, "truncated at eof");
+ assert_eq!(stream(b"plain \x1b$B text"), None, "iso-2022 escape");
+ }
+
/// reproduction of issue #50785
fn build_pcm16_wav_bytes() -> Vec<u8> {
let header: Vec<u8> = vec![
diff --git a/crates/worktree/tests/integration/worktree_tests.rs b/crates/worktree/tests/integration/worktree_tests.rs
index 62ce80e4f2b..6ca65a10b72 100644
--- a/crates/worktree/tests/integration/worktree_tests.rs
+++ b/crates/worktree/tests/integration/worktree_tests.rs
@@ -5593,7 +5593,8 @@ async fn test_load_file_encoding(cx: &mut TestAppContext) {
}
let loaded = loaded.unwrap();
assert_eq!(
- loaded.text, case.expected_text,
+ loaded.text.to_string(),
+ case.expected_text,
"Encoding mismatch for file: {}",
case.name
);