Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
extension: Perform compilation tasks in parallel (#55160)
Finn Evers9ee3c503a4bb
diff --git a/Cargo.lock b/Cargo.lockindex 21fd2010fa3..04be6b1aab6 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -6269,6 +6269,7 @@ dependencies = ["env_logger 0.11.8","extension","fs",+ "futures 0.3.32","gpui_platform","language","log",diff --git a/Cargo.toml b/Cargo.tomlindex 037afc59bfe..6e88facf82b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -591,6 +591,7 @@ fork = "0.4.0"futures = "0.3.32"futures-concurrency = "7.7.1"futures-lite = "1.13"+futures-util = "0.3.32"gh-workflow = { git = "https://github.com/zed-industries/gh-workflow", rev = "37f3c0575d379c218a9c455ee67585184e40d43f" }globset = "0.4"diff --git a/crates/extension/src/extension_builder.rs b/crates/extension/src/extension_builder.rsindex 2fc50434603..6229da6b8dc 100644--- a/crates/extension/src/extension_builder.rs+++ b/crates/extension/src/extension_builder.rs@@ -1,21 +1,26 @@use crate::{- ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, build_debug_adapter_schema_path,- parse_wasm_extension_version,+ ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, parse_wasm_extension_version,};use ::fs::Fs;use anyhow::{Context as _, Result, bail};-use futures::{StreamExt, io};+use futures::{+ FutureExt, StreamExt,+ channel::oneshot::{self, Sender},+ io,+};use heck::ToSnakeCase;use http_client::{self, AsyncBody, HttpClient};use language::LanguageConfig;+use semver::Version;use serde::Deserialize;use std::{env, fs, mem,+ num::NonZeroUsize,+ ops::Not,path::{Path, PathBuf},- str::FromStr,sync::Arc,};-use util::{command::Stdio, rel_path::PathExt};+use util::{ResultExt, command::Stdio, rel_path::PathExt};use wasm_encoder::{ComponentSectionId, Encode as _, RawSection, Section as _};use wasmparser::Parser;@@ -44,13 +49,24 @@ pub struct ExtensionBuilder {pub http: Arc<dyn HttpClient>,}+pub enum CompilationConcurrency {+ Unbounded,+ Bounded(NonZeroUsize),+}++const DEFAULT_COMPILATION_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(3).unwrap();+pub struct CompileExtensionOptions {pub release: bool,+ pub max_concurrency: CompilationConcurrency,}impl CompileExtensionOptions {pub const fn dev() -> Self {- Self { release: false }+ Self {+ release: false,+ max_concurrency: CompilationConcurrency::Bounded(DEFAULT_COMPILATION_CONCURRENCY),+ }}}@@ -79,7 +95,9 @@ impl ExtensionBuilder {options: CompileExtensionOptions,fs: Arc<dyn Fs>,) -> Result<()> {- populate_defaults(extension_manifest, extension_dir, fs).await?;+ let start = std::time::Instant::now();++ populate_defaults(extension_manifest, extension_dir, fs.clone()).await?;if extension_dir.is_relative() {bail!(@@ -88,58 +106,116 @@ impl ExtensionBuilder {);}- fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?;+ fs.create_dir(&self.cache_dir)+ .await+ .context("failed to create cache dir")?;- if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) {- log::info!("compiling Rust extension {}", extension_dir.display());- self.compile_rust_extension(extension_dir, extension_manifest, options)- .await- .context("failed to compile Rust extension")?;- log::info!("compiled Rust extension {}", extension_dir.display());- }+ let (tx, mut rx) = oneshot::channel();- for (debug_adapter_name, meta) in &mut extension_manifest.debug_adapters {- let debug_adapter_schema_path =- extension_dir.join(build_debug_adapter_schema_path(debug_adapter_name, meta)?);+ let clang_path = extension_manifest.grammars.is_empty().not().then(|| {+ std::iter::repeat_n(+ async {+ self.install_wasi_sdk_if_needed()+ .await+ .log_err()+ .map(Arc::new)+ }+ .shared(),+ extension_manifest.grammars.len(),+ )+ });++ let rust_compilation_task =+ (extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust)).then(|| {+ async {+ log::info!("compiling Rust extension {}", extension_dir.display());+ self.compile_rust_extension(extension_dir, extension_manifest, tx, &options)+ .await+ .context("failed to compile Rust extension")?;++ log::info!("compiled Rust extension {}", extension_dir.display());+ Ok(())+ }+ .boxed()+ });++ let grammar_compilation_tasks = extension_manifest+ .grammars+ .iter()+ .zip(clang_path.into_iter().flatten())+ .map(|((grammar_name, grammar_metadata), clang_path_task)| {+ async move {+ let snake_cased_grammar_name = grammar_name.to_snake_case();+ if grammar_name.as_ref() != snake_cased_grammar_name.as_str() {+ bail!(+ "grammar name '{grammar_name}' must be \+ written in snake_case: {snake_cased_grammar_name}"+ );+ }- let debug_adapter_schema = fs::read_to_string(&debug_adapter_schema_path)- .with_context(|| {- format!("failed to read debug adapter schema for `{debug_adapter_name}` from `{debug_adapter_schema_path:?}`")- })?;- _ = serde_json::Value::from_str(&debug_adapter_schema).with_context(|| {- format!("Debug adapter schema for `{debug_adapter_name}` (path: `{debug_adapter_schema_path:?}`) is not a valid JSON")- })?;- }- for (grammar_name, grammar_metadata) in &extension_manifest.grammars {- let snake_cased_grammar_name = grammar_name.to_snake_case();- if grammar_name.as_ref() != snake_cased_grammar_name.as_str() {- bail!(- "grammar name '{grammar_name}' must be written in snake_case: {snake_cased_grammar_name}"- );+ log::info!(+ "compiling grammar {grammar_name} for extension {}",+ extension_dir.display()+ );++ let clang_path = clang_path_task+ .await+ .context("Failed to resolve clang path")?;++ self.compile_grammar(+ extension_dir,+ grammar_name.as_ref(),+ grammar_metadata,+ &clang_path,+ )+ .await+ .with_context(|| format!("failed to compile grammar '{grammar_name}'"))?;+ log::info!(+ "compiled grammar {grammar_name} for extension {}",+ extension_dir.display()+ );++ Ok(())+ }+ .boxed()+ });++ let tasks = rust_compilation_task+ .into_iter()+ .chain(grammar_compilation_tasks)+ .collect::<Vec<_>>();++ match options.max_concurrency {+ CompilationConcurrency::Unbounded => {+ futures::future::try_join_all(tasks).await?;}+ CompilationConcurrency::Bounded(max_concurrency) => {+ let mut stream = futures::stream::iter(tasks).buffered(max_concurrency.get());- log::info!(- "compiling grammar {grammar_name} for extension {}",- extension_dir.display()- );- self.compile_grammar(extension_dir, grammar_name.as_ref(), grammar_metadata)- .await- .with_context(|| format!("failed to compile grammar '{grammar_name}'"))?;- log::info!(- "compiled grammar {grammar_name} for extension {}",- extension_dir.display()- );+ while let Some(result) = stream.next().await {+ result?;+ }+ }}- log::info!("finished compiling extension {}", extension_dir.display());+ if let Ok(version) = rx.try_recv() {+ extension_manifest.lib.version = version;+ }++ log::info!(+ "finished compiling extension {} in {time:.2}s",+ extension_dir.display(),+ time = start.elapsed().as_secs_f64(),+ );Ok(())}async fn compile_rust_extension(&self,extension_dir: &Path,- manifest: &mut ExtensionManifest,- options: CompileExtensionOptions,+ manifest: &ExtensionManifest,+ wasm_extension_api_version_tx: Sender<Version>,+ options: &CompileExtensionOptions,) -> anyhow::Result<()> {self.install_rust_wasm_target_if_needed().await?;@@ -201,7 +277,9 @@ impl ExtensionBuilder {let wasm_extension_api_version =parse_wasm_extension_version(&manifest.id, &component_bytes).context("compiled wasm did not contain a valid zed extension api version")?;- manifest.lib.version = Some(wasm_extension_api_version);+ wasm_extension_api_version_tx+ .send(wasm_extension_api_version)+ .map_err(|_| anyhow::anyhow!("Failed to send API version"))?;let extension_file = extension_dir.join("extension.wasm");fs::write(extension_file.clone(), &component_bytes)@@ -221,9 +299,8 @@ impl ExtensionBuilder {extension_dir: &Path,grammar_name: &str,grammar_metadata: &GrammarManifestEntry,+ clang_path: &Path,) -> Result<()> {- let clang_path = self.install_wasi_sdk_if_needed().await?;-let mut grammar_repo_dir = extension_dir.to_path_buf();grammar_repo_dir.extend(["grammars", grammar_name]);diff --git a/crates/extension_cli/Cargo.toml b/crates/extension_cli/Cargo.tomlindex 2170647845b..623187faaa3 100644--- a/crates/extension_cli/Cargo.toml+++ b/crates/extension_cli/Cargo.toml@@ -18,6 +18,7 @@ clap = { workspace = true, features = ["derive"] }cloud_api_types.workspace = trueenv_logger.workspace = trueextension.workspace = true+futures.workspace = truefs.workspace = truegpui_platform.workspace = truelanguage.workspace = truediff --git a/crates/extension_cli/src/main.rs b/crates/extension_cli/src/main.rsindex 0b7c491ec11..921990d284c 100644--- a/crates/extension_cli/src/main.rs+++ b/crates/extension_cli/src/main.rs@@ -3,12 +3,15 @@ use std::collections::HashMap;use std::env;use std::fs;use std::path::{Path, PathBuf};+use std::str::FromStr as _;use std::sync::Arc;-use ::fs::{CopyOptions, Fs, RealFs, copy_recursive};+use ::fs::{CopyOptions, Fs, RealFs, RemoveOptions, copy_recursive};use anyhow::{Context as _, Result, anyhow, bail};use clap::Parser;use cloud_api_types::ExtensionProvides;+use extension::build_debug_adapter_schema_path;+use extension::extension_builder::CompilationConcurrency;use extension::extension_builder::{CompileExtensionOptions, ExtensionBuilder};use extension::{ExtensionManifest, ExtensionSnippets};use language::LanguageConfig;@@ -47,6 +50,11 @@ async fn main() -> Result<()> {.source_dir.canonicalize().context("failed to canonicalize source_dir")?;++ fs.create_dir(&args.scratch_dir)+ .await+ .context("failed to create scratch dir")?;+let scratch_dir = args.scratch_dir.canonicalize()@@ -75,7 +83,10 @@ async fn main() -> Result<()> {.compile_extension(&extension_path,&mut manifest,- CompileExtensionOptions { release: true },+ CompileExtensionOptions {+ release: true,+ max_concurrency: CompilationConcurrency::Unbounded,+ },fs.clone(),).await@@ -88,9 +99,18 @@ async fn main() -> Result<()> {test_languages(&manifest, &extension_path, &grammars)?;test_themes(&manifest, &extension_path, fs.clone()).await?;test_snippets(&manifest, &extension_path, fs.clone()).await?;+ test_debug_adapter_schemas(&manifest, &extension_path, fs.clone()).await?;let archive_dir = output_dir.join("archive");- fs::remove_dir_all(&archive_dir).ok();+ fs.remove_dir(+ &archive_dir,+ RemoveOptions {+ recursive: true,+ ignore_if_not_exists: true,+ },+ )+ .await+ .ok();copy_extension_resources(&manifest, &extension_path, &archive_dir, fs.clone()).await.context("failed to copy extension resources")?;@@ -120,8 +140,16 @@ async fn main() -> Result<()> {wasm_api_version: manifest.lib.version.map(|version| version.to_string()),provides: extension_provides,})?;- fs::remove_dir_all(&archive_dir)?;- fs::write(output_dir.join("manifest.json"), manifest_json.as_bytes())?;+ fs.remove_dir(+ &archive_dir,+ RemoveOptions {+ recursive: true,+ ignore_if_not_exists: false,+ },+ )+ .await?;+ fs.write(&output_dir.join("manifest.json"), manifest_json.as_bytes())+ .await?;Ok(())}@@ -132,68 +160,107 @@ async fn copy_extension_resources(output_dir: &Path,fs: Arc<dyn Fs>,) -> Result<()> {- fs::create_dir_all(output_dir).context("failed to create output dir")?;+ fs.create_dir(output_dir)+ .await+ .context("failed to create output dir")?;let manifest_toml = toml::to_string(&manifest).context("failed to serialize manifest")?;- fs::write(output_dir.join("extension.toml"), &manifest_toml)+ fs.write(&output_dir.join("extension.toml"), manifest_toml.as_bytes())+ .await.context("failed to write extension.toml")?;if manifest.lib.kind.is_some() {- fs::copy(- extension_path.join("extension.wasm"),- output_dir.join("extension.wasm"),+ fs.copy_file(+ &extension_path.join("extension.wasm"),+ &output_dir.join("extension.wasm"),+ CopyOptions {+ overwrite: true,+ ignore_if_exists: false,+ },)+ .await.context("failed to copy extension.wasm")?;}if !manifest.grammars.is_empty() {let source_grammars_dir = extension_path.join("grammars");let output_grammars_dir = output_dir.join("grammars");- fs::create_dir_all(&output_grammars_dir)?;- for grammar_name in manifest.grammars.keys() {- let mut grammar_filename = PathBuf::from(grammar_name.as_ref());- grammar_filename.set_extension("wasm");- fs::copy(- source_grammars_dir.join(&grammar_filename),- output_grammars_dir.join(&grammar_filename),- )- .with_context(|| format!("failed to copy grammar '{}'", grammar_filename.display()))?;- }+ fs.create_dir(&output_grammars_dir).await?;+ futures::future::try_join_all(manifest.grammars.keys().map(|grammar_name| {+ let fs = fs.clone();+ let source_grammars_dir = source_grammars_dir.as_path();+ let output_grammars_dir = output_grammars_dir.as_path();+ async move {+ let mut grammar_filename = PathBuf::from(grammar_name.as_ref());+ grammar_filename.set_extension("wasm");+ fs.copy_file(+ &source_grammars_dir.join(&grammar_filename),+ &output_grammars_dir.join(&grammar_filename),+ CopyOptions {+ overwrite: true,+ ignore_if_exists: false,+ },+ )+ .await+ .with_context(|| format!("failed to copy grammar '{}'", grammar_filename.display()))+ }+ }))+ .await?;}if !manifest.themes.is_empty() {let output_themes_dir = output_dir.join("themes");- fs::create_dir_all(&output_themes_dir)?;- for theme_path in &manifest.themes {- let theme_path = theme_path.as_std_path();- fs::copy(- extension_path.join(theme_path),- output_themes_dir.join(theme_path.file_name().context("invalid theme path")?),- )- .with_context(|| format!("failed to copy theme '{}'", theme_path.display()))?;- }+ fs.create_dir(&output_themes_dir).await?;+ futures::future::try_join_all(manifest.themes.iter().map(|theme_path| {+ let fs = fs.clone();+ let output_themes_dir = output_themes_dir.as_path();+ async move {+ let theme_path = theme_path.as_std_path();+ fs.copy_file(+ &extension_path.join(theme_path),+ &output_themes_dir.join(theme_path.file_name().context("invalid theme path")?),+ CopyOptions {+ overwrite: true,+ ignore_if_exists: false,+ },+ )+ .await+ .with_context(|| format!("failed to copy theme '{}'", theme_path.display()))+ }+ }))+ .await?;}if !manifest.icon_themes.is_empty() {let output_icon_themes_dir = output_dir.join("icon_themes");- fs::create_dir_all(&output_icon_themes_dir)?;- for icon_theme_path in &manifest.icon_themes {- let icon_theme_path = icon_theme_path.as_std_path();- fs::copy(- extension_path.join(icon_theme_path),- output_icon_themes_dir.join(- icon_theme_path- .file_name()- .context("invalid icon theme path")?,- ),- )- .with_context(|| {- format!("failed to copy icon theme '{}'", icon_theme_path.display())- })?;- }+ fs.create_dir(&output_icon_themes_dir).await?;+ futures::future::try_join_all(manifest.icon_themes.iter().map(|icon_theme_path| {+ let fs = fs.clone();+ let output_icon_themes_dir = output_icon_themes_dir.as_path();+ async move {+ let icon_theme_path = icon_theme_path.as_std_path();+ fs.copy_file(+ &extension_path.join(icon_theme_path),+ &output_icon_themes_dir.join(+ icon_theme_path+ .file_name()+ .context("invalid icon theme path")?,+ ),+ CopyOptions {+ overwrite: true,+ ignore_if_exists: false,+ },+ )+ .await+ .with_context(|| {+ format!("failed to copy icon theme '{}'", icon_theme_path.display())+ })+ }+ }))+ .await?;let output_icons_dir = output_dir.join("icons");- fs::create_dir_all(&output_icons_dir)?;+ fs.create_dir(&output_icons_dir).await?;copy_recursive(fs.as_ref(),&extension_path.join("icons"),@@ -209,73 +276,90 @@ async fn copy_extension_resources(if !manifest.languages.is_empty() {let output_languages_dir = output_dir.join("languages");- fs::create_dir_all(&output_languages_dir)?;- for language_path in &manifest.languages {- let language_path = language_path.as_std_path();- copy_recursive(- fs.as_ref(),- &extension_path.join(language_path),- &output_languages_dir- .join(language_path.file_name().context("invalid language path")?),- CopyOptions {- overwrite: true,- ignore_if_exists: false,- },- )- .await- .with_context(|| {- format!("failed to copy language dir '{}'", language_path.display())- })?;- }+ fs.create_dir(&output_languages_dir).await?;+ futures::future::try_join_all(manifest.languages.iter().map(|language_path| {+ let fs = fs.clone();+ let output_languages_dir = output_languages_dir.clone();+ async move {+ let language_path = language_path.as_std_path();+ copy_recursive(+ fs.as_ref(),+ &extension_path.join(language_path),+ &output_languages_dir+ .join(language_path.file_name().context("invalid language path")?),+ CopyOptions {+ overwrite: true,+ ignore_if_exists: false,+ },+ )+ .await+ .with_context(|| {+ format!("failed to copy language dir '{}'", language_path.display())+ })+ }+ }))+ .await?;}if !manifest.debug_adapters.is_empty() {- for (debug_adapter, entry) in &manifest.debug_adapters {- let schema_path = extension::build_debug_adapter_schema_path(debug_adapter, entry)?;- let parent = schema_path- .parent()- .with_context(|| format!("invalid empty schema path for {debug_adapter}"))?;- let schema_path = schema_path.as_std_path();- fs::create_dir_all(output_dir.join(parent))?;- copy_recursive(- fs.as_ref(),- &extension_path.join(&schema_path),- &output_dir.join(&schema_path),- CopyOptions {- overwrite: true,- ignore_if_exists: false,- },- )- .await- .with_context(|| {- format!(- "failed to copy debug adapter schema '{}'",- schema_path.display(),- )- })?;- }+ futures::future::try_join_all(manifest.debug_adapters.iter().map(+ |(debug_adapter, entry)| {+ let fs = fs.clone();+ let debug_adapter = debug_adapter.clone();+ async move {+ let schema_path =+ extension::build_debug_adapter_schema_path(&debug_adapter, &entry)?;+ let parent = schema_path.parent().with_context(|| {+ format!("invalid empty schema path for {debug_adapter}")+ })?;+ let schema_path = schema_path.as_std_path();+ fs.create_dir(&output_dir.join(parent)).await?;+ copy_recursive(+ fs.as_ref(),+ &extension_path.join(schema_path),+ &output_dir.join(schema_path),+ CopyOptions {+ overwrite: true,+ ignore_if_exists: false,+ },+ )+ .await+ .with_context(|| {+ format!(+ "failed to copy debug adapter schema '{}'",+ schema_path.display(),+ )+ })+ }+ },+ ))+ .await?;}if let Some(snippets) = manifest.snippets.as_ref() {- for snippets_path in snippets.paths() {- let parent = snippets_path.parent();- if let Some(parent) = parent.filter(|p| p.components().next().is_some()) {- fs::create_dir_all(output_dir.join(parent))?;+ futures::future::try_join_all(snippets.paths().map(|snippets_path| {+ let fs = fs.clone();+ async move {+ let parent = snippets_path.parent();+ if let Some(parent) = parent.filter(|p| p.components().next().is_some()) {+ fs.create_dir(&output_dir.join(parent)).await?;+ }+ copy_recursive(+ fs.as_ref(),+ &extension_path.join(&snippets_path),+ &output_dir.join(&snippets_path),+ CopyOptions {+ overwrite: true,+ ignore_if_exists: false,+ },+ )+ .await+ .with_context(|| {+ format!("failed to copy snippets from '{}'", snippets_path.display())+ })}- copy_recursive(- fs.as_ref(),- &extension_path.join(&snippets_path),- &output_dir.join(&snippets_path),- CopyOptions {- overwrite: true,- ignore_if_exists: false,- },- )- .await- .with_context(|| {- format!("failed to copy snippets from '{}'", snippets_path.display())- })?;- }+ }))+ .await?;}Ok(())@@ -486,6 +570,40 @@ async fn test_snippets(Ok(())}+async fn test_debug_adapter_schemas(+ manifest: &ExtensionManifest,+ extension_path: &Path,+ fs: Arc<dyn Fs>,+) -> Result<()> {+ futures::future::try_join_all(manifest.debug_adapters.iter().map(+ |(debug_adapter_name, meta)| {+ let fs = fs.clone();+ async move {+ let debug_adapter_schema_path =+ extension_path.join(build_debug_adapter_schema_path(debug_adapter_name, meta)?);++ let debug_adapter_schema =+ fs.load(&debug_adapter_schema_path).await.with_context(|| {+ anyhow::anyhow!(+ "failed to read debug adapter schema for \+ `{debug_adapter_name}` from `{debug_adapter_schema_path:?}`"+ )+ })?;+ _ = serde_json::Value::from_str(&debug_adapter_schema).with_context(|| {+ anyhow::anyhow!(+ "Debug adapter schema for `{debug_adapter_name}`\+ (path: `{debug_adapter_schema_path:?}`) is not a valid JSON"+ )+ })?;++ Ok(())+ }+ },+ ))+ .await+ .map(|_| ())+}+#[cfg(test)]mod tests {use cloud_api_types::ExtensionProvides;diff --git a/crates/extension_host/benches/extension_compilation_benchmark.rs b/crates/extension_host/benches/extension_compilation_benchmark.rsindex b6dade97183..e8abdffbd86 100644--- a/crates/extension_host/benches/extension_compilation_benchmark.rs+++ b/crates/extension_host/benches/extension_compilation_benchmark.rs@@ -4,7 +4,7 @@ use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_mause extension::{ExtensionCapability, ExtensionHostProxy, ExtensionLibraryKind, ExtensionManifest,LanguageServerManifestEntry, LibManifestEntry, SchemaVersion,- extension_builder::{CompileExtensionOptions, ExtensionBuilder},+ extension_builder::{CompilationConcurrency, CompileExtensionOptions, ExtensionBuilder},};use extension_host::wasm_host::WasmHost;use fs::{Fs, RealFs};@@ -76,7 +76,10 @@ fn wasm_bytes(cx: &TestAppContext, manifest: &mut ExtensionManifest, fs: Arc<dyn.block_on(extension_builder.compile_extension(&path,manifest,- CompileExtensionOptions { release: true },+ CompileExtensionOptions {+ release: true,+ max_concurrency: CompilationConcurrency::Unbounded,+ },fs,)).unwrap();