Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
use std::{
hash::{Hash, Hasher},
path::{Path, PathBuf},
sync::Arc,
time::{Duration, SystemTime},
};
use anyhow::{Context as _, Result};
use client::{TypedEnvelope, proto};
use collections::{BTreeMap, FxHasher, HashSet};
use extension::{
Event, Extension, ExtensionDebugAdapterProviderProxy, ExtensionEvents, ExtensionHostProxy,
ExtensionLanguageProxy, ExtensionLanguageServerProxy, ExtensionManifest,
};
use fs::{Fs, RemoveOptions, RenameOptions};
use futures::{
StreamExt as _,
future::{FutureExt as _, join_all},
};
use gpui::{App, AppContext as _, AsyncApp, Context, Entity, Task, WeakEntity};
use http_client::HttpClient;
use language::{LanguageConfig, LanguageName, LanguageQueries, LoadedLanguage};
use lsp::LanguageServerName;
use node_runtime::NodeRuntime;
use util::ResultExt as _;
use crate::wasm_host::{WasmExtension, WasmHost};
#[derive(Clone, Debug)]
pub struct ExtensionVersion {
pub id: String,
pub version: String,
pub dev: bool,
pub content_fingerprint: Option<u64>,
}
pub(crate) const STALE_UPLOAD_TTL: Duration =
Duration::from_secs(crate::REMOTE_SYNC_TIMEOUT.as_secs() * 3);
pub struct HeadlessExtensionStore {
pub fs: Arc<dyn Fs>,
pub extension_dir: PathBuf,
pub proxy: Arc<ExtensionHostProxy>,
pub wasm_host: Arc<WasmHost>,
pub(crate) loaded_extensions: BTreeMap<Arc<str>, LoadedExtension>,
failed_removals: HashSet<Arc<str>>,
operation_lock: Arc<futures::lock::Mutex<()>>,
_stale_uploads_sweep: Task<()>,
}
#[derive(Clone)]
pub(crate) struct LoadedExtension {
pub version: Arc<str>,
pub languages: Vec<(LanguageName, LanguageConfig)>,
pub language_servers: Vec<(LanguageServerName, LanguageName)>,
pub debug_adapters: Vec<(Arc<str>, PathBuf)>,
pub debug_locators: Vec<Arc<str>>,
pub wasm_extension: Option<Arc<dyn Extension>>,
pub content_fingerprint: Option<u64>,
}
impl HeadlessExtensionStore {
pub fn new(
fs: Arc<dyn Fs>,
http_client: Arc<dyn HttpClient>,
extension_dir: PathBuf,
extension_host_proxy: Arc<ExtensionHostProxy>,
node_runtime: NodeRuntime,
cx: &mut App,
) -> Entity<Self> {
cx.new(|cx| {
let stale_uploads_sweep = cx.spawn({
let fs = fs.clone();
async move |_, cx| {
loop {
remove_stale_uploads(&fs, paths::remote_extensions_uploads_dir()).await;
cx.background_executor().timer(STALE_UPLOAD_TTL).await;
}
}
});
Self {
fs: fs.clone(),
wasm_host: WasmHost::new(
fs.clone(),
http_client.clone(),
node_runtime,
extension_host_proxy.clone(),
extension_dir.join("work"),
cx,
),
extension_dir,
proxy: extension_host_proxy,
loaded_extensions: BTreeMap::default(),
failed_removals: HashSet::default(),
operation_lock: Arc::default(),
_stale_uploads_sweep: stale_uploads_sweep,
}
})
}
pub fn sync_extensions(
&mut self,
extensions: Vec<ExtensionVersion>,
cx: &Context<Self>,
) -> Task<Result<Vec<ExtensionVersion>>> {
let operation_lock = self.operation_lock.clone();
cx.spawn(async move |store, cx| {
let _operation_guard = operation_lock.lock().await;
let (to_remove, to_load) = store.update(cx, |store, _cx| {
let on_client = HashSet::from_iter(extensions.iter().map(|e| e.id.as_str()));
store
.failed_removals
.retain(|id| !on_client.contains(id.as_ref()));
let mut to_remove = store
.loaded_extensions
.keys()
.filter(|id| !on_client.contains(id.as_ref()))
.cloned()
.collect::<Vec<Arc<str>>>();
to_remove.extend(store.failed_removals.drain());
let to_load = extensions
.into_iter()
.filter(|extension| {
let is_new = store
.loaded_extensions
.get(extension.id.as_str())
.is_none_or(|loaded| {
loaded.version.as_ref() != extension.version.as_str()
});
extension.dev || is_new
})
.collect::<Vec<ExtensionVersion>>();
(to_remove, to_load)
})?;
let mut extensions_changed = false;
let result = async {
let mut missing = Vec::new();
for extension_id in to_remove {
log::info!("removing extension: {extension_id}");
let (was_loaded, removal) = store.update(cx, |store, cx| {
(
store.loaded_extensions.contains_key(&extension_id),
store.uninstall_extension(&extension_id, cx),
)
})?;
if was_loaded {
extensions_changed = true;
}
if let Err(error) = removal.await {
log::error!("failed to remove extension: {extension_id}, {error:#}");
store.update(cx, |store, _cx| {
store.failed_removals.insert(extension_id.clone());
})?;
}
}
for extension in to_load {
match Self::load_extension(store.clone(), &extension, cx).await {
Ok(changed) => {
if changed {
extensions_changed = true;
}
if extension.dev
&& !Self::is_loaded_content_up_to_date(&store, &extension, cx)?
{
missing.push(extension)
}
}
Err(error) => {
log::info!("failed to load extension: {}, {:#}", extension.id, error);
missing.push(extension)
}
}
}
anyhow::Ok(missing)
}
.await;
if extensions_changed {
store.update(cx, |_, cx| notify_extensions_changed(cx)).ok();
}
result
})
}
fn is_loaded_content_up_to_date(
store: &WeakEntity<Self>,
extension: &ExtensionVersion,
cx: &AsyncApp,
) -> Result<bool> {
let loaded_fingerprint = store.read_with(cx, |store, _cx| {
store
.loaded_extensions
.get(extension.id.as_str())
.and_then(|loaded| loaded.content_fingerprint)
})?;
Ok(extension.content_fingerprint.is_some()
&& extension.content_fingerprint == loaded_fingerprint)
}
async fn load_extension(
store: WeakEntity<Self>,
extension: &ExtensionVersion,
cx: &mut AsyncApp,
) -> Result<bool> {
let (fs, wasm_host, extension_dir, loaded_fingerprint) =
store.read_with(cx, |store, _cx| {
(
store.fs.clone(),
store.wasm_host.clone(),
store.extension_dir.join(&extension.id),
store
.loaded_extensions
.get(extension.id.as_str())
.filter(|loaded| loaded.version.as_ref() == extension.version.as_str())
.and_then(|loaded| loaded.content_fingerprint),
)
})?;
let content_fingerprint = if extension.dev {
fingerprint_directory(&fs, &extension_dir, cx).await
} else {
None
};
if content_fingerprint.is_some() && content_fingerprint == loaded_fingerprint {
return Ok(false);
}
let loaded = Self::prepare_extension(
fs,
wasm_host,
extension_dir.clone(),
extension_dir,
extension,
content_fingerprint,
cx,
)
.await?;
let removal_tasks = store.update(cx, |store, cx| {
store.commit_extension(extension.id.as_str().into(), Some(loaded), cx)
})?;
for removal in join_all(removal_tasks).await {
removal.log_err();
}
Ok(true)
}
async fn prepare_extension(
fs: Arc<dyn Fs>,
wasm_host: Arc<WasmHost>,
load_dir: PathBuf,
installed_dir: PathBuf,
extension: &ExtensionVersion,
content_fingerprint: Option<u64>,
cx: &mut AsyncApp,
) -> Result<LoadedExtension> {
let manifest = Arc::new(ExtensionManifest::load(fs.clone(), &load_dir).await?);
debug_assert!(!manifest.languages.is_empty() || manifest.allow_remote_load());
anyhow::ensure!(
manifest.version.as_ref() == extension.version.as_str(),
"mismatched versions: ({}) != ({})",
manifest.version,
extension.version,
);
let mut languages = Vec::new();
for language_path in &manifest.languages {
let config_path = load_dir.join(language_path).join(LanguageConfig::FILE_NAME);
let config = fs.load(&config_path).await?;
let mut config = LanguageConfig::from_toml(&config)?;
config.grammar = None;
languages.push((config.name.clone(), config));
}
let mut language_servers = Vec::new();
let mut debug_adapters = Vec::new();
let mut debug_locators = Vec::new();
let mut wasm_extension: Option<Arc<dyn Extension>> = None;
if manifest.allow_remote_load() {
wasm_extension = Some(Arc::new(
WasmExtension::load(&load_dir, &manifest, wasm_host, cx).await?,
));
for (language_server_id, language_server_config) in &manifest.language_servers {
for language in language_server_config.languages() {
language_servers.push((language_server_id.clone(), language));
}
}
for (debug_adapter, meta) in &manifest.debug_adapters {
let schema_path = extension::build_debug_adapter_schema_path(debug_adapter, meta)?;
debug_adapters.push((debug_adapter.clone(), installed_dir.join(schema_path)));
}
debug_locators = manifest.debug_locators.keys().cloned().collect();
}
Ok(LoadedExtension {
version: extension.version.as_str().into(),
languages,
language_servers,
debug_adapters,
debug_locators,
wasm_extension,
content_fingerprint,
})
}
pub(crate) fn commit_extension(
&mut self,
extension_id: Arc<str>,
loaded: Option<LoadedExtension>,
cx: &mut App,
) -> Vec<Task<Result<()>>> {
let previous = match loaded {
Some(loaded) => self.loaded_extensions.insert(extension_id.clone(), loaded),
None => self.loaded_extensions.remove(&extension_id),
};
let current = self.loaded_extensions.get(&extension_id).cloned();
let mut removal_tasks = Vec::new();
let mut languages_to_remove = Vec::new();
let mut registrations = Vec::new();
if let Some(previous) = previous {
for (language, _) in &previous.languages {
if current.as_ref().is_some_and(|current| {
current.languages.iter().any(|(name, _)| name == language)
}) {
continue;
}
match self.surviving_language_config(language) {
Some(config) => registrations.push(language_registration_from_config(config)),
None => languages_to_remove.push(language.clone()),
}
}
for (server_name, language) in &previous.language_servers {
removal_tasks.push(self.proxy.remove_language_server(language, server_name, cx));
let in_current = current.as_ref().is_some_and(|current| {
current
.language_servers
.iter()
.any(|(name, language_name)| {
name == server_name && language_name == language
})
});
if in_current {
continue;
}
if let Some(extension) = self.surviving_language_server(server_name, language) {
self.proxy.register_language_server(
extension,
server_name.clone(),
language.clone(),
);
}
}
for (adapter_name, _) in &previous.debug_adapters {
if current.as_ref().is_some_and(|current| {
current
.debug_adapters
.iter()
.any(|(name, _)| name == adapter_name)
}) {
continue;
}
match self.surviving_debug_adapter(adapter_name) {
Some((schema_path, extension)) => {
self.proxy.register_debug_adapter(
extension,
adapter_name.clone(),
&schema_path,
);
}
None => self.proxy.unregister_debug_adapter(adapter_name.clone()),
}
}
for locator_name in &previous.debug_locators {
if current.as_ref().is_some_and(|current| {
current
.debug_locators
.iter()
.any(|name| name == locator_name)
}) {
continue;
}
match self.surviving_debug_locator(locator_name) {
Some(extension) => {
self.proxy
.register_debug_locator(extension, locator_name.clone());
}
None => self.proxy.unregister_debug_locator(locator_name.clone()),
}
}
}
if let Some(current) = ¤t {
for (_, config) in ¤t.languages {
registrations.push(language_registration_from_config(config.clone()));
}
if let Some(wasm_extension) = ¤t.wasm_extension {
for (server_name, language) in ¤t.language_servers {
self.proxy.register_language_server(
wasm_extension.clone(),
server_name.clone(),
language.clone(),
);
log::info!("Loaded language server: {server_name}");
}
for (adapter_name, schema_path) in ¤t.debug_adapters {
self.proxy.register_debug_adapter(
wasm_extension.clone(),
adapter_name.clone(),
schema_path,
);
log::info!("Loaded debug adapter: {adapter_name}");
}
for locator_name in ¤t.debug_locators {
self.proxy
.register_debug_locator(wasm_extension.clone(), locator_name.clone());
log::info!("Loaded debug locator: {locator_name}");
}
}
}
self.proxy
.update_languages(&languages_to_remove, &[], registrations);
removal_tasks
}
fn surviving_language_config(&self, language: &LanguageName) -> Option<LanguageConfig> {
self.loaded_extensions.values().find_map(|extension| {
extension
.languages
.iter()
.find(|(name, _)| name == language)
.map(|(_, config)| config.clone())
})
}
fn surviving_language_server(
&self,
server_name: &LanguageServerName,
language: &LanguageName,
) -> Option<Arc<dyn Extension>> {
self.loaded_extensions.values().find_map(|extension| {
let wasm_extension = extension.wasm_extension.clone()?;
extension
.language_servers
.iter()
.any(|(name, language_name)| name == server_name && language_name == language)
.then_some(wasm_extension)
})
}
fn surviving_debug_adapter(
&self,
adapter_name: &Arc<str>,
) -> Option<(PathBuf, Arc<dyn Extension>)> {
self.loaded_extensions.values().find_map(|extension| {
let wasm_extension = extension.wasm_extension.clone()?;
extension
.debug_adapters
.iter()
.find(|(name, _)| name == adapter_name)
.map(|(_, schema_path)| (schema_path.clone(), wasm_extension))
})
}
fn surviving_debug_locator(&self, locator_name: &Arc<str>) -> Option<Arc<dyn Extension>> {
self.loaded_extensions.values().find_map(|extension| {
let wasm_extension = extension.wasm_extension.clone()?;
extension
.debug_locators
.iter()
.any(|name| name == locator_name)
.then_some(wasm_extension)
})
}
fn uninstall_extension(
&mut self,
extension_id: &Arc<str>,
cx: &mut Context<Self>,
) -> Task<Result<()>> {
let removal_tasks = self.commit_extension(extension_id.clone(), None, cx);
let path = self.extension_dir.join(extension_id.as_ref());
let fs = self.fs.clone();
cx.spawn(async move |_, _cx| {
for removal in join_all(removal_tasks).await {
removal.log_err();
}
fs.remove_dir(
&path,
RemoveOptions {
recursive: true,
ignore_if_not_exists: true,
},
)
.await
.with_context(|| format!("Removing directory {path:?}"))
})
}
pub fn install_extension(
&mut self,
extension: ExtensionVersion,
tmp_path: PathBuf,
cx: &mut Context<Self>,
) -> Task<Result<()>> {
let path = self.extension_dir.join(&extension.id);
let fs = self.fs.clone();
let wasm_host = self.wasm_host.clone();
let operation_lock = self.operation_lock.clone();
cx.spawn(async move |store, cx| {
let _operation_guard = operation_lock.lock().await;
let loaded_fingerprint = store.read_with(cx, |store, _cx| {
store
.loaded_extensions
.get(extension.id.as_str())
.filter(|loaded| loaded.version.as_ref() == extension.version.as_str())
.and_then(|loaded| loaded.content_fingerprint)
})?;
let content_fingerprint = if extension.dev {
fingerprint_directory(&fs, &tmp_path, cx).await
} else {
None
};
if content_fingerprint.is_some() && content_fingerprint == loaded_fingerprint {
fs.remove_dir(
&tmp_path,
RemoveOptions {
recursive: true,
ignore_if_not_exists: true,
},
)
.await
.log_err();
return Ok(());
}
let mut installed_dir_touched = false;
let result = async {
let loaded = Self::prepare_extension(
fs.clone(),
wasm_host,
tmp_path.clone(),
path.clone(),
&extension,
content_fingerprint,
cx,
)
.await?;
installed_dir_touched = true;
fs.remove_dir(
&path,
RemoveOptions {
recursive: true,
ignore_if_not_exists: true,
},
)
.await
.with_context(|| format!("Removing existing directory {path:?}"))?;
fs.rename(&tmp_path, &path, RenameOptions::default())
.await
.with_context(|| format!("Failed to rename {tmp_path:?} to {path:?}"))?;
let removal_tasks = store.update(cx, |store, cx| {
store.commit_extension(extension.id.as_str().into(), Some(loaded), cx)
})?;
installed_dir_touched = false;
for removal in join_all(removal_tasks).await {
removal.log_err();
}
store.update(cx, |_, cx| notify_extensions_changed(cx))?;
anyhow::Ok(())
}
.await;
if result.is_err() {
fs.remove_dir(
&tmp_path,
RemoveOptions {
recursive: true,
ignore_if_not_exists: true,
},
)
.await
.log_err();
if installed_dir_touched
&& let Ok(removal_tasks) = store.update(cx, |store, cx| {
store
.failed_removals
.insert(Arc::from(extension.id.as_str()));
store.commit_extension(extension.id.as_str().into(), None, cx)
})
{
for removal in join_all(removal_tasks).await {
removal.log_err();
}
store.update(cx, |_, cx| notify_extensions_changed(cx)).ok();
}
}
result
})
}
pub async fn handle_sync_extensions(
extension_store: Entity<HeadlessExtensionStore>,
envelope: TypedEnvelope<proto::SyncExtensions>,
mut cx: AsyncApp,
) -> Result<proto::SyncExtensionsResponse> {
let requested_extensions =
envelope
.payload
.extensions
.into_iter()
.map(|p| ExtensionVersion {
id: p.id,
version: p.version,
dev: p.dev,
content_fingerprint: p.content_fingerprint,
});
let missing_extensions = extension_store
.update(&mut cx, |extension_store, cx| {
extension_store.sync_extensions(requested_extensions.collect(), cx)
})
.await?;
let fs = extension_store.read_with(&cx, |extension_store, _cx| extension_store.fs.clone());
remove_stale_uploads(&fs, paths::remote_extensions_uploads_dir()).await;
Ok(proto::SyncExtensionsResponse {
missing_extensions: missing_extensions
.into_iter()
.map(|e| proto::Extension {
id: e.id,
version: e.version,
dev: e.dev,
content_fingerprint: e.content_fingerprint,
})
.collect(),
tmp_dir: paths::remote_extensions_uploads_dir()
.to_string_lossy()
.to_string(),
})
}
pub async fn handle_install_extension(
extensions: Entity<HeadlessExtensionStore>,
envelope: TypedEnvelope<proto::InstallExtension>,
mut cx: AsyncApp,
) -> Result<proto::Ack> {
let extension = envelope
.payload
.extension
.context("Invalid InstallExtension request")?;
extensions
.update(&mut cx, |extensions, cx| {
extensions.install_extension(
ExtensionVersion {
id: extension.id,
version: extension.version,
dev: extension.dev,
content_fingerprint: extension.content_fingerprint,
},
PathBuf::from(envelope.payload.tmp_dir),
cx,
)
})
.await?;
Ok(proto::Ack {})
}
}
pub(crate) async fn remove_stale_uploads(fs: &Arc<dyn Fs>, uploads_dir: &Path) {
let Ok(mut entries) = fs.read_dir(uploads_dir).await else {
return;
};
let now = SystemTime::now();
while let Some(entry) = entries.next().await {
let Some(path) = entry.log_err() else {
continue;
};
let Ok(Some(metadata)) = fs.metadata(&path).await else {
continue;
};
let is_stale = match now.duration_since(metadata.mtime.timestamp_for_user()) {
Ok(age) => age > STALE_UPLOAD_TTL,
Err(error) => error.duration() > STALE_UPLOAD_TTL,
};
if !is_stale {
continue;
}
if metadata.is_dir {
fs.remove_dir(
&path,
RemoveOptions {
recursive: true,
ignore_if_not_exists: true,
},
)
.await
.log_err();
} else {
fs.remove_file(
&path,
RemoveOptions {
recursive: false,
ignore_if_not_exists: true,
},
)
.await
.log_err();
}
}
}
async fn fingerprint_directory(fs: &Arc<dyn Fs>, path: &Path, cx: &AsyncApp) -> Option<u64> {
let fs = fs.clone();
let path = path.to_path_buf();
cx.background_executor()
.spawn(async move { hash_directory_contents(&fs, &path).await })
.await
.log_err()
}
pub(crate) async fn hash_directory_contents(fs: &Arc<dyn Fs>, root: &Path) -> Result<u64> {
let mut hasher = FxHasher::default();
let mut directories = vec![root.to_path_buf()];
while let Some(directory) = directories.pop() {
let mut paths = Vec::new();
let mut entries = fs.read_dir(&directory).await?;
while let Some(entry) = entries.next().await {
paths.push(entry?);
}
paths.sort();
for path in paths {
let metadata = fs
.metadata(&path)
.await?
.with_context(|| format!("missing metadata for {path:?}"))?;
for component in path.strip_prefix(root)?.components() {
component.as_os_str().to_string_lossy().hash(&mut hasher);
}
if metadata.is_symlink {
2_u8.hash(&mut hasher);
fs.read_link(&path)
.await?
.to_string_lossy()
.hash(&mut hasher);
} else if metadata.is_dir {
1_u8.hash(&mut hasher);
directories.push(path);
} else {
0_u8.hash(&mut hasher);
fs.load_bytes(&path).await?.hash(&mut hasher);
}
}
}
Ok(hasher.finish())
}
fn notify_extensions_changed(cx: &mut App) {
if let Some(events) = ExtensionEvents::try_global(cx) {
events.update(cx, |events, cx| {
events.emit(Event::ExtensionsInstalledChanged, cx)
});
}
}
fn language_registration_from_config(config: LanguageConfig) -> language::LanguageRegistration {
language::LanguageRegistration {
name: config.name.clone(),
grammar_name: None,
matcher: config.matcher.clone(),
hidden: config.hidden,
load: Arc::new(move || {
let config = config.clone();
async move {
Ok(LoadedLanguage {
config,
queries: LanguageQueries::default(),
context_provider: None,
toolchain_provider: None,
manifest_name: None,
})
}
.boxed()
}),
}
}