Skip to content

web search done right

remove unused feature flags from tantivy

Mikkel Denker28d2eff2c2e6
diff --git a/crates/tantivy/Cargo.toml b/crates/tantivy/Cargo.toml
index af0bdadb..cc9f40cb 100644
--- a/crates/tantivy/Cargo.toml
+++ b/crates/tantivy/Cargo.toml
@@ -97,8 +97,6 @@ lz4-compression = ["lz4_flex"]
failpoints = ["fail", "fail/failpoints"]
-quickwit = ["futures-util"]
-
# Following the "fail" crate best practises, we isolate
# tests that define specific behavior in fail check points
# in a different binary.
diff --git a/crates/tantivy/src/core/executor.rs b/crates/tantivy/src/core/executor.rs
index 19b1e42f..28f0fdf3 100644
--- a/crates/tantivy/src/core/executor.rs
+++ b/crates/tantivy/src/core/executor.rs
@@ -1,8 +1,5 @@
use std::sync::Arc;
-#[cfg(feature = "quickwit")]
-use futures_util::{future::Either, FutureExt};
-
use crate::TantivyError;
/// Executor makes it possible to run tasks in single thread or
@@ -15,13 +12,6 @@ pub enum Executor {
ThreadPool(Arc<rayon::ThreadPool>),
}
-#[cfg(feature = "quickwit")]
-impl From<Arc<rayon::ThreadPool>> for Executor {
- fn from(thread_pool: Arc<rayon::ThreadPool>) -> Self {
- Executor::ThreadPool(thread_pool)
- }
-}
-
impl Executor {
/// Creates an Executor that performs all task in the caller thread.
pub fn single_thread() -> Executor {
@@ -97,32 +87,6 @@ impl Executor {
}
}
}
-
- /// Spawn a task on the pool, returning a future completing on task success.
- ///
- /// If the task panic, returns `Err(())`.
- #[cfg(feature = "quickwit")]
- pub fn spawn_blocking<T: Send + 'static>(
- &self,
- cpu_intensive_task: impl FnOnce() -> T + Send + 'static,
- ) -> impl std::future::Future<Output = Result<T, ()>> {
- match self {
- Executor::SingleThread => Either::Left(std::future::ready(Ok(cpu_intensive_task()))),
- Executor::ThreadPool(pool) => {
- let (sender, receiver) = oneshot::channel();
- pool.spawn(|| {
- if sender.is_closed() {
- return;
- }
- let task_result = cpu_intensive_task();
- let _ = sender.send(task_result);
- });
-
- let res = receiver.map(|res| res.map_err(|_| ()));
- Either::Right(res)
- }
- }
- }
}
#[cfg(test)]
@@ -178,62 +142,4 @@ mod tests {
assert_eq!(result[i], i * 2);
}
}
-
- #[cfg(feature = "quickwit")]
- #[test]
- fn test_cancel_cpu_intensive_tasks() {
- use std::sync::atomic::{AtomicU64, Ordering};
- use std::sync::Arc;
-
- let counter: Arc<AtomicU64> = Default::default();
-
- let other_counter: Arc<AtomicU64> = Default::default();
-
- let mut futures = Vec::new();
- let mut other_futures = Vec::new();
-
- let (tx, rx) = crossbeam_channel::bounded::<()>(0);
- let rx = Arc::new(rx);
- let executor = Executor::multi_thread(3, "search-test").unwrap();
- for _ in 0..1000 {
- let counter_clone: Arc<AtomicU64> = counter.clone();
- let other_counter_clone: Arc<AtomicU64> = other_counter.clone();
-
- let rx_clone = rx.clone();
- let rx_clone2 = rx.clone();
- let fut = executor.spawn_blocking(move || {
- counter_clone.fetch_add(1, Ordering::SeqCst);
- let _ = rx_clone.recv();
- });
- futures.push(fut);
- let other_fut = executor.spawn_blocking(move || {
- other_counter_clone.fetch_add(1, Ordering::SeqCst);
- let _ = rx_clone2.recv();
- });
- other_futures.push(other_fut);
- }
-
- // We execute 100 futures.
- for _ in 0..100 {
- tx.send(()).unwrap();
- }
-
- let counter_val = counter.load(Ordering::SeqCst);
- let other_counter_val = other_counter.load(Ordering::SeqCst);
- assert!(counter_val >= 30);
- assert!(other_counter_val >= 30);
-
- drop(other_futures);
-
- // We execute 100 futures.
- for _ in 0..100 {
- tx.send(()).unwrap();
- }
-
- let counter_val2 = counter.load(Ordering::SeqCst);
- assert!(counter_val2 >= counter_val + 100 - 6);
-
- let other_counter_val2 = other_counter.load(Ordering::SeqCst);
- assert!(other_counter_val2 <= other_counter_val + 6);
- }
}
diff --git a/crates/tantivy/src/core/searcher.rs b/crates/tantivy/src/core/searcher.rs
index 3106f87f..ec54fb34 100644
--- a/crates/tantivy/src/core/searcher.rs
+++ b/crates/tantivy/src/core/searcher.rs
@@ -103,17 +103,6 @@ impl Searcher {
cache_stats
}
- /// Fetches a document in an asynchronous manner.
- #[cfg(feature = "quickwit")]
- pub async fn doc_async<D: DocumentDeserialize>(
- &self,
- doc_address: DocAddress,
- ) -> crate::Result<D> {
- let executor = self.inner.index.search_executor();
- let store_reader = &self.inner.store_readers[doc_address.segment_ord as usize];
- store_reader.get_async(doc_address.doc_id, executor).await
- }
-
/// Access the schema associated with the index of this searcher.
pub fn schema(&self) -> &Schema {
&self.inner.schema
diff --git a/crates/tantivy/src/fastfield/readers.rs b/crates/tantivy/src/fastfield/readers.rs
index cc65b5d7..401de1d6 100644
--- a/crates/tantivy/src/fastfield/readers.rs
+++ b/crates/tantivy/src/fastfield/readers.rs
@@ -31,12 +31,7 @@ impl FastFieldReaders {
}
fn resolve_field(&self, column_name: &str) -> crate::Result<Option<String>> {
- let default_field_opt: Option<Field> = if cfg!(feature = "quickwit") {
- self.schema.get_field("_dynamic").ok()
- } else {
- None
- };
- self.resolve_column_name_given_default_field(column_name, default_field_opt)
+ self.resolve_column_name_given_default_field(column_name, None)
}
pub(crate) fn space_usage(&self, schema: &Schema) -> io::Result<PerFieldSpaceUsage> {
diff --git a/crates/tantivy/src/store/reader.rs b/crates/tantivy/src/store/reader.rs
index 5bbdb591..0becb78c 100644
--- a/crates/tantivy/src/store/reader.rs
+++ b/crates/tantivy/src/store/reader.rs
@@ -18,8 +18,6 @@ use crate::schema::document::{BinaryDocumentDeserializer, DocumentDeserialize};
use crate::space_usage::StoreSpaceUsage;
use crate::store::index::Checkpoint;
use crate::DocId;
-#[cfg(feature = "quickwit")]
-use crate::Executor;
pub(crate) const DOCSTORE_CACHE_CAPACITY: usize = 100;
@@ -336,67 +334,6 @@ fn block_read_index(block: &[u8], doc_pos: u32) -> crate::Result<Range<usize>> {
Ok(start_offset..end_offset)
}
-#[cfg(feature = "quickwit")]
-impl StoreReader {
- /// Advanced API.
- ///
- /// In most cases use [`get_async`](Self::get_async)
- ///
- /// Loads and decompresses a block asynchronously.
- async fn read_block_async(
- &self,
- checkpoint: &Checkpoint,
- executor: &Executor,
- ) -> io::Result<Block> {
- let cache_key = checkpoint.byte_range.start;
- if let Some(block) = self.cache.get_from_cache(checkpoint.byte_range.start) {
- return Ok(block);
- }
-
- let compressed_block = self
- .data
- .slice(checkpoint.byte_range.clone())
- .read_bytes_async()
- .await?;
-
- let decompressor = self.decompressor;
- let maybe_decompressed_block = executor
- .spawn_blocking(move || decompressor.decompress(compressed_block.as_ref()))
- .await
- .expect("decompression panicked");
- let decompressed_block = OwnedBytes::new(maybe_decompressed_block?);
-
- self.cache
- .put_into_cache(cache_key, decompressed_block.clone());
-
- Ok(decompressed_block)
- }
-
- /// Reads raw bytes of a given document asynchronously.
- pub async fn get_document_bytes_async(
- &self,
- doc_id: DocId,
- executor: &Executor,
- ) -> crate::Result<OwnedBytes> {
- let checkpoint = self.block_checkpoint(doc_id)?;
- let block = self.read_block_async(&checkpoint, executor).await?;
- Self::get_document_bytes_from_block(block, doc_id, &checkpoint)
- }
-
- /// Fetches a document asynchronously. Async version of [`get`](Self::get).
- pub async fn get_async<D: DocumentDeserialize>(
- &self,
- doc_id: DocId,
- executor: &Executor,
- ) -> crate::Result<D> {
- let mut doc_bytes = self.get_document_bytes_async(doc_id, executor).await?;
-
- let deserializer = BinaryDocumentDeserializer::from_reader(&mut doc_bytes)
- .map_err(crate::TantivyError::from)?;
- D::deserialize(deserializer).map_err(crate::TantivyError::from)
- }
-}
-
#[cfg(test)]
mod tests {
use std::path::Path;
diff --git a/crates/tantivy/src/termdict/mod.rs b/crates/tantivy/src/termdict/mod.rs
index 84334b7a..a3be740d 100644
--- a/crates/tantivy/src/termdict/mod.rs
+++ b/crates/tantivy/src/termdict/mod.rs
@@ -47,12 +47,8 @@ enum DictionaryType {
SSTable = 2,
}
-#[cfg(not(feature = "quickwit"))]
const CURRENT_TYPE: DictionaryType = DictionaryType::Fst;
-#[cfg(feature = "quickwit")]
-const CURRENT_TYPE: DictionaryType = DictionaryType::SSTable;
-
// TODO in the future this should become an enum of supported dictionaries
/// A TermDictionary wrapping either an FST based dictionary or a SSTable based one.
pub struct TermDictionary(InnerTermDict);