web search done right
keep fastfield reader open across searches 52% of time seems to be spend on opening the fastfields
Mikkel Denker45d824537436
diff --git a/crates/core/benches/search_preindexed.rs b/crates/core/benches/search_preindexed.rsindex 3a3a93d4..d38c760b 100644--- a/crates/core/benches/search_preindexed.rs+++ b/crates/core/benches/search_preindexed.rs@@ -1,5 +1,6 @@use criterion::{criterion_group, criterion_main, Criterion};use optics::HostRankings;+use rand::seq::SliceRandom;use stract::{bangs::Bangs,config::{@@ -149,11 +150,13 @@ pub fn criterion_benchmark(c: &mut Criterion) {},};- let queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)+ let mut queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path).unwrap().all().unwrap();+ queries.shuffle(&mut rand::thread_rng());+let mut searcher = LocalSearcher::new(index);searcher.set_inbound_similarity(InboundSimilarity::open("data/centrality/inbound_similarity").unwrap(),diff --git a/crates/core/src/fastfield_reader.rs b/crates/core/src/fastfield_reader.rsindex acf67408..2b8a82fc 100644--- a/crates/core/src/fastfield_reader.rs+++ b/crates/core/src/fastfield_reader.rs@@ -98,6 +98,7 @@ impl From<FieldValue> for Option<u64> {}}+#[derive(Clone)]pub enum FieldReader {U64(Arc<dyn ColumnValues<u64>>),}diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rsindex 8ed8f36d..bca36f20 100644--- a/crates/core/src/inverted_index.rs+++ b/crates/core/src/inverted_index.rs@@ -163,6 +163,7 @@ pub struct InvertedIndex {reader: IndexReader,schema: Arc<Schema>,snippet_config: SnippetConfig,+ fastfield_reader: FastFieldReader,}impl InvertedIndex {@@ -223,6 +224,8 @@ impl InvertedIndex {let reader: IndexReader = tantivy_index.reader_builder().try_into()?;+ let fastfield_reader = FastFieldReader::new(&reader.searcher());+Ok(InvertedIndex {writer: None,reader,@@ -230,9 +233,14 @@ impl InvertedIndex {path: path.as_ref().to_str().unwrap().to_string(),tantivy_index,snippet_config: SnippetConfig::default(),+ fastfield_reader,})}+ pub fn fastfield_reader(&self) -> FastFieldReader {+ self.fastfield_reader.clone()+ }+pub fn prepare_writer(&mut self) -> Result<()> {if self.writer.is_some() {return Ok(());@@ -262,10 +270,6 @@ impl InvertedIndex {.set_merge_policy(Box::new(merge_policy));}- pub fn fastfield_reader(&self, tv_searcher: &tantivy::Searcher) -> FastFieldReader {- FastFieldReader::new(tv_searcher)- }-pub fn tokenizers(&self) -> &TokenizerManager {self.tantivy_index.tokenizers()}@@ -295,6 +299,7 @@ impl InvertedIndex {.expect("writer has not been prepared").commit()?;self.reader.reload()?;+ self.fastfield_reader = FastFieldReader::new(&self.reader.searcher());Ok(())}@@ -354,7 +359,7 @@ impl InvertedIndex {pub fn local_search_ctx(&self) -> Ctx {let tv_searcher = self.tv_searcher();Ctx {- fastfield_reader: self.fastfield_reader(&tv_searcher),+ fastfield_reader: self.fastfield_reader.clone(),tv_searcher,}}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex f6d35b16..cc74e305 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -194,7 +194,7 @@ wherelet mut ranker = Ranker::new(aggregator,- guard.inverted_index().fastfield_reader(&ctx.tv_searcher),+ guard.inverted_index().fastfield_reader(),self.collector_config.clone(),);@@ -290,7 +290,7 @@ whereranker.collector(ctx.clone()),)?;- let fastfield_reader = guard.inverted_index().fastfield_reader(&ctx.tv_searcher);+ let fastfield_reader = guard.inverted_index().fastfield_reader();let ranking_websites = guard.inverted_index().retrieve_ranking_websites(ctx,