Skip to content

web search done right

keep fastfield reader open across searches 52% of time seems to be spend on opening the fastfields

Mikkel Denker45d824537436
diff --git a/crates/core/benches/search_preindexed.rs b/crates/core/benches/search_preindexed.rs
index 3a3a93d4..d38c760b 100644
--- a/crates/core/benches/search_preindexed.rs
+++ b/crates/core/benches/search_preindexed.rs
@@ -1,5 +1,6 @@
use criterion::{criterion_group, criterion_main, Criterion};
use optics::HostRankings;
+use rand::seq::SliceRandom;
use stract::{
bangs::Bangs,
config::{
@@ -149,11 +150,13 @@ pub fn criterion_benchmark(c: &mut Criterion) {
},
};
- let queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)
+ let mut queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)
.unwrap()
.all()
.unwrap();
+ queries.shuffle(&mut rand::thread_rng());
+
let mut searcher = LocalSearcher::new(index);
searcher.set_inbound_similarity(
InboundSimilarity::open("data/centrality/inbound_similarity").unwrap(),
diff --git a/crates/core/src/fastfield_reader.rs b/crates/core/src/fastfield_reader.rs
index acf67408..2b8a82fc 100644
--- a/crates/core/src/fastfield_reader.rs
+++ b/crates/core/src/fastfield_reader.rs
@@ -98,6 +98,7 @@ impl From<FieldValue> for Option<u64> {
}
}
+#[derive(Clone)]
pub enum FieldReader {
U64(Arc<dyn ColumnValues<u64>>),
}
diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rs
index 8ed8f36d..bca36f20 100644
--- a/crates/core/src/inverted_index.rs
+++ b/crates/core/src/inverted_index.rs
@@ -163,6 +163,7 @@ pub struct InvertedIndex {
reader: IndexReader,
schema: Arc<Schema>,
snippet_config: SnippetConfig,
+ fastfield_reader: FastFieldReader,
}
impl InvertedIndex {
@@ -223,6 +224,8 @@ impl InvertedIndex {
let reader: IndexReader = tantivy_index.reader_builder().try_into()?;
+ let fastfield_reader = FastFieldReader::new(&reader.searcher());
+
Ok(InvertedIndex {
writer: None,
reader,
@@ -230,9 +233,14 @@ impl InvertedIndex {
path: path.as_ref().to_str().unwrap().to_string(),
tantivy_index,
snippet_config: SnippetConfig::default(),
+ fastfield_reader,
})
}
+ pub fn fastfield_reader(&self) -> FastFieldReader {
+ self.fastfield_reader.clone()
+ }
+
pub fn prepare_writer(&mut self) -> Result<()> {
if self.writer.is_some() {
return Ok(());
@@ -262,10 +270,6 @@ impl InvertedIndex {
.set_merge_policy(Box::new(merge_policy));
}
- pub fn fastfield_reader(&self, tv_searcher: &tantivy::Searcher) -> FastFieldReader {
- FastFieldReader::new(tv_searcher)
- }
-
pub fn tokenizers(&self) -> &TokenizerManager {
self.tantivy_index.tokenizers()
}
@@ -295,6 +299,7 @@ impl InvertedIndex {
.expect("writer has not been prepared")
.commit()?;
self.reader.reload()?;
+ self.fastfield_reader = FastFieldReader::new(&self.reader.searcher());
Ok(())
}
@@ -354,7 +359,7 @@ impl InvertedIndex {
pub fn local_search_ctx(&self) -> Ctx {
let tv_searcher = self.tv_searcher();
Ctx {
- fastfield_reader: self.fastfield_reader(&tv_searcher),
+ fastfield_reader: self.fastfield_reader.clone(),
tv_searcher,
}
}
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index f6d35b16..cc74e305 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -194,7 +194,7 @@ where
let mut ranker = Ranker::new(
aggregator,
- guard.inverted_index().fastfield_reader(&ctx.tv_searcher),
+ guard.inverted_index().fastfield_reader(),
self.collector_config.clone(),
);
@@ -290,7 +290,7 @@ where
ranker.collector(ctx.clone()),
)?;
- let fastfield_reader = guard.inverted_index().fastfield_reader(&ctx.tv_searcher);
+ let fastfield_reader = guard.inverted_index().fastfield_reader();
let ranking_websites = guard.inverted_index().retrieve_ranking_websites(
ctx,