web search done right
bench based on queries from autosuggest. this will hopefully show us where the hot paths are when caches aren't hit
Mikkel Denker9bcaf054c3a3
diff --git a/.dockerignore b/.dockerignoredeleted file mode 100644index e8131628..00000000--- a/.dockerignore+++ /dev/null@@ -1,11 +0,0 @@-**/.DS_Store-**/flamegraph.svg-**/target-**/static/**/*.css-**/node_modules-**/package-lock.json-**/*.log-**/data/-**/data.tar.gz-**/testcases-.git/\ No newline at end of filediff --git a/.gitignore b/.gitignoreindex 0389b5ca..1e96e4da 100644--- a/.gitignore+++ b/.gitignore@@ -14,3 +14,4 @@ ltr_scores.json.venvproptest-regressions*.pending-snap+.ipynb_checkpointsdiff --git a/.ignore b/.ignoredeleted file mode 100644index 93dba0ca..00000000--- a/.ignore+++ /dev/null@@ -1 +0,0 @@-data/**/*diff --git a/crates/core/benches/search_preindexed.rs b/crates/core/benches/search_preindexed.rsindex 00ab8cca..3a3a93d4 100644--- a/crates/core/benches/search_preindexed.rs+++ b/crates/core/benches/search_preindexed.rs@@ -101,12 +101,8 @@ macro_rules! bench {.search(&SearchQuery {query: $query.to_string(),host_rankings: Some(HostRankings {- liked: vec![- "docs.rs".to_string(),- "ycombinator.com".to_string(),- "nih.gov".to_string(),- ],- disliked: vec!["pinterest.com".to_string()],+ liked: vec![],+ disliked: vec![],blocked: vec![],}),..Default::default()@@ -153,6 +149,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {},};+ let queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)+ .unwrap()+ .all()+ .unwrap();+let mut searcher = LocalSearcher::new(index);searcher.set_inbound_similarity(InboundSimilarity::open("data/centrality/inbound_similarity").unwrap(),@@ -168,16 +169,20 @@ pub fn criterion_benchmark(c: &mut Criterion) {let searcher: ApiSearcher<Searcher, LiveSearcher> =ApiSearcher::new(searcher, None, None, None, bangs, config);- for _ in 0..1000 {- bench!("the", searcher, c);- bench!("dtu", searcher, c);- bench!("the best", searcher, c);- bench!("the circle of life", searcher, c);- bench!("what", searcher, c);- bench!("a", searcher, c);- bench!("sun", searcher, c);- bench!("what a sun", searcher, c);+ for query in &queries {+ bench!(query, searcher, c);}++ // for _ in 0..1000 {+ // bench!("the", searcher, c);+ // bench!("dtu", searcher, c);+ // bench!("the best", searcher, c);+ // bench!("the circle of life", searcher, c);+ // bench!("what", searcher, c);+ // bench!("a", searcher, c);+ // bench!("sun", searcher, c);+ // bench!("what a sun", searcher, c);+ // }}criterion_group!(benches, criterion_benchmark);diff --git a/crates/core/src/autosuggest.rs b/crates/core/src/autosuggest.rsindex f5ce584b..b04520dd 100644--- a/crates/core/src/autosuggest.rs+++ b/crates/core/src/autosuggest.rs@@ -60,4 +60,8 @@ impl Autosuggest {.take(10).collect())}++ pub fn all(&self) -> Result<Vec<String>> {+ Ok(self.queries.into_stream().into_strs()?)+ }}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 3b1a5c1c..1016c076 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -32,7 +32,7 @@ pub mod inverted_index;pub mod mapreduce;mod api;-mod autosuggest;+pub mod autosuggest;pub mod bangs;mod bloom;mod collector;