Skip to content

web search done right

bench based on queries from autosuggest. this will hopefully show us where the hot paths are when caches aren't hit

Mikkel Denker9bcaf054c3a3
diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index e8131628..00000000
--- a/.dockerignore
+++ /dev/null
@@ -1,11 +0,0 @@
-**/.DS_Store
-**/flamegraph.svg
-**/target
-**/static/**/*.css
-**/node_modules
-**/package-lock.json
-**/*.log
-**/data/
-**/data.tar.gz
-**/testcases
-.git/
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 0389b5ca..1e96e4da 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ ltr_scores.json
.venv
proptest-regressions
*.pending-snap
+.ipynb_checkpoints
diff --git a/.ignore b/.ignore
deleted file mode 100644
index 93dba0ca..00000000
--- a/.ignore
+++ /dev/null
@@ -1 +0,0 @@
-data/**/*
diff --git a/crates/core/benches/search_preindexed.rs b/crates/core/benches/search_preindexed.rs
index 00ab8cca..3a3a93d4 100644
--- a/crates/core/benches/search_preindexed.rs
+++ b/crates/core/benches/search_preindexed.rs
@@ -101,12 +101,8 @@ macro_rules! bench {
.search(&SearchQuery {
query: $query.to_string(),
host_rankings: Some(HostRankings {
- liked: vec![
- "docs.rs".to_string(),
- "ycombinator.com".to_string(),
- "nih.gov".to_string(),
- ],
- disliked: vec!["pinterest.com".to_string()],
+ liked: vec![],
+ disliked: vec![],
blocked: vec![],
}),
..Default::default()
@@ -153,6 +149,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {
},
};
+ let queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)
+ .unwrap()
+ .all()
+ .unwrap();
+
let mut searcher = LocalSearcher::new(index);
searcher.set_inbound_similarity(
InboundSimilarity::open("data/centrality/inbound_similarity").unwrap(),
@@ -168,16 +169,20 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let searcher: ApiSearcher<Searcher, LiveSearcher> =
ApiSearcher::new(searcher, None, None, None, bangs, config);
- for _ in 0..1000 {
- bench!("the", searcher, c);
- bench!("dtu", searcher, c);
- bench!("the best", searcher, c);
- bench!("the circle of life", searcher, c);
- bench!("what", searcher, c);
- bench!("a", searcher, c);
- bench!("sun", searcher, c);
- bench!("what a sun", searcher, c);
+ for query in &queries {
+ bench!(query, searcher, c);
}
+
+ // for _ in 0..1000 {
+ // bench!("the", searcher, c);
+ // bench!("dtu", searcher, c);
+ // bench!("the best", searcher, c);
+ // bench!("the circle of life", searcher, c);
+ // bench!("what", searcher, c);
+ // bench!("a", searcher, c);
+ // bench!("sun", searcher, c);
+ // bench!("what a sun", searcher, c);
+ // }
}
criterion_group!(benches, criterion_benchmark);
diff --git a/crates/core/src/autosuggest.rs b/crates/core/src/autosuggest.rs
index f5ce584b..b04520dd 100644
--- a/crates/core/src/autosuggest.rs
+++ b/crates/core/src/autosuggest.rs
@@ -60,4 +60,8 @@ impl Autosuggest {
.take(10)
.collect())
}
+
+ pub fn all(&self) -> Result<Vec<String>> {
+ Ok(self.queries.into_stream().into_strs()?)
+ }
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 3b1a5c1c..1016c076 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -32,7 +32,7 @@ pub mod inverted_index;
pub mod mapreduce;
mod api;
-mod autosuggest;
+pub mod autosuggest;
pub mod bangs;
mod bloom;
mod collector;