Skip to content

web search done right

move search bench into an example so we can easier profile with perf

Mikkel Denker5164789a3240
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index ce800b1e..229f6456 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -114,14 +114,6 @@ maplit = {workspace = true}
proptest = {workspace = true}
proptest-derive = {workspace = true}
-[[bench]]
-harness = false
-name = "search_preindexed"
-
-[[bench]]
-harness = false
-name = "search_preindexed_optic"
-
[[bench]]
harness = false
name = "harmonic_centrality"
@@ -146,5 +138,8 @@ name = "naive_bayes"
harness = false
name = "bitvec_similarity"
+[[example]]
+name = "search_preindexed"
+
[[example]]
name = "bitvec_similarity"
diff --git a/crates/core/benches/search_preindexed_optic.rs b/crates/core/benches/search_preindexed_optic.rs
deleted file mode 100644
index 48c8c694..00000000
--- a/crates/core/benches/search_preindexed_optic.rs
+++ /dev/null
@@ -1,170 +0,0 @@
-use criterion::{criterion_group, criterion_main, Criterion};
-use optics::Optic;
-use stract::{
- bangs::Bangs,
- config::{
- ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, LLMConfig, WidgetsConfig,
- },
- entity_index::EntityMatch,
- image_store::Image,
- index::Index,
- inverted_index::RetrievedWebpage,
- ranking::{inbound_similarity::InboundSimilarity, pipeline::RetrievedWebpageRanking},
- searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery, ShardId},
- Result,
-};
-struct Searcher(LocalSearcher<Index>);
-
-impl stract::searcher::distributed::SearchClient for Searcher {
- async fn search_initial(
- &self,
- query: &SearchQuery,
- ) -> Vec<stract::searcher::InitialSearchResultShard> {
- let res = self.0.search_initial(query, true).unwrap();
-
- vec![stract::searcher::InitialSearchResultShard {
- local_result: res,
- shard: ShardId::new(0),
- }]
- }
-
- async fn retrieve_webpages(
- &self,
- top_websites: &[(usize, stract::searcher::ScoredWebsitePointer)],
- query: &str,
- ) -> Vec<(usize, stract::ranking::pipeline::RetrievedWebpageRanking)> {
- let pointers = top_websites
- .iter()
- .map(|(_, p)| p.website.pointer.clone())
- .collect::<Vec<_>>();
-
- let res = self
- .0
- .retrieve_websites(&pointers, query)
- .unwrap()
- .into_iter()
- .zip(top_websites.iter().map(|(i, p)| (*i, p.website.clone())))
- .map(|(ret, (i, ran))| (i, RetrievedWebpageRanking::new(ret, ran)))
- .collect::<Vec<_>>();
-
- res
- }
-
- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- Ok(self.0.get_webpage(url))
- }
-
- async fn get_homepage_descriptions(
- &self,
- urls: &[url::Url],
- ) -> std::collections::HashMap<url::Url, String> {
- let mut res = std::collections::HashMap::new();
-
- for url in urls {
- if let Some(homepage) = self.0.get_homepage(url) {
- if let Some(desc) = homepage.description() {
- res.insert(url.clone(), desc.clone());
- }
- }
- }
-
- res
- }
-
- async fn get_entity_image(
- &self,
- _image_id: &str,
- _max_height: Option<u64>,
- _max_width: Option<u64>,
- ) -> Result<Option<Image>> {
- Ok(None)
- }
-
- async fn search_entity(&self, _query: &str) -> Option<EntityMatch> {
- None
- }
-}
-
-macro_rules! bench {
- ($query:tt, $searcher:ident, $optic:ident, $c:ident) => {
- let runtime = tokio::runtime::Builder::new_current_thread()
- .build()
- .unwrap();
-
- let mut desc = "search '".to_string();
- desc.push_str($query);
- desc.push('\'');
- desc.push_str(" with optic");
- $c.bench_function(desc.as_str(), |b| {
- b.iter(|| {
- runtime.block_on(async {
- $searcher
- .search(&SearchQuery {
- query: $query.to_string(),
- optic: Some(Optic::parse($optic).unwrap()),
- ..Default::default()
- })
- .await
- .unwrap()
- })
- })
- });
- };
-}
-
-pub fn criterion_benchmark(c: &mut Criterion) {
- let index = Index::open("data/index").unwrap();
- let optic = include_str!("../../optics/testcases/samples/discussions.optic");
-
- let config = ApiConfig {
- queries_csv_path: "data/queries_us.csv".to_string(),
- host: "0.0.0.0:8000".parse().unwrap(),
- prometheus_host: "0.0.0.0:8001".parse().unwrap(),
- crossencoder_model_path: None,
- lambda_model_path: None,
- spell_checker_path: Some("data/web_spell".to_string()),
- bangs_path: "data/bangs.json".to_string(),
- summarizer_path: "data/summarizer".to_string(),
- query_store_db_host: None,
- cluster_id: "api".to_string(),
- gossip_seed_nodes: None,
- gossip_addr: "0.0.0.0:8002".parse().unwrap(),
- collector: CollectorConfig::default(),
- thresholds: ApiThresholds::default(),
- widgets: WidgetsConfig {
- thesaurus_paths: vec!["data/english-wordnet-2022-subset.ttl".to_string()],
- calculator_fetch_currencies_exchange: false,
- },
- correction_config: CorrectionConfig::default(),
- llm: LLMConfig {
- api_base: "http://localhost:4000/v1".to_string(),
- model: "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf".to_string(),
- api_key: None,
- },
- };
-
- let mut searcher = LocalSearcher::new(index);
- searcher.set_inbound_similarity(
- InboundSimilarity::open("data/centrality/inbound_similarity").unwrap(),
- );
- let bangs = Bangs::from_path(&config.bangs_path);
-
- let searcher = Searcher(searcher);
-
- let searcher: ApiSearcher<Searcher, LiveSearcher> =
- ApiSearcher::new(searcher, None, None, None, bangs, config);
-
- for _ in 0..1000 {
- bench!("the", searcher, optic, c);
- bench!("dtu", searcher, optic, c);
- bench!("the best", searcher, optic, c);
- bench!("the circle of life", searcher, optic, c);
- bench!("what", searcher, optic, c);
- bench!("a", searcher, optic, c);
- bench!("sun", searcher, optic, c);
- bench!("what a sun", searcher, optic, c);
- }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
diff --git a/crates/core/benches/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
similarity index 76%
rename from crates/core/benches/search_preindexed.rs
rename to crates/core/examples/search_preindexed.rs
index d38c760b..c1cc9dad 100644
--- a/crates/core/benches/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -1,4 +1,3 @@
-use criterion::{criterion_group, criterion_main, Criterion};
use optics::HostRankings;
use rand::seq::SliceRandom;
use stract::{
@@ -86,37 +85,8 @@ impl stract::searcher::distributed::SearchClient for Searcher {
}
}
-macro_rules! bench {
- ($query:tt, $searcher:ident, $c:ident) => {
- let runtime = tokio::runtime::Builder::new_current_thread()
- .build()
- .unwrap();
-
- let mut desc = "search '".to_string();
- desc.push_str($query);
- desc.push('\'');
- $c.bench_function(desc.as_str(), |b| {
- b.iter(|| {
- runtime.block_on(async {
- $searcher
- .search(&SearchQuery {
- query: $query.to_string(),
- host_rankings: Some(HostRankings {
- liked: vec![],
- disliked: vec![],
- blocked: vec![],
- }),
- ..Default::default()
- })
- .await
- .unwrap()
- })
- })
- });
- };
-}
-
-pub fn criterion_benchmark(c: &mut Criterion) {
+#[tokio::main]
+pub async fn main() {
let index = Index::open("data/index").unwrap();
let collector_conf = CollectorConfig {
@@ -173,20 +143,23 @@ pub fn criterion_benchmark(c: &mut Criterion) {
ApiSearcher::new(searcher, None, None, None, bangs, config);
for query in &queries {
- bench!(query, searcher, c);
- }
+ let mut desc = "search '".to_string();
+ desc.push_str(query);
+ desc.push('\'');
- // for _ in 0..1000 {
- // bench!("the", searcher, c);
- // bench!("dtu", searcher, c);
- // bench!("the best", searcher, c);
- // bench!("the circle of life", searcher, c);
- // bench!("what", searcher, c);
- // bench!("a", searcher, c);
- // bench!("sun", searcher, c);
- // bench!("what a sun", searcher, c);
- // }
+ println!("{}", desc);
+
+ searcher
+ .search(&SearchQuery {
+ query: query.to_string(),
+ host_rankings: Some(HostRankings {
+ liked: vec![],
+ disliked: vec![],
+ blocked: vec![],
+ }),
+ ..Default::default()
+ })
+ .await
+ .unwrap();
+ }
}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
diff --git a/crates/core/src/entrypoint/indexer.rs b/crates/core/src/entrypoint/indexer.rs
index 5dabcfde..05b4d811 100644
--- a/crates/core/src/entrypoint/indexer.rs
+++ b/crates/core/src/entrypoint/indexer.rs
@@ -453,6 +453,8 @@ impl Indexer {
std::fs::remove_dir_all(other_path).unwrap();
}
+ index.inverted_index.merge_into_max_segments(1).unwrap();
+
Ok(())
}
}