web search done right
fixed bug that caused queries with special characters to crash ('c++' etc) 'c++' gets tokenized as ['c', '+', '+'] which we use in a phrase query to enforce that the result must have 'c++' in sequence instead of simply having 'c' somewhere on the page and '+' another place. however, some fields don't have the necesarry position data stored which caused these queries to crash when trying to perform the phrase query on these fields
Mikkel Denker76cd7e8f630a
diff --git a/Cargo.lock b/Cargo.lockindex 3fb4dd83..bc99be0a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4657,7 +4657,6 @@ dependencies = ["nom","num-bigint","num-rational",- "num_cpus","once_cell","openraft","optics",diff --git a/Cargo.toml b/Cargo.tomlindex a2fd4326..e28763ce 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -82,7 +82,6 @@ min-max-heap = "1.3.0"nom = "7.1.3"num-bigint = "0.4.1"num-rational = "0.4.1"-num_cpus = "1.15.0"once_cell = "1.13.1"openraft = {version = "0.9.1", features = ["storage-v2",diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 2e6e07bf..6724c6a3 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -74,7 +74,6 @@ min-max-heap = {workspace = true}nom = {workspace = true}num-bigint = {workspace = true}num-rational = {workspace = true}-num_cpus = {workspace = true}once_cell = {workspace = true}openraft = {workspace = true}optics = {path = "../optics"}diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex 3146e510..3816d389 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -361,7 +361,10 @@ impl CrawlPlanner {let next_queue = AtomicUsize::new(0);- let num_threads = self.config.num_threads.unwrap_or_else(num_cpus::get);+ let num_threads = self+ .config+ .num_threads+ .unwrap_or_else(|| usize::from(std::thread::available_parallelism().unwrap()));let pool = ThreadPoolBuilder::new().num_threads(num_threads)diff --git a/crates/core/src/entrypoint/canonical.rs b/crates/core/src/entrypoint/canonical.rsindex f9efbded..feeef1de 100644--- a/crates/core/src/entrypoint/canonical.rs+++ b/crates/core/src/entrypoint/canonical.rs@@ -81,7 +81,7 @@ pub fn create(config: CanonicalIndexConfig) -> Result<()> {}).collect();- let num_workers = num_cpus::get();+ let num_workers = usize::from(std::thread::available_parallelism()?);let mut handles = Vec::new();for i in 0..num_workers {diff --git a/crates/core/src/entrypoint/feed_indexer.rs b/crates/core/src/entrypoint/feed_indexer.rsindex 2695f181..1b99bb27 100644--- a/crates/core/src/entrypoint/feed_indexer.rs+++ b/crates/core/src/entrypoint/feed_indexer.rs@@ -126,7 +126,7 @@ pub fn build(config: FeedIndexingConfig) -> Result<()> {pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {let num_indexes = indexes.len();let mut it = indexes.into_iter();- let num_cores = num_cpus::get();+ let num_cores = usize::from(std::thread::available_parallelism()?);let mut threads = Vec::new();diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rsindex 04a75fe5..068c8f7c 100644--- a/crates/core/src/entrypoint/indexer/mod.rs+++ b/crates/core/src/entrypoint/indexer/mod.rs@@ -72,7 +72,7 @@ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {let num_indexes = indexes.len();let mut it = indexes.into_iter();- let num_cores = num_cpus::get();+ let num_cores = usize::from(std::thread::available_parallelism()?);let mut threads = Vec::new();diff --git a/crates/core/src/entrypoint/web_spell.rs b/crates/core/src/entrypoint/web_spell.rsindex 2e31fe23..403e1c36 100644--- a/crates/core/src/entrypoint/web_spell.rs+++ b/crates/core/src/entrypoint/web_spell.rs@@ -135,7 +135,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {}).collect_vec();- let num_workers = (num_cpus::get()).min(jobs.len());+ let num_workers = usize::from(std::thread::available_parallelism()?).min(jobs.len());let mut handlers = Vec::new();for i in 0..num_workers {diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 6c1bfc86..6ca96659 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -204,7 +204,7 @@ impl Webgraph {None};- let num_workers = num_cpus::get();+ let num_workers = usize::from(std::thread::available_parallelism()?);let mut handlers = Vec::new();let host_path = &config.host_graph_base_path;diff --git a/crates/core/src/executor.rs b/crates/core/src/executor.rsindex 4a87f251..8556548c 100644--- a/crates/core/src/executor.rs+++ b/crates/core/src/executor.rs@@ -32,7 +32,7 @@ impl Executor {}pub fn multi_thread(prefix: &'static str) -> Result<Executor> {- Self::with_threads(num_cpus::get(), prefix)+ Self::with_threads(std::thread::available_parallelism()?.into(), prefix)}pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 0ebb93de..37b155ad 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -1322,4 +1322,61 @@ mod tests {assert_eq!(res.num_websites, approx_count::Count::Approximate(1_000));}++ #[test]+ fn test_search_special_characters() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");++ let webpage = Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>C++</title>+ </head>+ <body>+ {CONTENT} test+ </body>+ </html>+ "#,+ CONTENT = crate::rand_words(100)+ ),+ "https://www.a.com",+ )+ .unwrap();++ index.insert(&webpage).unwrap();++ index.commit().expect("failed to commit index");++ let ctx = index.local_search_ctx();++ let query = Query::parse(+ &ctx,+ &SearchQuery {+ query: "c++".to_string(),+ ..Default::default()+ },+ &index,+ )+ .expect("Failed to parse query");++ let ranker = Ranker::new(+ SignalComputer::new(Some(&query)),+ ctx.fastfield_reader.clone(),+ CollectorConfig::default(),+ );++ let res = index+ .search_initial(&query, &ctx, ranker.collector(ctx.clone()))+ .unwrap();++ assert_eq!(res.top_websites.len(), 1);++ let webpages = index.retrieve_websites(&res.top_websites, &query).unwrap();++ assert_eq!(webpages.len(), 1);+ assert_eq!(webpages[0].title, "C++");+ assert_eq!(webpages[0].url, "https://www.a.com/");+ }}diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rsindex 72a31576..ce0d0d24 100644--- a/crates/core/src/query/plan/mod.rs+++ b/crates/core/src/query/plan/mod.rs@@ -147,8 +147,21 @@ impl Query {if terms.len() == 1 {let term = terms.remove(0);Some(Box::new(tantivy::query::TermQuery::new(term, option)))- } else {+ } else if !terms.is_empty() && option.has_positions() {Some(Box::new(tantivy::query::PhraseQuery::new(terms)))+ } else {+ Some(Box::new(tantivy::query::BooleanQuery::new(+ terms+ .into_iter()+ .map(|term| {+ (+ tantivy::query::Occur::Must,+ Box::new(tantivy::query::TermQuery::new(term, option))+ as Box<dyn tantivy::query::Query + 'static>,+ )+ })+ .collect(),+ )))}}SimpleOrPhrase::Phrase(p) => {diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex b283b5a4..8b112306 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -1159,7 +1159,7 @@ impl TextField for FlattenedSchemaOrgJson {doc.add_pre_tokenized_text(self.tantivy_field(schema).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),- dbg!(cache.pretokenized_schema_json().clone()),+ cache.pretokenized_schema_json().clone(),);Ok(())diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rsindex 90d22169..daf66c59 100644--- a/crates/core/src/webpage/html/fn_cache.rs+++ b/crates/core/src/webpage/html/fn_cache.rs@@ -106,7 +106,7 @@ impl<'a> FnCache<'a> {pub fn pretokenized_schema_json(&mut self) -> &PreTokenizedString {if self.pretokenized_schema_json.is_none() {self.pretokenized_schema_json =- match schema_org::flattened_json(dbg!(self.schema_org().clone())) {+ match schema_org::flattened_json(self.schema_org().clone()) {Ok(mut f) => {let mut tokens = Vec::new();