Skip to content

web search done right

fixed bug that caused queries with special characters to crash ('c++' etc) 'c++' gets tokenized as ['c', '+', '+'] which we use in a phrase query to enforce that the result must have 'c++' in sequence instead of simply having 'c' somewhere on the page and '+' another place. however, some fields don't have the necesarry position data stored which caused these queries to crash when trying to perform the phrase query on these fields

Mikkel Denker76cd7e8f630a
diff --git a/Cargo.lock b/Cargo.lock
index 3fb4dd83..bc99be0a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4657,7 +4657,6 @@ dependencies = [
"nom",
"num-bigint",
"num-rational",
- "num_cpus",
"once_cell",
"openraft",
"optics",
diff --git a/Cargo.toml b/Cargo.toml
index a2fd4326..e28763ce 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -82,7 +82,6 @@ min-max-heap = "1.3.0"
nom = "7.1.3"
num-bigint = "0.4.1"
num-rational = "0.4.1"
-num_cpus = "1.15.0"
once_cell = "1.13.1"
openraft = {version = "0.9.1", features = [
"storage-v2",
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 2e6e07bf..6724c6a3 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -74,7 +74,6 @@ min-max-heap = {workspace = true}
nom = {workspace = true}
num-bigint = {workspace = true}
num-rational = {workspace = true}
-num_cpus = {workspace = true}
once_cell = {workspace = true}
openraft = {workspace = true}
optics = {path = "../optics"}
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index 3146e510..3816d389 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -361,7 +361,10 @@ impl CrawlPlanner {
let next_queue = AtomicUsize::new(0);
- let num_threads = self.config.num_threads.unwrap_or_else(num_cpus::get);
+ let num_threads = self
+ .config
+ .num_threads
+ .unwrap_or_else(|| usize::from(std::thread::available_parallelism().unwrap()));
let pool = ThreadPoolBuilder::new()
.num_threads(num_threads)
diff --git a/crates/core/src/entrypoint/canonical.rs b/crates/core/src/entrypoint/canonical.rs
index f9efbded..feeef1de 100644
--- a/crates/core/src/entrypoint/canonical.rs
+++ b/crates/core/src/entrypoint/canonical.rs
@@ -81,7 +81,7 @@ pub fn create(config: CanonicalIndexConfig) -> Result<()> {
})
.collect();
- let num_workers = num_cpus::get();
+ let num_workers = usize::from(std::thread::available_parallelism()?);
let mut handles = Vec::new();
for i in 0..num_workers {
diff --git a/crates/core/src/entrypoint/feed_indexer.rs b/crates/core/src/entrypoint/feed_indexer.rs
index 2695f181..1b99bb27 100644
--- a/crates/core/src/entrypoint/feed_indexer.rs
+++ b/crates/core/src/entrypoint/feed_indexer.rs
@@ -126,7 +126,7 @@ pub fn build(config: FeedIndexingConfig) -> Result<()> {
pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {
let num_indexes = indexes.len();
let mut it = indexes.into_iter();
- let num_cores = num_cpus::get();
+ let num_cores = usize::from(std::thread::available_parallelism()?);
let mut threads = Vec::new();
diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rs
index 04a75fe5..068c8f7c 100644
--- a/crates/core/src/entrypoint/indexer/mod.rs
+++ b/crates/core/src/entrypoint/indexer/mod.rs
@@ -72,7 +72,7 @@ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {
pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {
let num_indexes = indexes.len();
let mut it = indexes.into_iter();
- let num_cores = num_cpus::get();
+ let num_cores = usize::from(std::thread::available_parallelism()?);
let mut threads = Vec::new();
diff --git a/crates/core/src/entrypoint/web_spell.rs b/crates/core/src/entrypoint/web_spell.rs
index 2e31fe23..403e1c36 100644
--- a/crates/core/src/entrypoint/web_spell.rs
+++ b/crates/core/src/entrypoint/web_spell.rs
@@ -135,7 +135,7 @@ pub fn run(config: WebSpellConfig) -> Result<()> {
})
.collect_vec();
- let num_workers = (num_cpus::get()).min(jobs.len());
+ let num_workers = usize::from(std::thread::available_parallelism()?).min(jobs.len());
let mut handlers = Vec::new();
for i in 0..num_workers {
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 6c1bfc86..6ca96659 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -204,7 +204,7 @@ impl Webgraph {
None
};
- let num_workers = num_cpus::get();
+ let num_workers = usize::from(std::thread::available_parallelism()?);
let mut handlers = Vec::new();
let host_path = &config.host_graph_base_path;
diff --git a/crates/core/src/executor.rs b/crates/core/src/executor.rs
index 4a87f251..8556548c 100644
--- a/crates/core/src/executor.rs
+++ b/crates/core/src/executor.rs
@@ -32,7 +32,7 @@ impl Executor {
}
pub fn multi_thread(prefix: &'static str) -> Result<Executor> {
- Self::with_threads(num_cpus::get(), prefix)
+ Self::with_threads(std::thread::available_parallelism()?.into(), prefix)
}
pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 0ebb93de..37b155ad 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -1322,4 +1322,61 @@ mod tests {
assert_eq!(res.num_websites, approx_count::Count::Approximate(1_000));
}
+
+ #[test]
+ fn test_search_special_characters() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ let webpage = Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>C++</title>
+ </head>
+ <body>
+ {CONTENT} test
+ </body>
+ </html>
+ "#,
+ CONTENT = crate::rand_words(100)
+ ),
+ "https://www.a.com",
+ )
+ .unwrap();
+
+ index.insert(&webpage).unwrap();
+
+ index.commit().expect("failed to commit index");
+
+ let ctx = index.local_search_ctx();
+
+ let query = Query::parse(
+ &ctx,
+ &SearchQuery {
+ query: "c++".to_string(),
+ ..Default::default()
+ },
+ &index,
+ )
+ .expect("Failed to parse query");
+
+ let ranker = Ranker::new(
+ SignalComputer::new(Some(&query)),
+ ctx.fastfield_reader.clone(),
+ CollectorConfig::default(),
+ );
+
+ let res = index
+ .search_initial(&query, &ctx, ranker.collector(ctx.clone()))
+ .unwrap();
+
+ assert_eq!(res.top_websites.len(), 1);
+
+ let webpages = index.retrieve_websites(&res.top_websites, &query).unwrap();
+
+ assert_eq!(webpages.len(), 1);
+ assert_eq!(webpages[0].title, "C++");
+ assert_eq!(webpages[0].url, "https://www.a.com/");
+ }
}
diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rs
index 72a31576..ce0d0d24 100644
--- a/crates/core/src/query/plan/mod.rs
+++ b/crates/core/src/query/plan/mod.rs
@@ -147,8 +147,21 @@ impl Query {
if terms.len() == 1 {
let term = terms.remove(0);
Some(Box::new(tantivy::query::TermQuery::new(term, option)))
- } else {
+ } else if !terms.is_empty() && option.has_positions() {
Some(Box::new(tantivy::query::PhraseQuery::new(terms)))
+ } else {
+ Some(Box::new(tantivy::query::BooleanQuery::new(
+ terms
+ .into_iter()
+ .map(|term| {
+ (
+ tantivy::query::Occur::Must,
+ Box::new(tantivy::query::TermQuery::new(term, option))
+ as Box<dyn tantivy::query::Query + 'static>,
+ )
+ })
+ .collect(),
+ )))
}
}
SimpleOrPhrase::Phrase(p) => {
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index b283b5a4..8b112306 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -1159,7 +1159,7 @@ impl TextField for FlattenedSchemaOrgJson {
doc.add_pre_tokenized_text(
self.tantivy_field(schema)
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
- dbg!(cache.pretokenized_schema_json().clone()),
+ cache.pretokenized_schema_json().clone(),
);
Ok(())
diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rs
index 90d22169..daf66c59 100644
--- a/crates/core/src/webpage/html/fn_cache.rs
+++ b/crates/core/src/webpage/html/fn_cache.rs
@@ -106,7 +106,7 @@ impl<'a> FnCache<'a> {
pub fn pretokenized_schema_json(&mut self) -> &PreTokenizedString {
if self.pretokenized_schema_json.is_none() {
self.pretokenized_schema_json =
- match schema_org::flattened_json(dbg!(self.schema_org().clone())) {
+ match schema_org::flattened_json(self.schema_org().clone()) {
Ok(mut f) => {
let mut tokens = Vec::new();