Skip to content

web search done right

refactoring

Mikkel Denker43fb159742e0
diff --git a/src/search_index.rs b/src/index.rs
similarity index 86%
rename from src/search_index.rs
rename to src/index.rs
index 6cf2b806..d987ba1e 100644
--- a/src/search_index.rs
+++ b/src/index.rs
@@ -13,11 +13,12 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::collector::Count;
+use tantivy::collector::{Collector, Count};
use tantivy::{DocAddress, Document, LeasedItem, Searcher};
use crate::query::Query;
use crate::schema::{create_schema, Field, ALL_FIELDS};
+use crate::searcher::SearchResult;
use crate::webpage::Webpage;
use crate::Result;
use crate::{ranking, snippet};
@@ -59,14 +60,14 @@ impl Index {
Ok(())
}
- pub fn search(&self, query: &Query) -> Result<SearchResult> {
+ pub fn search<C>(&self, query: &Query, collector: C) -> Result<SearchResult>
+ where
+ C: Collector<Fruit = Vec<(f64, tantivy::DocAddress)>>,
+ {
let tantivy_query = query.tantivy(&self.schema, self.tantivy_index.tokenizers());
let searcher = self.reader.searcher();
- let (count, docs) = searcher.search(
- &tantivy_query,
- &(Count, ranking::initial_collector(query.clone())),
- )?;
+ let (count, docs) = searcher.search(&tantivy_query, &(Count, collector))?;
let mut webpages: Vec<RetrievedWebpage> = docs
.into_iter()
@@ -94,16 +95,6 @@ impl Index {
}
}
-pub struct SearchResult {
- pub num_docs: usize,
- pub documents: Vec<RetrievedWebpage>,
-}
-
-#[derive(Default)]
-pub struct FastWebpage {
- pub host: String,
-}
-
#[derive(Default)]
pub struct RetrievedWebpage {
pub title: String,
@@ -149,7 +140,7 @@ impl From<Document> for RetrievedWebpage {
#[cfg(test)]
mod tests {
- use crate::webpage::Link;
+ use crate::{ranking::Ranker, webpage::Link};
use super::*;
@@ -157,8 +148,11 @@ mod tests {
fn simple_search() {
let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("website").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
- let result = index.search(&query).expect("Search failed");
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
@@ -178,7 +172,9 @@ mod tests {
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let result = index.search(&query).expect("Search failed");
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -188,6 +184,7 @@ mod tests {
fn document_not_matching() {
let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("this query should not match").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
index
.insert(Webpage::new(
@@ -205,7 +202,9 @@ mod tests {
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let result = index.search(&query).expect("Search failed");
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
}
@@ -214,6 +213,7 @@ mod tests {
fn english_stemming() {
let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("runner").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
index
.insert(Webpage::new(
@@ -231,7 +231,9 @@ mod tests {
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let result = index.search(&query).expect("Search failed");
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
}
@@ -240,6 +242,7 @@ mod tests {
fn searchable_backlinks() {
let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("great site").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
index
.insert(Webpage::new(
@@ -277,7 +280,9 @@ mod tests {
index.commit().expect("failed to commit index");
- let mut result = index.search(&query).expect("Search failed");
+ let mut result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
result
.documents
@@ -292,6 +297,7 @@ mod tests {
fn limited_top_docs() {
let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("runner").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
for _ in 0..100 {
index
@@ -312,7 +318,9 @@ mod tests {
index.commit().expect("failed to commit index");
- let result = index.search(&query).expect("Search failed");
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
assert_eq!(result.documents.len(), 20);
}
@@ -320,6 +328,7 @@ mod tests {
fn host_search() {
let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("dr").expect("Failed to parse query");
+ let ranker = Ranker::new(query.clone());
index
.insert(Webpage::new(
@@ -337,7 +346,9 @@ mod tests {
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let result = index.search(&query).expect("Search failed");
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
}
diff --git a/src/lib.rs b/src/lib.rs
index f4efedec..3efff74d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -19,11 +19,12 @@ use std::num::ParseIntError;
use tantivy::TantivyError;
use thiserror::Error;
+mod index;
mod indexer;
mod query;
mod ranking;
mod schema;
-mod search_index;
+mod searcher;
mod snippet;
mod warc;
mod webgraph;
diff --git a/src/ranking.rs b/src/ranking.rs
deleted file mode 100644
index 80dee0dd..00000000
--- a/src/ranking.rs
+++ /dev/null
@@ -1,261 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use crate::query::Query;
-use crate::schema::Field;
-use crate::search_index::FastWebpage;
-use std::sync::Arc;
-use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
-use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};
-use tantivy::{
- collector::{Collector, TopDocs},
- DocId, Score, SegmentReader,
-};
-
-struct InitialScoreTweaker {
- query: Arc<Query>,
-}
-
-impl InitialScoreTweaker {
- pub fn new(query: Query) -> Self {
- Self {
- query: Arc::new(query),
- }
- }
-}
-
-struct InitialSegmentScoreTweaker {
- query: Arc<Query>,
- centrality_reader: DynamicFastFieldReader<f64>,
- domain_reader: BytesFastFieldReader,
-}
-
-impl InitialSegmentScoreTweaker {
- fn navigational_score(&self, terms: Vec<String>, doc: DocId) -> f64 {
- let bytes = self.domain_reader.get_bytes(doc);
- let domain = bincode::deserialize(bytes).expect("Failed to deserialize domain");
-
- if is_navigational(terms, domain) {
- 1.0
- } else {
- 0.0
- }
- }
-}
-
-impl ScoreTweaker<f64> for InitialScoreTweaker {
- type Child = InitialSegmentScoreTweaker;
-
- fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {
- let centrality_field = segment_reader
- .schema()
- .get_field(Field::Centrality.as_str())
- .expect("Faild to load centrality field");
- let centrality_reader = segment_reader
- .fast_fields()
- .f64(centrality_field)
- .expect("Failed to get centrality fast-field reader");
-
- let domain_field = segment_reader
- .schema()
- .get_field(Field::Domain.as_str())
- .expect("Faild to load domain field");
- let domain_reader = segment_reader
- .fast_fields()
- .bytes(domain_field)
- .expect("Failed to get domain fast-field reader");
-
- Ok(InitialSegmentScoreTweaker {
- query: self.query.clone(),
- centrality_reader,
- domain_reader,
- })
- }
-}
-
-impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
- fn score(&mut self, doc: DocId, score: Score) -> f64 {
- let centrality = self.centrality_reader.get(doc);
- let navigational = self.navigational_score(self.query.terms.clone(), doc);
- score as f64 + 1000.0 * centrality + 100.0 * navigational
- }
-}
-
-fn jaccard_sim(mut terms: Vec<String>, domain_parts: &[&str]) -> f64 {
- debug_assert_eq!(domain_parts.len(), 2);
-
- terms.sort();
- terms.dedup();
-
- let intersection_size: f64 = terms
- .iter()
- .map(|term| {
- if domain_parts.contains(&term.as_str()) {
- 1.0
- } else {
- 0.0
- }
- })
- .sum();
-
- intersection_size / (terms.len() as f64 + domain_parts.len() as f64 - intersection_size)
-}
-
-fn is_navigational(mut terms: Vec<String>, domain: &str) -> bool {
- let (name, remaining) = domain.split_once(".").unwrap();
- jaccard_sim(terms, &[name, remaining]) >= 0.5
-}
-
-pub(crate) fn initial_collector(
- query: Query,
-) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
- let score_tweaker = InitialScoreTweaker::new(query);
- TopDocs::with_limit(20).tweak_score(score_tweaker)
-}
-
-#[cfg(test)]
-mod tests {
-
- use super::*;
- use crate::{
- query::Query,
- search_index::Index,
- webpage::{Link, Webpage},
- };
-
- #[test]
- fn test_jaccard_sim() {
- assert_eq!(
- jaccard_sim(vec!["dr".to_string(), "dk".to_string()], &["dr", "dk"]),
- 1.0
- );
-
- assert_eq!(
- jaccard_sim(
- vec![
- "dr".to_string(),
- "dk".to_string(),
- "dr".to_string(),
- "dr".to_string()
- ],
- &["dr", "dk"]
- ),
- 1.0
- );
- assert_eq!(jaccard_sim(vec!["dr".to_string()], &["dr", "dk"]), 0.5);
- }
-
- #[test]
- fn harmonic_ranking() {
- let query = Query::parse("great site").expect("Failed to parse query");
-
- for _ in 0..10 {
- let mut index = Index::temporary().expect("Unable to open index");
-
- index
- .insert(Webpage::new(
- r#"
- <html>
- <head>
- <title>Website A</title>
- </head>
- <a href="https://www.b.com">B site is great</a>
- </html>
- "#,
- "https://www.a.com",
- vec![],
- 0.0,
- ))
- .expect("failed to parse webpage");
- index
- .insert(Webpage::new(
- r#"
- <html>
- <head>
- <title>Website B</title>
- </head>
- </html>
- "#,
- "https://www.b.com",
- vec![Link {
- source: "https://www.a.com".to_string(),
- destination: "https://www.b.com".to_string(),
- text: "B site is great".to_string(),
- }],
- 5.0,
- ))
- .expect("failed to parse webpage");
-
- index.commit().expect("failed to commit index");
- let result = index.search(&query).expect("Search failed");
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.b.com");
- assert_eq!(result.documents[1].url, "https://www.a.com");
- }
- }
-
- #[test]
- fn navigational() {
- assert!(is_navigational(vec!["dr".to_string()], "dr.dk"));
- assert!(is_navigational(
- vec!["dr".to_string(), "dk".to_string()],
- "dr.dk"
- ));
- }
-
- #[test]
- fn navigational_search() {
- let query = Query::parse("dr dk").expect("Failed to parse query");
-
- let mut index = Index::temporary().expect("Unable to open index");
-
- index
- .insert(Webpage::new(
- r#"
- <html>
- <head>
- <title>Website A</title>
- </head>
- </html>
- "#,
- "https://www.dr.dk",
- vec![],
- 0.0,
- ))
- .expect("failed to parse webpage");
- index
- .insert(Webpage::new(
- r#"
- <html>
- <head>
- <title>Website B</title>
- dr dk dr dk dr dk dr dk
- </head>
- </html>
- "#,
- "https://www.b.com",
- vec![],
- 0.0003,
- ))
- .expect("failed to parse webpage");
-
- index.commit().expect("failed to commit index");
- let result = index.search(&query).expect("Search failed");
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.dr.dk");
- assert_eq!(result.documents[1].url, "https://www.b.com");
- }
-}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
new file mode 100644
index 00000000..433c2ee7
--- /dev/null
+++ b/src/ranking/initial.rs
@@ -0,0 +1,146 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::query::Query;
+use crate::schema::Field;
+use std::sync::Arc;
+use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
+use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};
+use tantivy::{DocId, Score, SegmentReader};
+
+pub(crate) struct InitialScoreTweaker {
+ query: Arc<Query>,
+}
+
+impl InitialScoreTweaker {
+ pub fn new(query: Arc<Query>) -> Self {
+ Self { query }
+ }
+}
+
+pub(crate) struct InitialSegmentScoreTweaker {
+ centrality_reader: DynamicFastFieldReader<f64>,
+ domain_reader: BytesFastFieldReader,
+ sorted_dedupped_terms: Vec<String>,
+}
+
+impl InitialSegmentScoreTweaker {
+ fn navigational_score(&self, sorted_dedupped_terms: &[String], doc: DocId) -> f64 {
+ let bytes = self.domain_reader.get_bytes(doc);
+ let domain = bincode::deserialize(bytes).expect("Failed to deserialize domain");
+
+ if is_navigational(sorted_dedupped_terms, domain) {
+ 1.0
+ } else {
+ 0.0
+ }
+ }
+}
+
+impl ScoreTweaker<f64> for InitialScoreTweaker {
+ type Child = InitialSegmentScoreTweaker;
+
+ fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {
+ let centrality_field = segment_reader
+ .schema()
+ .get_field(Field::Centrality.as_str())
+ .expect("Faild to load centrality field");
+ let centrality_reader = segment_reader
+ .fast_fields()
+ .f64(centrality_field)
+ .expect("Failed to get centrality fast-field reader");
+
+ let domain_field = segment_reader
+ .schema()
+ .get_field(Field::Domain.as_str())
+ .expect("Faild to load domain field");
+ let domain_reader = segment_reader
+ .fast_fields()
+ .bytes(domain_field)
+ .expect("Failed to get domain fast-field reader");
+
+ let mut sorted_dedupped_terms = self.query.terms.clone();
+
+ sorted_dedupped_terms.sort();
+ sorted_dedupped_terms.dedup();
+
+ Ok(InitialSegmentScoreTweaker {
+ centrality_reader,
+ domain_reader,
+ sorted_dedupped_terms,
+ })
+ }
+}
+
+impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
+ fn score(&mut self, doc: DocId, score: Score) -> f64 {
+ let centrality = self.centrality_reader.get(doc);
+ let navigational = self.navigational_score(&self.sorted_dedupped_terms, doc);
+ score as f64 + 1000.0 * centrality + 100.0 * navigational
+ }
+}
+
+fn jaccard_sim(sorted_dedupped_terms: &[String], domain_parts: &[&str]) -> f64 {
+ debug_assert_eq!(domain_parts.len(), 2);
+
+ let intersection_size: f64 = sorted_dedupped_terms
+ .iter()
+ .map(|term| {
+ if domain_parts.contains(&term.as_str()) {
+ 1.0
+ } else {
+ 0.0
+ }
+ })
+ .sum();
+
+ intersection_size
+ / (sorted_dedupped_terms.len() as f64 + domain_parts.len() as f64 - intersection_size)
+}
+
+fn is_navigational(sorted_dedupped_terms: &[String], domain: &str) -> bool {
+ let (name, remaining) = domain.split_once('.').unwrap();
+ jaccard_sim(sorted_dedupped_terms, &[name, remaining]) >= 0.5
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::{
+ index::Index,
+ query::Query,
+ webpage::{Link, Webpage},
+ };
+
+ #[test]
+ fn test_jaccard_sim() {
+ assert_eq!(
+ jaccard_sim(&["dr".to_string(), "dk".to_string()], &["dr", "dk"]),
+ 1.0
+ );
+
+ assert_eq!(jaccard_sim(&["dr".to_string()], &["dr", "dk"]), 0.5);
+ }
+
+ #[test]
+ fn navigational() {
+ assert!(is_navigational(&["dr".to_string()], "dr.dk"));
+ assert!(is_navigational(
+ &["dk".to_string(), "dr".to_string()],
+ "dr.dk"
+ ));
+ }
+}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
new file mode 100644
index 00000000..c660a38c
--- /dev/null
+++ b/src/ranking/mod.rs
@@ -0,0 +1,140 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod initial;
+
+use std::sync::Arc;
+
+use crate::query::Query;
+use initial::InitialScoreTweaker;
+use tantivy::collector::{Collector, TopDocs};
+
+pub struct Ranker {
+ query: Arc<Query>,
+}
+
+impl Ranker {
+ pub fn new(query: Query) -> Self {
+ Ranker {
+ query: Arc::new(query),
+ }
+ }
+
+ pub(crate) fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {
+ let score_tweaker = InitialScoreTweaker::new(self.query.clone());
+ TopDocs::with_limit(20).tweak_score(score_tweaker)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::{
+ index::Index,
+ query::Query,
+ searcher::Searcher,
+ webpage::{Link, Webpage},
+ };
+
+ #[test]
+ fn harmonic_ranking() {
+ for _ in 0..10 {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ <a href="https://www.b.com">B site is great</a>
+ </html>
+ "#,
+ "https://www.a.com",
+ vec![],
+ 0.0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ </html>
+ "#,
+ "https://www.b.com",
+ vec![Link {
+ source: "https://www.a.com".to_string(),
+ destination: "https://www.b.com".to_string(),
+ text: "B site is great".to_string(),
+ }],
+ 5.0,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = Searcher::from(index);
+ let result = searcher.search("great site").expect("Search failed");
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.b.com");
+ assert_eq!(result.documents[1].url, "https://www.a.com");
+ }
+ }
+
+ #[test]
+ fn navigational_search() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ </html>
+ "#,
+ "https://www.dr.dk",
+ vec![],
+ 0.0,
+ ))
+ .expect("failed to parse webpage");
+ index
+ .insert(Webpage::new(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ dr dk dr dk dr dk dr dk
+ </head>
+ </html>
+ "#,
+ "https://www.b.com",
+ vec![],
+ 0.0003,
+ ))
+ .expect("failed to parse webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = Searcher::from(index);
+ let result = searcher.search("dr dk").expect("Search failed");
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.dr.dk");
+ assert_eq!(result.documents[1].url, "https://www.b.com");
+ }
+}
diff --git a/src/searcher.rs b/src/searcher.rs
new file mode 100644
index 00000000..c6862c47
--- /dev/null
+++ b/src/searcher.rs
@@ -0,0 +1,48 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::index::{Index, RetrievedWebpage};
+use crate::query::Query;
+use crate::ranking::Ranker;
+use crate::Result;
+
+pub struct SearchResult {
+ pub num_docs: usize,
+ pub documents: Vec<RetrievedWebpage>,
+}
+
+pub struct Searcher {
+ index: Index,
+}
+
+#[cfg(test)]
+impl From<Index> for Searcher {
+ fn from(index: Index) -> Self {
+ Self::from_index(index)
+ }
+}
+
+impl Searcher {
+ fn from_index(index: Index) -> Self {
+ Searcher { index }
+ }
+
+ pub fn search(&self, query: &str) -> Result<SearchResult> {
+ let query = Query::parse(query)?;
+ let ranker = Ranker::new(query.clone());
+ self.index.search(&query, ranker.collector())
+ }
+}
diff --git a/src/snippet.rs b/src/snippet.rs
index 5351116c..344f0990 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -14,7 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::{Document, Searcher, SnippetGenerator};
+use tantivy::Searcher as TantivySearcher;
+use tantivy::SnippetGenerator;
use crate::{query::Query, schema::Field, Result};
@@ -22,9 +23,9 @@ use crate::{query::Query, schema::Field, Result};
/// In the future we want to implement something closer to the method described in https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf.
/// This will require us to store each paragraph of the webpage separately to get adequate performance.
/// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.
-pub fn generate(query: &Query, text: &str, searcher: &Searcher) -> Result<String> {
+pub fn generate(query: &Query, text: &str, searcher: &TantivySearcher) -> Result<String> {
let generator = SnippetGenerator::create(
- &searcher,
+ searcher,
&query.tantivy(searcher.schema(), searcher.index().tokenizers()),
searcher
.schema()
@@ -40,7 +41,7 @@ pub fn generate(query: &Query, text: &str, searcher: &Searcher) -> Result<String
#[cfg(test)]
mod tests {
- use crate::{search_index::Index, webpage::Webpage};
+ use crate::{index::Index, searcher::Searcher, webpage::Webpage};
use super::*;
@@ -59,7 +60,6 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn snippet_during_search() {
let mut index = Index::temporary().expect("Unable to open index");
- let query = Query::parse("rust language").expect("Failed to parse query");
index
.insert(Webpage::new(
@@ -81,7 +81,9 @@ Survey in 2016, 2017, and 2018."#;
.expect("failed to parse webpage");
index.commit().expect("failed to commit index");
- let result = index.search(&query).expect("Search failed");
+ let searcher = Searcher::from(index);
+
+ let result = searcher.search("rust language").expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
diff --git a/src/webpage.rs b/src/webpage.rs
index 739dfcd7..ffbe4562 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -180,11 +180,11 @@ impl Html {
let host = self.host();
let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();
if num_punctuations > 1 {
- let domain_index = host.rfind(".").unwrap();
- let mut start_index = host[..domain_index].rfind(".").unwrap();
+ let domain_index = host.rfind('.').unwrap();
+ let mut start_index = host[..domain_index].rfind('.').unwrap();
if &host[start_index + 1..] == "co.uk" {
- start_index = host[start_index..].rfind(".").unwrap();
+ start_index = host[start_index..].rfind('.').unwrap();
}
&host[start_index + 1..]