web search done right
refactoring
Mikkel Denker43fb159742e0
diff --git a/src/search_index.rs b/src/index.rssimilarity index 86%rename from src/search_index.rsrename to src/index.rsindex 6cf2b806..d987ba1e 100644--- a/src/search_index.rs+++ b/src/index.rs@@ -13,11 +13,12 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::collector::Count;+use tantivy::collector::{Collector, Count};use tantivy::{DocAddress, Document, LeasedItem, Searcher};use crate::query::Query;use crate::schema::{create_schema, Field, ALL_FIELDS};+use crate::searcher::SearchResult;use crate::webpage::Webpage;use crate::Result;use crate::{ranking, snippet};@@ -59,14 +60,14 @@ impl Index {Ok(())}- pub fn search(&self, query: &Query) -> Result<SearchResult> {+ pub fn search<C>(&self, query: &Query, collector: C) -> Result<SearchResult>+ where+ C: Collector<Fruit = Vec<(f64, tantivy::DocAddress)>>,+ {let tantivy_query = query.tantivy(&self.schema, self.tantivy_index.tokenizers());let searcher = self.reader.searcher();- let (count, docs) = searcher.search(- &tantivy_query,- &(Count, ranking::initial_collector(query.clone())),- )?;+ let (count, docs) = searcher.search(&tantivy_query, &(Count, collector))?;let mut webpages: Vec<RetrievedWebpage> = docs.into_iter()@@ -94,16 +95,6 @@ impl Index {}}-pub struct SearchResult {- pub num_docs: usize,- pub documents: Vec<RetrievedWebpage>,-}--#[derive(Default)]-pub struct FastWebpage {- pub host: String,-}-#[derive(Default)]pub struct RetrievedWebpage {pub title: String,@@ -149,7 +140,7 @@ impl From<Document> for RetrievedWebpage {#[cfg(test)]mod tests {- use crate::webpage::Link;+ use crate::{ranking::Ranker, webpage::Link};use super::*;@@ -157,8 +148,11 @@ mod tests {fn simple_search() {let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("website").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());- let result = index.search(&query).expect("Search failed");+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");assert_eq!(result.documents.len(), 0);assert_eq!(result.num_docs, 0);@@ -178,7 +172,9 @@ mod tests {.expect("failed to parse webpage");index.commit().expect("failed to commit index");- let result = index.search(&query).expect("Search failed");+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");assert_eq!(result.num_docs, 1);assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.example.com");@@ -188,6 +184,7 @@ mod tests {fn document_not_matching() {let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("this query should not match").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());index.insert(Webpage::new(@@ -205,7 +202,9 @@ mod tests {.expect("failed to parse webpage");index.commit().expect("failed to commit index");- let result = index.search(&query).expect("Search failed");+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");assert_eq!(result.documents.len(), 0);assert_eq!(result.num_docs, 0);}@@ -214,6 +213,7 @@ mod tests {fn english_stemming() {let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("runner").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());index.insert(Webpage::new(@@ -231,7 +231,9 @@ mod tests {.expect("failed to parse webpage");index.commit().expect("failed to commit index");- let result = index.search(&query).expect("Search failed");+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.example.com");}@@ -240,6 +242,7 @@ mod tests {fn searchable_backlinks() {let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("great site").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());index.insert(Webpage::new(@@ -277,7 +280,9 @@ mod tests {index.commit().expect("failed to commit index");- let mut result = index.search(&query).expect("Search failed");+ let mut result = index+ .search(&query, ranker.collector())+ .expect("Search failed");result.documents@@ -292,6 +297,7 @@ mod tests {fn limited_top_docs() {let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("runner").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());for _ in 0..100 {index@@ -312,7 +318,9 @@ mod tests {index.commit().expect("failed to commit index");- let result = index.search(&query).expect("Search failed");+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");assert_eq!(result.documents.len(), 20);}@@ -320,6 +328,7 @@ mod tests {fn host_search() {let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("dr").expect("Failed to parse query");+ let ranker = Ranker::new(query.clone());index.insert(Webpage::new(@@ -337,7 +346,9 @@ mod tests {.expect("failed to parse webpage");index.commit().expect("failed to commit index");- let result = index.search(&query).expect("Search failed");+ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.dr.dk");}diff --git a/src/lib.rs b/src/lib.rsindex f4efedec..3efff74d 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -19,11 +19,12 @@ use std::num::ParseIntError;use tantivy::TantivyError;use thiserror::Error;+mod index;mod indexer;mod query;mod ranking;mod schema;-mod search_index;+mod searcher;mod snippet;mod warc;mod webgraph;diff --git a/src/ranking.rs b/src/ranking.rsdeleted file mode 100644index 80dee0dd..00000000--- a/src/ranking.rs+++ /dev/null@@ -1,261 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use crate::query::Query;-use crate::schema::Field;-use crate::search_index::FastWebpage;-use std::sync::Arc;-use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};-use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};-use tantivy::{- collector::{Collector, TopDocs},- DocId, Score, SegmentReader,-};--struct InitialScoreTweaker {- query: Arc<Query>,-}--impl InitialScoreTweaker {- pub fn new(query: Query) -> Self {- Self {- query: Arc::new(query),- }- }-}--struct InitialSegmentScoreTweaker {- query: Arc<Query>,- centrality_reader: DynamicFastFieldReader<f64>,- domain_reader: BytesFastFieldReader,-}--impl InitialSegmentScoreTweaker {- fn navigational_score(&self, terms: Vec<String>, doc: DocId) -> f64 {- let bytes = self.domain_reader.get_bytes(doc);- let domain = bincode::deserialize(bytes).expect("Failed to deserialize domain");-- if is_navigational(terms, domain) {- 1.0- } else {- 0.0- }- }-}--impl ScoreTweaker<f64> for InitialScoreTweaker {- type Child = InitialSegmentScoreTweaker;-- fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {- let centrality_field = segment_reader- .schema()- .get_field(Field::Centrality.as_str())- .expect("Faild to load centrality field");- let centrality_reader = segment_reader- .fast_fields()- .f64(centrality_field)- .expect("Failed to get centrality fast-field reader");-- let domain_field = segment_reader- .schema()- .get_field(Field::Domain.as_str())- .expect("Faild to load domain field");- let domain_reader = segment_reader- .fast_fields()- .bytes(domain_field)- .expect("Failed to get domain fast-field reader");-- Ok(InitialSegmentScoreTweaker {- query: self.query.clone(),- centrality_reader,- domain_reader,- })- }-}--impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {- fn score(&mut self, doc: DocId, score: Score) -> f64 {- let centrality = self.centrality_reader.get(doc);- let navigational = self.navigational_score(self.query.terms.clone(), doc);- score as f64 + 1000.0 * centrality + 100.0 * navigational- }-}--fn jaccard_sim(mut terms: Vec<String>, domain_parts: &[&str]) -> f64 {- debug_assert_eq!(domain_parts.len(), 2);-- terms.sort();- terms.dedup();-- let intersection_size: f64 = terms- .iter()- .map(|term| {- if domain_parts.contains(&term.as_str()) {- 1.0- } else {- 0.0- }- })- .sum();-- intersection_size / (terms.len() as f64 + domain_parts.len() as f64 - intersection_size)-}--fn is_navigational(mut terms: Vec<String>, domain: &str) -> bool {- let (name, remaining) = domain.split_once(".").unwrap();- jaccard_sim(terms, &[name, remaining]) >= 0.5-}--pub(crate) fn initial_collector(- query: Query,-) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {- let score_tweaker = InitialScoreTweaker::new(query);- TopDocs::with_limit(20).tweak_score(score_tweaker)-}--#[cfg(test)]-mod tests {-- use super::*;- use crate::{- query::Query,- search_index::Index,- webpage::{Link, Webpage},- };-- #[test]- fn test_jaccard_sim() {- assert_eq!(- jaccard_sim(vec!["dr".to_string(), "dk".to_string()], &["dr", "dk"]),- 1.0- );-- assert_eq!(- jaccard_sim(- vec![- "dr".to_string(),- "dk".to_string(),- "dr".to_string(),- "dr".to_string()- ],- &["dr", "dk"]- ),- 1.0- );- assert_eq!(jaccard_sim(vec!["dr".to_string()], &["dr", "dk"]), 0.5);- }-- #[test]- fn harmonic_ranking() {- let query = Query::parse("great site").expect("Failed to parse query");-- for _ in 0..10 {- let mut index = Index::temporary().expect("Unable to open index");-- index- .insert(Webpage::new(- r#"- <html>- <head>- <title>Website A</title>- </head>- <a href="https://www.b.com">B site is great</a>- </html>- "#,- "https://www.a.com",- vec![],- 0.0,- ))- .expect("failed to parse webpage");- index- .insert(Webpage::new(- r#"- <html>- <head>- <title>Website B</title>- </head>- </html>- "#,- "https://www.b.com",- vec![Link {- source: "https://www.a.com".to_string(),- destination: "https://www.b.com".to_string(),- text: "B site is great".to_string(),- }],- 5.0,- ))- .expect("failed to parse webpage");-- index.commit().expect("failed to commit index");- let result = index.search(&query).expect("Search failed");- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.b.com");- assert_eq!(result.documents[1].url, "https://www.a.com");- }- }-- #[test]- fn navigational() {- assert!(is_navigational(vec!["dr".to_string()], "dr.dk"));- assert!(is_navigational(- vec!["dr".to_string(), "dk".to_string()],- "dr.dk"- ));- }-- #[test]- fn navigational_search() {- let query = Query::parse("dr dk").expect("Failed to parse query");-- let mut index = Index::temporary().expect("Unable to open index");-- index- .insert(Webpage::new(- r#"- <html>- <head>- <title>Website A</title>- </head>- </html>- "#,- "https://www.dr.dk",- vec![],- 0.0,- ))- .expect("failed to parse webpage");- index- .insert(Webpage::new(- r#"- <html>- <head>- <title>Website B</title>- dr dk dr dk dr dk dr dk- </head>- </html>- "#,- "https://www.b.com",- vec![],- 0.0003,- ))- .expect("failed to parse webpage");-- index.commit().expect("failed to commit index");- let result = index.search(&query).expect("Search failed");- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.dr.dk");- assert_eq!(result.documents[1].url, "https://www.b.com");- }-}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsnew file mode 100644index 00000000..433c2ee7--- /dev/null+++ b/src/ranking/initial.rs@@ -0,0 +1,146 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::query::Query;+use crate::schema::Field;+use std::sync::Arc;+use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};+use tantivy::fastfield::{BytesFastFieldReader, DynamicFastFieldReader, FastFieldReader};+use tantivy::{DocId, Score, SegmentReader};++pub(crate) struct InitialScoreTweaker {+ query: Arc<Query>,+}++impl InitialScoreTweaker {+ pub fn new(query: Arc<Query>) -> Self {+ Self { query }+ }+}++pub(crate) struct InitialSegmentScoreTweaker {+ centrality_reader: DynamicFastFieldReader<f64>,+ domain_reader: BytesFastFieldReader,+ sorted_dedupped_terms: Vec<String>,+}++impl InitialSegmentScoreTweaker {+ fn navigational_score(&self, sorted_dedupped_terms: &[String], doc: DocId) -> f64 {+ let bytes = self.domain_reader.get_bytes(doc);+ let domain = bincode::deserialize(bytes).expect("Failed to deserialize domain");++ if is_navigational(sorted_dedupped_terms, domain) {+ 1.0+ } else {+ 0.0+ }+ }+}++impl ScoreTweaker<f64> for InitialScoreTweaker {+ type Child = InitialSegmentScoreTweaker;++ fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {+ let centrality_field = segment_reader+ .schema()+ .get_field(Field::Centrality.as_str())+ .expect("Faild to load centrality field");+ let centrality_reader = segment_reader+ .fast_fields()+ .f64(centrality_field)+ .expect("Failed to get centrality fast-field reader");++ let domain_field = segment_reader+ .schema()+ .get_field(Field::Domain.as_str())+ .expect("Faild to load domain field");+ let domain_reader = segment_reader+ .fast_fields()+ .bytes(domain_field)+ .expect("Failed to get domain fast-field reader");++ let mut sorted_dedupped_terms = self.query.terms.clone();++ sorted_dedupped_terms.sort();+ sorted_dedupped_terms.dedup();++ Ok(InitialSegmentScoreTweaker {+ centrality_reader,+ domain_reader,+ sorted_dedupped_terms,+ })+ }+}++impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {+ fn score(&mut self, doc: DocId, score: Score) -> f64 {+ let centrality = self.centrality_reader.get(doc);+ let navigational = self.navigational_score(&self.sorted_dedupped_terms, doc);+ score as f64 + 1000.0 * centrality + 100.0 * navigational+ }+}++fn jaccard_sim(sorted_dedupped_terms: &[String], domain_parts: &[&str]) -> f64 {+ debug_assert_eq!(domain_parts.len(), 2);++ let intersection_size: f64 = sorted_dedupped_terms+ .iter()+ .map(|term| {+ if domain_parts.contains(&term.as_str()) {+ 1.0+ } else {+ 0.0+ }+ })+ .sum();++ intersection_size+ / (sorted_dedupped_terms.len() as f64 + domain_parts.len() as f64 - intersection_size)+}++fn is_navigational(sorted_dedupped_terms: &[String], domain: &str) -> bool {+ let (name, remaining) = domain.split_once('.').unwrap();+ jaccard_sim(sorted_dedupped_terms, &[name, remaining]) >= 0.5+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::{+ index::Index,+ query::Query,+ webpage::{Link, Webpage},+ };++ #[test]+ fn test_jaccard_sim() {+ assert_eq!(+ jaccard_sim(&["dr".to_string(), "dk".to_string()], &["dr", "dk"]),+ 1.0+ );++ assert_eq!(jaccard_sim(&["dr".to_string()], &["dr", "dk"]), 0.5);+ }++ #[test]+ fn navigational() {+ assert!(is_navigational(&["dr".to_string()], "dr.dk"));+ assert!(is_navigational(+ &["dk".to_string(), "dr".to_string()],+ "dr.dk"+ ));+ }+}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsnew file mode 100644index 00000000..c660a38c--- /dev/null+++ b/src/ranking/mod.rs@@ -0,0 +1,140 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod initial;++use std::sync::Arc;++use crate::query::Query;+use initial::InitialScoreTweaker;+use tantivy::collector::{Collector, TopDocs};++pub struct Ranker {+ query: Arc<Query>,+}++impl Ranker {+ pub fn new(query: Query) -> Self {+ Ranker {+ query: Arc::new(query),+ }+ }++ pub(crate) fn collector(&self) -> impl Collector<Fruit = Vec<(f64, tantivy::DocAddress)>> {+ let score_tweaker = InitialScoreTweaker::new(self.query.clone());+ TopDocs::with_limit(20).tweak_score(score_tweaker)+ }+}++#[cfg(test)]+mod tests {+ use crate::{+ index::Index,+ query::Query,+ searcher::Searcher,+ webpage::{Link, Webpage},+ };++ #[test]+ fn harmonic_ranking() {+ for _ in 0..10 {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ <a href="https://www.b.com">B site is great</a>+ </html>+ "#,+ "https://www.a.com",+ vec![],+ 0.0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website B</title>+ </head>+ </html>+ "#,+ "https://www.b.com",+ vec![Link {+ source: "https://www.a.com".to_string(),+ destination: "https://www.b.com".to_string(),+ text: "B site is great".to_string(),+ }],+ 5.0,+ ))+ .expect("failed to parse webpage");++ index.commit().expect("failed to commit index");+ let searcher = Searcher::from(index);+ let result = searcher.search("great site").expect("Search failed");+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.b.com");+ assert_eq!(result.documents[1].url, "https://www.a.com");+ }+ }++ #[test]+ fn navigational_search() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ </html>+ "#,+ "https://www.dr.dk",+ vec![],+ 0.0,+ ))+ .expect("failed to parse webpage");+ index+ .insert(Webpage::new(+ r#"+ <html>+ <head>+ <title>Website B</title>+ dr dk dr dk dr dk dr dk+ </head>+ </html>+ "#,+ "https://www.b.com",+ vec![],+ 0.0003,+ ))+ .expect("failed to parse webpage");++ index.commit().expect("failed to commit index");+ let searcher = Searcher::from(index);+ let result = searcher.search("dr dk").expect("Search failed");+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.dr.dk");+ assert_eq!(result.documents[1].url, "https://www.b.com");+ }+}diff --git a/src/searcher.rs b/src/searcher.rsnew file mode 100644index 00000000..c6862c47--- /dev/null+++ b/src/searcher.rs@@ -0,0 +1,48 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::index::{Index, RetrievedWebpage};+use crate::query::Query;+use crate::ranking::Ranker;+use crate::Result;++pub struct SearchResult {+ pub num_docs: usize,+ pub documents: Vec<RetrievedWebpage>,+}++pub struct Searcher {+ index: Index,+}++#[cfg(test)]+impl From<Index> for Searcher {+ fn from(index: Index) -> Self {+ Self::from_index(index)+ }+}++impl Searcher {+ fn from_index(index: Index) -> Self {+ Searcher { index }+ }++ pub fn search(&self, query: &str) -> Result<SearchResult> {+ let query = Query::parse(query)?;+ let ranker = Ranker::new(query.clone());+ self.index.search(&query, ranker.collector())+ }+}diff --git a/src/snippet.rs b/src/snippet.rsindex 5351116c..344f0990 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -14,7 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::{Document, Searcher, SnippetGenerator};+use tantivy::Searcher as TantivySearcher;+use tantivy::SnippetGenerator;use crate::{query::Query, schema::Field, Result};@@ -22,9 +23,9 @@ use crate::{query::Query, schema::Field, Result};/// In the future we want to implement something closer to the method described in https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf./// This will require us to store each paragraph of the webpage separately to get adequate performance./// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.-pub fn generate(query: &Query, text: &str, searcher: &Searcher) -> Result<String> {+pub fn generate(query: &Query, text: &str, searcher: &TantivySearcher) -> Result<String> {let generator = SnippetGenerator::create(- &searcher,+ searcher,&query.tantivy(searcher.schema(), searcher.index().tokenizers()),searcher.schema()@@ -40,7 +41,7 @@ pub fn generate(query: &Query, text: &str, searcher: &Searcher) -> Result<String#[cfg(test)]mod tests {- use crate::{search_index::Index, webpage::Webpage};+ use crate::{index::Index, searcher::Searcher, webpage::Webpage};use super::*;@@ -59,7 +60,6 @@ Survey in 2016, 2017, and 2018."#;#[test]fn snippet_during_search() {let mut index = Index::temporary().expect("Unable to open index");- let query = Query::parse("rust language").expect("Failed to parse query");index.insert(Webpage::new(@@ -81,7 +81,9 @@ Survey in 2016, 2017, and 2018."#;.expect("failed to parse webpage");index.commit().expect("failed to commit index");- let result = index.search(&query).expect("Search failed");+ let searcher = Searcher::from(index);++ let result = searcher.search("rust language").expect("Search failed");assert_eq!(result.num_docs, 1);assert_eq!(result.documents.len(), 1);diff --git a/src/webpage.rs b/src/webpage.rsindex 739dfcd7..ffbe4562 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -180,11 +180,11 @@ impl Html {let host = self.host();let num_punctuations: usize = host.chars().map(|c| if c == '.' { 1 } else { 0 }).sum();if num_punctuations > 1 {- let domain_index = host.rfind(".").unwrap();- let mut start_index = host[..domain_index].rfind(".").unwrap();+ let domain_index = host.rfind('.').unwrap();+ let mut start_index = host[..domain_index].rfind('.').unwrap();if &host[start_index + 1..] == "co.uk" {- start_index = host[start_index..].rfind(".").unwrap();+ start_index = host[start_index..].rfind('.').unwrap();}&host[start_index + 1..]