web search done right
goggle performance speedup
Mikkel Denker15fe347dc5d5
diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rsindex 40f64d82..85ae6c87 100644--- a/benches/search-preindexed-goggle.rs+++ b/benches/search-preindexed-goggle.rs@@ -22,12 +22,13 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();let searcher = Searcher::new(index, None, None);- let goggle = include_str!("../testcases/goggles/quickstart.goggle");+ let goggle = include_str!("../testcases/goggles/hacker_news.goggle");// for _ in 0..10 {bench!("the", searcher, goggle, c);bench!("dtu", searcher, goggle, c);bench!("the best", searcher, goggle, c);+ bench!("the circle of life", searcher, goggle, c);// }}diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rsindex 48a2f0e6..0033eca2 100644--- a/benches/search-preindexed.rs+++ b/benches/search-preindexed.rs@@ -18,11 +18,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();let searcher = Searcher::new(index, None, None);- for _ in 0..10 {- bench!("the", searcher, c);- bench!("dtu", searcher, c);- bench!("the best", searcher, c);- }+ // for _ in 0..100 {+ bench!("the", searcher, c);+ bench!("dtu", searcher, c);+ bench!("the best", searcher, c);+ bench!("the circle of life", searcher, c);+ // }}criterion_group!(benches, criterion_benchmark);diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex d59b1c0f..18b3d29f 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -58,7 +58,7 @@ impl InvertedIndex {tantivy::Index::create_in_dir(&path, schema.clone())?};- tantivy_index.set_default_multithread_executor()?;+ // tantivy_index.set_default_multithread_executor()?;let tokenizer = Tokenizer::default();tantivy_index@@ -70,7 +70,7 @@ impl InvertedIndex {.tokenizers().register(tokenizer.as_str(), tokenizer);- let writer = tantivy_index.writer_with_num_threads(1, 4_000_000_000)?;+ let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;let merge_policy = NoMergePolicy::default();writer.set_merge_policy(Box::new(merge_policy));@@ -366,9 +366,11 @@ impl From<Document> for RetrievedWebpage {| Field::HostCentrality| Field::PageCentrality| Field::Site+ | Field::Domain+ | Field::SiteNoTokenizer+ | Field::DomainNoTokenizer| Field::StemmedTitle| Field::CleanBody- | Field::Domain| Field::DomainIfHomepage| Field::IsHomepage| Field::SiteHashdiff --git a/src/query/mod.rs b/src/query/mod.rsindex 1d943ae8..39e3f593 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -24,6 +24,7 @@ use tantivy::{pub mod intersection;pub mod parser;+pub mod union;use parser::Term;const MAX_SIMILAR_TERMS: usize = 10;@@ -49,7 +50,7 @@ fn proximity_queries(let tantivy_field = schema.get_field(field.as_str()).unwrap();let tantivy_entry = schema.get_field_entry(tantivy_field);- for (boost, slop) in [(4, 0), (3, 4), (2, 16), (1, 32)] {+ for (boost, slop) in [(6, 0), (5, 1), (4, 2), (3, 4), (2, 16), (1, 32)] {let mut terms = Vec::new();let mut num_terms = 0;diff --git a/src/query/union.rs b/src/query/union.rsnew file mode 100644index 00000000..0323f79a--- /dev/null+++ b/src/query/union.rs@@ -0,0 +1,220 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{cell::RefCell, cmp::Ordering};++use min_max_heap::MinMaxHeap;+use tantivy::{+ query::{Explanation, Scorer},+ DocId, DocSet, TERMINATED,+};++struct DocsetHead<T> {+ docset: RefCell<T>,+ min_doc: DocId,+}++impl<T> PartialOrd for DocsetHead<T> {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.min_doc.partial_cmp(&other.min_doc)+ }+}++impl<T> Ord for DocsetHead<T> {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(Ordering::Equal)+ }+}++impl<T> PartialEq for DocsetHead<T> {+ fn eq(&self, other: &Self) -> bool {+ self.min_doc == other.min_doc+ }+}++impl<T> Eq for DocsetHead<T> {}++struct Union<T> {+ docsets: MinMaxHeap<DocsetHead<T>>,+}++impl<T: Scorer> From<Vec<T>> for Union<T> {+ fn from(docsets: Vec<T>) -> Self {+ let mut heap = MinMaxHeap::with_capacity(docsets.len());++ for docset in docsets+ .into_iter()+ .filter(|docset| docset.doc() != TERMINATED)+ {+ heap.push(DocsetHead {+ min_doc: docset.doc(),+ docset: RefCell::new(docset),+ })+ }++ Union { docsets: heap }+ }+}++impl<T: DocSet> DocSet for Union<T> {+ fn advance(&mut self) -> tantivy::DocId {+ let old_min = self.doc();++ if old_min == TERMINATED {+ return TERMINATED;+ }++ {+ loop {+ let head = self.docsets.peek_min().unwrap();++ if head.min_doc != old_min {+ break;+ }++ let mut head = self.docsets.peek_min_mut().unwrap();++ head.min_doc = head.docset.get_mut().advance();+ }+ }++ self.docsets.peek_min().unwrap().min_doc+ }++ fn seek(&mut self, target: DocId) -> DocId {+ let old_min = self.doc();++ if old_min > target {+ return old_min;+ }++ if old_min == TERMINATED {+ return TERMINATED;+ }++ {+ loop {+ let head = self.docsets.peek_min().unwrap();++ if head.min_doc != old_min || head.min_doc == target {+ break;+ }++ let mut head = self.docsets.peek_min_mut().unwrap();++ head.min_doc = head.docset.get_mut().seek(target);+ }+ }++ self.docsets.peek_min().unwrap().min_doc+ }++ fn doc(&self) -> tantivy::DocId {+ self.docsets+ .peek_min()+ .map(|head| head.min_doc)+ .unwrap_or(TERMINATED)+ }++ fn size_hint(&self) -> u32 {+ self.docsets+ .iter()+ .map(|head| head.docset.borrow().size_hint())+ .max()+ .unwrap_or(0)+ }+}++impl<T: Scorer> Scorer for Union<T> {+ fn score(&mut self) -> tantivy::Score {+ let cur_doc = self.doc();+ self.docsets+ .iter()+ .filter(|head| head.min_doc == cur_doc)+ .map(|head| head.docset.borrow_mut().score())+ .sum()+ }+}++#[derive(Debug)]+pub struct UnionQuery {+ subqueries: Vec<Box<dyn tantivy::query::Query>>,+}++impl From<Vec<Box<dyn tantivy::query::Query>>> for UnionQuery {+ fn from(subqueries: Vec<Box<dyn tantivy::query::Query>>) -> Self {+ Self { subqueries }+ }+}++impl Clone for UnionQuery {+ fn clone(&self) -> Self {+ self.subqueries+ .iter()+ .map(|subquery| subquery.box_clone())+ .collect::<Vec<_>>()+ .into()+ }+}++impl tantivy::query::Query for UnionQuery {+ fn weight(+ &self,+ searcher: &tantivy::Searcher,+ scoring_enabled: bool,+ ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {+ let mut weights = Vec::new();++ for query in &self.subqueries {+ weights.push(query.weight(searcher, scoring_enabled)?);+ }++ Ok(Box::new(UnionWeight { weights }))+ }+}++struct UnionWeight {+ weights: Vec<Box<dyn tantivy::query::Weight>>,+}++impl tantivy::query::Weight for UnionWeight {+ fn scorer(+ &self,+ reader: &tantivy::SegmentReader,+ boost: tantivy::Score,+ ) -> tantivy::Result<Box<dyn Scorer>> {+ let mut scorers = Vec::new();++ for weight in &self.weights {+ scorers.push(weight.scorer(reader, boost)?);+ }++ Ok(Box::new(Union::from(scorers)))+ }++ fn explain(&self, reader: &tantivy::SegmentReader, doc: DocId) -> tantivy::Result<Explanation> {+ let mut scorer = self.scorer(reader, 1.0)?;+ let mut explanation = Explanation::new("Union. Sum of ...", scorer.score());++ for weight in &self.weights {+ if let Ok(child_explanation) = weight.explain(reader, doc) {+ explanation.add_detail(child_explanation);+ }+ }++ Ok(explanation)+ }+}diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex 40505a7a..a18c88c0 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -19,12 +19,11 @@ mod pattern_query;use std::convert::TryFrom;-use crate::{schema::Field, tokenizer, Result};+use crate::{query::union::UnionQuery, schema::Field, Result};use itertools::Itertools;use tantivy::{- query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone, TermQuery},+ query::{BooleanQuery, BoostQuery, Occur, QueryClone, TermQuery},schema::{IndexRecordOption, Schema},- tokenizer::TextAnalyzer,};use self::{@@ -164,10 +163,11 @@ impl Goggle {{vec![(Occur::Must,- BooleanQuery::from(+ UnionQuery::from(self.instructions.iter()- .map(|instruction| instruction.as_tantivy(schema))+ .filter_map(|instruction| instruction.as_tantivy(schema))+ .map(|query| BooleanQuery::from(vec![query]).box_clone()).collect_vec(),).box_clone(),@@ -175,47 +175,22 @@ impl Goggle {} else {self.instructions.iter()- .map(|instruction| instruction.as_tantivy(schema))+ .filter_map(|instruction| instruction.as_tantivy(schema)).collect()}}}-fn process_tantivy_term(- term: &str,- analyzer: TextAnalyzer,- tantivy_field: tantivy::schema::Field,-) -> Vec<tantivy::Term> {- let mut terms: Vec<tantivy::Term> = Vec::new();- let mut token_stream = analyzer.token_stream(term);- token_stream.process(&mut |token| {- let term = tantivy::Term::from_field_text(tantivy_field, &token.text);- terms.push(term);- });-- terms-}-fn process_site(site: &str, field: tantivy::schema::Field) -> Box<dyn tantivy::query::Query> {- let mut terms = process_tantivy_term(- site,- TextAnalyzer::new(tokenizer::Normal::default(), vec![]),- field,- );-- if terms.len() > 1 {- Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>- } else {- let term = terms.pop().unwrap();- Box::new(TermQuery::new(- term,- IndexRecordOption::WithFreqsAndPositions,- ))- }+ let term = tantivy::Term::from_field_text(field, site);+ Box::new(TermQuery::new(+ term,+ IndexRecordOption::WithFreqsAndPositions,+ ))}impl Instruction {- pub fn as_tantivy(&self, schema: &Schema) -> (Occur, Box<dyn tantivy::query::Query>) {+ pub fn as_tantivy(&self, schema: &Schema) -> Option<(Occur, Box<dyn tantivy::query::Query>)> {let mut subqueries = Vec::new();let mut field = None;@@ -226,14 +201,14 @@ impl Instruction {PatternOption::Site(site) if field.is_none() => {field = Some(schema.get_field(Field::Site.as_str()).unwrap());- let domain_field = schema.get_field(Field::Domain.as_str()).unwrap();- let site_field = schema.get_field(Field::Site.as_str()).unwrap();+ let domain_field = schema.get_field(Field::DomainNoTokenizer.as_str()).unwrap();+ let site_field = schema.get_field(Field::SiteNoTokenizer.as_str()).unwrap();subqueries.push((Occur::Must,- BooleanQuery::new(vec![- (Occur::Should, process_site(site, domain_field)),- (Occur::Should, process_site(site, site_field)),+ UnionQuery::from(vec![+ process_site(site, domain_field),+ process_site(site, site_field),]).box_clone(),));@@ -265,24 +240,26 @@ impl Instruction {subqueries.push((Occur::Must, query));}+ if subqueries.is_empty() {+ return None;+ }++ let subquery = if subqueries.len() == 1 {+ subqueries.pop().unwrap().1+ } else {+ BooleanQuery::from(subqueries).box_clone()+ };+match action {- Action::Boost(boost) => (+ Action::Boost(boost) => Some((Occur::Should,- BoostQuery::new(- BooleanQuery::from(subqueries).box_clone(),- boost as f32 + 1.0,- )- .box_clone(),- ),- Action::Downrank(boost) => (+ BoostQuery::new(subquery, boost as f32 + 1.0).box_clone(),+ )),+ Action::Downrank(boost) => Some((Occur::Should,- BoostQuery::new(- BooleanQuery::from(subqueries).box_clone(),- 1.0 / (boost as f32 + 1.0),- )- .box_clone(),- ),- Action::Discard => (Occur::MustNot, BooleanQuery::from(subqueries).box_clone()),+ BoostQuery::new(subquery, 1.0 / (boost as f32 + 1.0)).box_clone(),+ )),+ Action::Discard => Some((Occur::MustNot, subquery)),}}diff --git a/src/schema.rs b/src/schema.rsindex c3c2b057..ceabaf9d 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -32,6 +32,8 @@ pub enum Field {Url,Site,Domain,+ SiteNoTokenizer,+ DomainNoTokenizer,/// this field is only set if the webpage is the homepage for the site. Allows us to boostDomainIfHomepage,IsHomepage,@@ -51,7 +53,7 @@ pub enum Field {SiteHash,}-pub static ALL_FIELDS: [Field; 24] = [+pub static ALL_FIELDS: [Field; 26] = [Field::Title,Field::CleanBody,Field::StemmedTitle,@@ -60,6 +62,8 @@ pub static ALL_FIELDS: [Field; 24] = [Field::Url,Field::Site,Field::Domain,+ Field::SiteNoTokenizer,+ Field::DomainNoTokenizer,Field::DomainIfHomepage,Field::IsHomepage,Field::BacklinkText,@@ -101,6 +105,18 @@ impl Field {Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),Field::Site => IndexingOption::Text(self.default_text_options()),Field::Domain => IndexingOption::Text(self.default_text_options()),+ Field::SiteNoTokenizer => IndexingOption::Text(+ TextOptions::default().set_indexing_options(+ TextFieldIndexing::default()+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),+ ),+ ),+ Field::DomainNoTokenizer => IndexingOption::Text(+ TextOptions::default().set_indexing_options(+ TextFieldIndexing::default()+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),+ ),+ ),Field::AllBody => IndexingOption::Text(self.default_text_options().set_stored()),Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),Field::IsHomepage => IndexingOption::Numeric(@@ -182,12 +198,14 @@ impl Field {Field::CleanBody => "body",Field::Url => "url",Field::Site => "site",+ Field::Domain => "domain",+ Field::SiteNoTokenizer => "site_no_tokenizer",+ Field::DomainNoTokenizer => "domain_no_tokenizer",Field::BacklinkText => "backlink_text",Field::HostCentrality => "host_centrality",Field::PageCentrality => "page_centrality",Field::StemmedTitle => "stemmed_title",Field::StemmedCleanBody => "stemmed_body",- Field::Domain => "domain",Field::DomainIfHomepage => "domain_if_homepage",Field::IsHomepage => "is_homepage",Field::FetchTimeMs => "fetch_time_ms",@@ -219,6 +237,8 @@ impl Field {Field::HostCentrality| Field::PageCentrality| Field::SiteHash+ | Field::SiteNoTokenizer+ | Field::DomainNoTokenizer| Field::IsHomepage| Field::PrimaryImage| Field::FetchTimeMsdiff --git a/src/warc.rs b/src/warc.rsindex c8632d39..ec5fd27f 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -20,6 +20,7 @@ use std::collections::BTreeMap;use std::fs::File;use std::io::{BufRead, BufReader, Cursor, Read, Seek, SeekFrom, Write};use std::path::Path;+use std::time::Duration;use flate2::read::MultiGzDecoder;use futures::StreamExt;@@ -139,7 +140,9 @@ impl WarcFile {let client = reqwest::ClientBuilder::new().tcp_keepalive(None)- .pool_idle_timeout(None)+ .pool_idle_timeout(Duration::from_secs(30 * 60))+ .timeout(Duration::from_secs(30 * 60))+ .connect_timeout(Duration::from_secs(30 * 60)).build()?;let res = client.get(url).send().await?;diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex f4fc2a8e..a669e41c 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -558,6 +558,32 @@ impl Html {Field::Url => doc.add_pre_tokenized_text(tantivy_field, url.clone()),Field::Site => doc.add_text(tantivy_field, self.url().site()),Field::Domain => doc.add_text(tantivy_field, self.url().domain()),+ Field::SiteNoTokenizer => doc.add_pre_tokenized_text(+ tantivy_field,+ PreTokenizedString {+ text: self.url().site().to_string(),+ tokens: vec![tantivy::tokenizer::Token {+ offset_from: 0,+ offset_to: self.url().site().len(),+ position: 0,+ text: self.url().site().to_string(),+ position_length: 1,+ }],+ },+ ),+ Field::DomainNoTokenizer => doc.add_pre_tokenized_text(+ tantivy_field,+ PreTokenizedString {+ text: self.url().domain().to_string(),+ tokens: vec![tantivy::tokenizer::Token {+ offset_from: 0,+ offset_to: self.url().domain().len(),+ position: 0,+ text: self.url().domain().to_string(),+ position_length: 1,+ }],+ },+ ),Field::DomainIfHomepage => {if self.url().is_homepage() {doc.add_text(tantivy_field, self.url().domain());