Skip to content

web search done right

goggle performance speedup

Mikkel Denker15fe347dc5d5
diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rs
index 40f64d82..85ae6c87 100644
--- a/benches/search-preindexed-goggle.rs
+++ b/benches/search-preindexed-goggle.rs
@@ -22,12 +22,13 @@ macro_rules! bench {
pub fn criterion_benchmark(c: &mut Criterion) {
let index = Index::open(INDEX_PATH).unwrap();
let searcher = Searcher::new(index, None, None);
- let goggle = include_str!("../testcases/goggles/quickstart.goggle");
+ let goggle = include_str!("../testcases/goggles/hacker_news.goggle");
// for _ in 0..10 {
bench!("the", searcher, goggle, c);
bench!("dtu", searcher, goggle, c);
bench!("the best", searcher, goggle, c);
+ bench!("the circle of life", searcher, goggle, c);
// }
}
diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rs
index 48a2f0e6..0033eca2 100644
--- a/benches/search-preindexed.rs
+++ b/benches/search-preindexed.rs
@@ -18,11 +18,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let index = Index::open(INDEX_PATH).unwrap();
let searcher = Searcher::new(index, None, None);
- for _ in 0..10 {
- bench!("the", searcher, c);
- bench!("dtu", searcher, c);
- bench!("the best", searcher, c);
- }
+ // for _ in 0..100 {
+ bench!("the", searcher, c);
+ bench!("dtu", searcher, c);
+ bench!("the best", searcher, c);
+ bench!("the circle of life", searcher, c);
+ // }
}
criterion_group!(benches, criterion_benchmark);
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index d59b1c0f..18b3d29f 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -58,7 +58,7 @@ impl InvertedIndex {
tantivy::Index::create_in_dir(&path, schema.clone())?
};
- tantivy_index.set_default_multithread_executor()?;
+ // tantivy_index.set_default_multithread_executor()?;
let tokenizer = Tokenizer::default();
tantivy_index
@@ -70,7 +70,7 @@ impl InvertedIndex {
.tokenizers()
.register(tokenizer.as_str(), tokenizer);
- let writer = tantivy_index.writer_with_num_threads(1, 4_000_000_000)?;
+ let writer = tantivy_index.writer_with_num_threads(1, 1_000_000_000)?;
let merge_policy = NoMergePolicy::default();
writer.set_merge_policy(Box::new(merge_policy));
@@ -366,9 +366,11 @@ impl From<Document> for RetrievedWebpage {
| Field::HostCentrality
| Field::PageCentrality
| Field::Site
+ | Field::Domain
+ | Field::SiteNoTokenizer
+ | Field::DomainNoTokenizer
| Field::StemmedTitle
| Field::CleanBody
- | Field::Domain
| Field::DomainIfHomepage
| Field::IsHomepage
| Field::SiteHash
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 1d943ae8..39e3f593 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -24,6 +24,7 @@ use tantivy::{
pub mod intersection;
pub mod parser;
+pub mod union;
use parser::Term;
const MAX_SIMILAR_TERMS: usize = 10;
@@ -49,7 +50,7 @@ fn proximity_queries(
let tantivy_field = schema.get_field(field.as_str()).unwrap();
let tantivy_entry = schema.get_field_entry(tantivy_field);
- for (boost, slop) in [(4, 0), (3, 4), (2, 16), (1, 32)] {
+ for (boost, slop) in [(6, 0), (5, 1), (4, 2), (3, 4), (2, 16), (1, 32)] {
let mut terms = Vec::new();
let mut num_terms = 0;
diff --git a/src/query/union.rs b/src/query/union.rs
new file mode 100644
index 00000000..0323f79a
--- /dev/null
+++ b/src/query/union.rs
@@ -0,0 +1,220 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{cell::RefCell, cmp::Ordering};
+
+use min_max_heap::MinMaxHeap;
+use tantivy::{
+ query::{Explanation, Scorer},
+ DocId, DocSet, TERMINATED,
+};
+
+struct DocsetHead<T> {
+ docset: RefCell<T>,
+ min_doc: DocId,
+}
+
+impl<T> PartialOrd for DocsetHead<T> {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.min_doc.partial_cmp(&other.min_doc)
+ }
+}
+
+impl<T> Ord for DocsetHead<T> {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.partial_cmp(other).unwrap_or(Ordering::Equal)
+ }
+}
+
+impl<T> PartialEq for DocsetHead<T> {
+ fn eq(&self, other: &Self) -> bool {
+ self.min_doc == other.min_doc
+ }
+}
+
+impl<T> Eq for DocsetHead<T> {}
+
+struct Union<T> {
+ docsets: MinMaxHeap<DocsetHead<T>>,
+}
+
+impl<T: Scorer> From<Vec<T>> for Union<T> {
+ fn from(docsets: Vec<T>) -> Self {
+ let mut heap = MinMaxHeap::with_capacity(docsets.len());
+
+ for docset in docsets
+ .into_iter()
+ .filter(|docset| docset.doc() != TERMINATED)
+ {
+ heap.push(DocsetHead {
+ min_doc: docset.doc(),
+ docset: RefCell::new(docset),
+ })
+ }
+
+ Union { docsets: heap }
+ }
+}
+
+impl<T: DocSet> DocSet for Union<T> {
+ fn advance(&mut self) -> tantivy::DocId {
+ let old_min = self.doc();
+
+ if old_min == TERMINATED {
+ return TERMINATED;
+ }
+
+ {
+ loop {
+ let head = self.docsets.peek_min().unwrap();
+
+ if head.min_doc != old_min {
+ break;
+ }
+
+ let mut head = self.docsets.peek_min_mut().unwrap();
+
+ head.min_doc = head.docset.get_mut().advance();
+ }
+ }
+
+ self.docsets.peek_min().unwrap().min_doc
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ let old_min = self.doc();
+
+ if old_min > target {
+ return old_min;
+ }
+
+ if old_min == TERMINATED {
+ return TERMINATED;
+ }
+
+ {
+ loop {
+ let head = self.docsets.peek_min().unwrap();
+
+ if head.min_doc != old_min || head.min_doc == target {
+ break;
+ }
+
+ let mut head = self.docsets.peek_min_mut().unwrap();
+
+ head.min_doc = head.docset.get_mut().seek(target);
+ }
+ }
+
+ self.docsets.peek_min().unwrap().min_doc
+ }
+
+ fn doc(&self) -> tantivy::DocId {
+ self.docsets
+ .peek_min()
+ .map(|head| head.min_doc)
+ .unwrap_or(TERMINATED)
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.docsets
+ .iter()
+ .map(|head| head.docset.borrow().size_hint())
+ .max()
+ .unwrap_or(0)
+ }
+}
+
+impl<T: Scorer> Scorer for Union<T> {
+ fn score(&mut self) -> tantivy::Score {
+ let cur_doc = self.doc();
+ self.docsets
+ .iter()
+ .filter(|head| head.min_doc == cur_doc)
+ .map(|head| head.docset.borrow_mut().score())
+ .sum()
+ }
+}
+
+#[derive(Debug)]
+pub struct UnionQuery {
+ subqueries: Vec<Box<dyn tantivy::query::Query>>,
+}
+
+impl From<Vec<Box<dyn tantivy::query::Query>>> for UnionQuery {
+ fn from(subqueries: Vec<Box<dyn tantivy::query::Query>>) -> Self {
+ Self { subqueries }
+ }
+}
+
+impl Clone for UnionQuery {
+ fn clone(&self) -> Self {
+ self.subqueries
+ .iter()
+ .map(|subquery| subquery.box_clone())
+ .collect::<Vec<_>>()
+ .into()
+ }
+}
+
+impl tantivy::query::Query for UnionQuery {
+ fn weight(
+ &self,
+ searcher: &tantivy::Searcher,
+ scoring_enabled: bool,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
+ let mut weights = Vec::new();
+
+ for query in &self.subqueries {
+ weights.push(query.weight(searcher, scoring_enabled)?);
+ }
+
+ Ok(Box::new(UnionWeight { weights }))
+ }
+}
+
+struct UnionWeight {
+ weights: Vec<Box<dyn tantivy::query::Weight>>,
+}
+
+impl tantivy::query::Weight for UnionWeight {
+ fn scorer(
+ &self,
+ reader: &tantivy::SegmentReader,
+ boost: tantivy::Score,
+ ) -> tantivy::Result<Box<dyn Scorer>> {
+ let mut scorers = Vec::new();
+
+ for weight in &self.weights {
+ scorers.push(weight.scorer(reader, boost)?);
+ }
+
+ Ok(Box::new(Union::from(scorers)))
+ }
+
+ fn explain(&self, reader: &tantivy::SegmentReader, doc: DocId) -> tantivy::Result<Explanation> {
+ let mut scorer = self.scorer(reader, 1.0)?;
+ let mut explanation = Explanation::new("Union. Sum of ...", scorer.score());
+
+ for weight in &self.weights {
+ if let Ok(child_explanation) = weight.explain(reader, doc) {
+ explanation.add_detail(child_explanation);
+ }
+ }
+
+ Ok(explanation)
+ }
+}
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 40505a7a..a18c88c0 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -19,12 +19,11 @@ mod pattern_query;
use std::convert::TryFrom;
-use crate::{schema::Field, tokenizer, Result};
+use crate::{query::union::UnionQuery, schema::Field, Result};
use itertools::Itertools;
use tantivy::{
- query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, QueryClone, TermQuery},
+ query::{BooleanQuery, BoostQuery, Occur, QueryClone, TermQuery},
schema::{IndexRecordOption, Schema},
- tokenizer::TextAnalyzer,
};
use self::{
@@ -164,10 +163,11 @@ impl Goggle {
{
vec![(
Occur::Must,
- BooleanQuery::from(
+ UnionQuery::from(
self.instructions
.iter()
- .map(|instruction| instruction.as_tantivy(schema))
+ .filter_map(|instruction| instruction.as_tantivy(schema))
+ .map(|query| BooleanQuery::from(vec![query]).box_clone())
.collect_vec(),
)
.box_clone(),
@@ -175,47 +175,22 @@ impl Goggle {
} else {
self.instructions
.iter()
- .map(|instruction| instruction.as_tantivy(schema))
+ .filter_map(|instruction| instruction.as_tantivy(schema))
.collect()
}
}
}
-fn process_tantivy_term(
- term: &str,
- analyzer: TextAnalyzer,
- tantivy_field: tantivy::schema::Field,
-) -> Vec<tantivy::Term> {
- let mut terms: Vec<tantivy::Term> = Vec::new();
- let mut token_stream = analyzer.token_stream(term);
- token_stream.process(&mut |token| {
- let term = tantivy::Term::from_field_text(tantivy_field, &token.text);
- terms.push(term);
- });
-
- terms
-}
-
fn process_site(site: &str, field: tantivy::schema::Field) -> Box<dyn tantivy::query::Query> {
- let mut terms = process_tantivy_term(
- site,
- TextAnalyzer::new(tokenizer::Normal::default(), vec![]),
- field,
- );
-
- if terms.len() > 1 {
- Box::new(PhraseQuery::new(terms)) as Box<dyn tantivy::query::Query>
- } else {
- let term = terms.pop().unwrap();
- Box::new(TermQuery::new(
- term,
- IndexRecordOption::WithFreqsAndPositions,
- ))
- }
+ let term = tantivy::Term::from_field_text(field, site);
+ Box::new(TermQuery::new(
+ term,
+ IndexRecordOption::WithFreqsAndPositions,
+ ))
}
impl Instruction {
- pub fn as_tantivy(&self, schema: &Schema) -> (Occur, Box<dyn tantivy::query::Query>) {
+ pub fn as_tantivy(&self, schema: &Schema) -> Option<(Occur, Box<dyn tantivy::query::Query>)> {
let mut subqueries = Vec::new();
let mut field = None;
@@ -226,14 +201,14 @@ impl Instruction {
PatternOption::Site(site) if field.is_none() => {
field = Some(schema.get_field(Field::Site.as_str()).unwrap());
- let domain_field = schema.get_field(Field::Domain.as_str()).unwrap();
- let site_field = schema.get_field(Field::Site.as_str()).unwrap();
+ let domain_field = schema.get_field(Field::DomainNoTokenizer.as_str()).unwrap();
+ let site_field = schema.get_field(Field::SiteNoTokenizer.as_str()).unwrap();
subqueries.push((
Occur::Must,
- BooleanQuery::new(vec![
- (Occur::Should, process_site(site, domain_field)),
- (Occur::Should, process_site(site, site_field)),
+ UnionQuery::from(vec![
+ process_site(site, domain_field),
+ process_site(site, site_field),
])
.box_clone(),
));
@@ -265,24 +240,26 @@ impl Instruction {
subqueries.push((Occur::Must, query));
}
+ if subqueries.is_empty() {
+ return None;
+ }
+
+ let subquery = if subqueries.len() == 1 {
+ subqueries.pop().unwrap().1
+ } else {
+ BooleanQuery::from(subqueries).box_clone()
+ };
+
match action {
- Action::Boost(boost) => (
+ Action::Boost(boost) => Some((
Occur::Should,
- BoostQuery::new(
- BooleanQuery::from(subqueries).box_clone(),
- boost as f32 + 1.0,
- )
- .box_clone(),
- ),
- Action::Downrank(boost) => (
+ BoostQuery::new(subquery, boost as f32 + 1.0).box_clone(),
+ )),
+ Action::Downrank(boost) => Some((
Occur::Should,
- BoostQuery::new(
- BooleanQuery::from(subqueries).box_clone(),
- 1.0 / (boost as f32 + 1.0),
- )
- .box_clone(),
- ),
- Action::Discard => (Occur::MustNot, BooleanQuery::from(subqueries).box_clone()),
+ BoostQuery::new(subquery, 1.0 / (boost as f32 + 1.0)).box_clone(),
+ )),
+ Action::Discard => Some((Occur::MustNot, subquery)),
}
}
diff --git a/src/schema.rs b/src/schema.rs
index c3c2b057..ceabaf9d 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -32,6 +32,8 @@ pub enum Field {
Url,
Site,
Domain,
+ SiteNoTokenizer,
+ DomainNoTokenizer,
/// this field is only set if the webpage is the homepage for the site. Allows us to boost
DomainIfHomepage,
IsHomepage,
@@ -51,7 +53,7 @@ pub enum Field {
SiteHash,
}
-pub static ALL_FIELDS: [Field; 24] = [
+pub static ALL_FIELDS: [Field; 26] = [
Field::Title,
Field::CleanBody,
Field::StemmedTitle,
@@ -60,6 +62,8 @@ pub static ALL_FIELDS: [Field; 24] = [
Field::Url,
Field::Site,
Field::Domain,
+ Field::SiteNoTokenizer,
+ Field::DomainNoTokenizer,
Field::DomainIfHomepage,
Field::IsHomepage,
Field::BacklinkText,
@@ -101,6 +105,18 @@ impl Field {
Field::Url => IndexingOption::Text(self.default_text_options().set_stored()),
Field::Site => IndexingOption::Text(self.default_text_options()),
Field::Domain => IndexingOption::Text(self.default_text_options()),
+ Field::SiteNoTokenizer => IndexingOption::Text(
+ TextOptions::default().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ ),
+ ),
+ Field::DomainNoTokenizer => IndexingOption::Text(
+ TextOptions::default().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+ ),
+ ),
Field::AllBody => IndexingOption::Text(self.default_text_options().set_stored()),
Field::DomainIfHomepage => IndexingOption::Text(self.default_text_options()),
Field::IsHomepage => IndexingOption::Numeric(
@@ -182,12 +198,14 @@ impl Field {
Field::CleanBody => "body",
Field::Url => "url",
Field::Site => "site",
+ Field::Domain => "domain",
+ Field::SiteNoTokenizer => "site_no_tokenizer",
+ Field::DomainNoTokenizer => "domain_no_tokenizer",
Field::BacklinkText => "backlink_text",
Field::HostCentrality => "host_centrality",
Field::PageCentrality => "page_centrality",
Field::StemmedTitle => "stemmed_title",
Field::StemmedCleanBody => "stemmed_body",
- Field::Domain => "domain",
Field::DomainIfHomepage => "domain_if_homepage",
Field::IsHomepage => "is_homepage",
Field::FetchTimeMs => "fetch_time_ms",
@@ -219,6 +237,8 @@ impl Field {
Field::HostCentrality
| Field::PageCentrality
| Field::SiteHash
+ | Field::SiteNoTokenizer
+ | Field::DomainNoTokenizer
| Field::IsHomepage
| Field::PrimaryImage
| Field::FetchTimeMs
diff --git a/src/warc.rs b/src/warc.rs
index c8632d39..ec5fd27f 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -20,6 +20,7 @@ use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Cursor, Read, Seek, SeekFrom, Write};
use std::path::Path;
+use std::time::Duration;
use flate2::read::MultiGzDecoder;
use futures::StreamExt;
@@ -139,7 +140,9 @@ impl WarcFile {
let client = reqwest::ClientBuilder::new()
.tcp_keepalive(None)
- .pool_idle_timeout(None)
+ .pool_idle_timeout(Duration::from_secs(30 * 60))
+ .timeout(Duration::from_secs(30 * 60))
+ .connect_timeout(Duration::from_secs(30 * 60))
.build()?;
let res = client.get(url).send().await?;
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index f4fc2a8e..a669e41c 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -558,6 +558,32 @@ impl Html {
Field::Url => doc.add_pre_tokenized_text(tantivy_field, url.clone()),
Field::Site => doc.add_text(tantivy_field, self.url().site()),
Field::Domain => doc.add_text(tantivy_field, self.url().domain()),
+ Field::SiteNoTokenizer => doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: self.url().site().to_string(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: self.url().site().len(),
+ position: 0,
+ text: self.url().site().to_string(),
+ position_length: 1,
+ }],
+ },
+ ),
+ Field::DomainNoTokenizer => doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: self.url().domain().to_string(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: self.url().domain().len(),
+ position: 0,
+ text: self.url().domain().to_string(),
+ position_length: 1,
+ }],
+ },
+ ),
Field::DomainIfHomepage => {
if self.url().is_homepage() {
doc.add_text(tantivy_field, self.url().domain());