Skip to content

web search done right

Remove term proximity ranking (#20)

Mikkel Denker22468e434aaa
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index bb6e6431..e2c03061 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,7 +1,11 @@
warc_paths_file = "warc.paths"
centrality_store_path = "./data/centrality"
+limit_warc_files = 10
[warc_source]
-type = "Local"
-folder = "./data/warc_files"
-names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
\ No newline at end of file
+# type = "Local"
+# folder = "./data/warc_files"
+# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
+type = "HTTP"
+base_url = "https://data.commoncrawl.org/"
+warc_paths_file = "warc.paths"
diff --git a/justfile b/justfile
index 800ca07e..f2a197ee 100644
--- a/justfile
+++ b/justfile
@@ -19,7 +19,7 @@
wget -nc -P data/warc_files https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-05/segments/1642320299852.23/warc/CC-MAIN-20220116093137-20220116123137-00049.warc.gz
rm -rf data/index/
cargo run --release -- indexer local configs/indexer/local.toml
- mv data/index/CC-MAIN-20220116093137-20220116123137-00049.warc.gz/* data/index/
+ mv data/index/CC-MAIN-*/* data/index/
just frontend
@entity:
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index bcda8372..75e3b4f8 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -190,6 +190,7 @@ impl Indexer {
warc_paths
.into_iter()
+ .take(config.limit_warc_files.unwrap_or(usize::MAX))
.map(|path| Job {
config: job_config.clone(),
warc_path: path,
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 094ceaf0..b732bcd5 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -708,84 +708,6 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.example.com");
}
- #[test]
- fn proximity_ranking() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse("termA termB", index.schema()).expect("Failed to parse query");
- let ranker = Ranker::new(query.clone());
-
- index
- .insert(Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT} termA termB d d d d d d d d d
- </body>
- </html>
- "#
- ),
- "https://www.first.com",
- vec![],
- 1.0,
- 500,
- ))
- .expect("failed to parse webpage");
- index
- .insert(Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT} termA d d d d d d d d d termB
- </body>
- </html>
- "#
- ),
- "https://www.third.com",
- vec![],
- 1.0,
- 500,
- ))
- .expect("failed to parse webpage");
- index
- .insert(Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT} termA d d d d termB d d d d d
- </body>
- </html>
- "#
- ),
- "https://www.second.com",
- vec![],
- 1.0,
- 500,
- ))
- .expect("failed to parse webpage");
- index.commit().expect("failed to commit index");
-
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
- assert_eq!(result.num_docs, 3);
- assert_eq!(result.documents.len(), 3);
- assert_eq!(result.documents[0].url, "https://www.first.com");
- assert_eq!(result.documents[1].url, "https://www.second.com");
- assert_eq!(result.documents[2].url, "https://www.third.com");
- }
-
#[test]
fn fetch_time_ranking() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
diff --git a/src/lib.rs b/src/lib.rs
index 86b90356..787cd41f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -60,6 +60,7 @@ pub struct IndexingMasterConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingLocalConfig {
+ limit_warc_files: Option<usize>,
warc_source: WarcSource,
centrality_store_path: String,
}
diff --git a/src/query/bm25.rs b/src/query/bm25.rs
deleted file mode 100644
index a88ab939..00000000
--- a/src/query/bm25.rs
+++ /dev/null
@@ -1,142 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use serde::{Deserialize, Serialize};
-
-use tantivy::fieldnorm::FieldNormReader;
-use tantivy::query::Explanation;
-use tantivy::{Score, Searcher, Term};
-
-const K1: Score = 1.2;
-const B: Score = 0.75;
-
-pub(crate) fn idf(doc_freq: u64, doc_count: u64) -> Score {
- assert!(doc_count >= doc_freq, "{} >= {}", doc_count, doc_freq);
- let x = ((doc_count - doc_freq) as Score + 0.5) / (doc_freq as Score + 0.5);
- (1.0 + x).ln()
-}
-
-fn cached_tf_component(fieldnorm: u32, average_fieldnorm: Score) -> Score {
- K1 * (1.0 - B + B * fieldnorm as Score / average_fieldnorm)
-}
-
-fn compute_tf_cache(average_fieldnorm: Score) -> [Score; 256] {
- let mut cache: [Score; 256] = [0.0; 256];
- for (fieldnorm_id, cache_mut) in cache.iter_mut().enumerate() {
- let fieldnorm = FieldNormReader::id_to_fieldnorm(fieldnorm_id as u8);
- *cache_mut = cached_tf_component(fieldnorm, average_fieldnorm);
- }
- cache
-}
-
-#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
-pub struct Bm25Params {
- pub idf: Score,
- pub avg_fieldnorm: Score,
-}
-
-#[derive(Clone)]
-pub struct Bm25Weight {
- idf_explain: Explanation,
- weight: Score,
- cache: [Score; 256],
- average_fieldnorm: Score,
-}
-
-impl Bm25Weight {
- pub fn boost_by(&self, boost: Score) -> Bm25Weight {
- Bm25Weight {
- idf_explain: self.idf_explain.clone(),
- weight: self.weight * boost,
- cache: self.cache,
- average_fieldnorm: self.average_fieldnorm,
- }
- }
-
- pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> tantivy::Result<Bm25Weight> {
- assert!(!terms.is_empty(), "Bm25 requires at least one term");
- let field = terms[0].field();
- for term in &terms[1..] {
- assert_eq!(
- term.field(),
- field,
- "All terms must belong to the same field."
- );
- }
-
- let mut total_num_tokens = 0u64;
- let mut total_num_docs = 0u64;
- for segment_reader in searcher.segment_readers() {
- let inverted_index = segment_reader.inverted_index(field)?;
- total_num_tokens += inverted_index.total_num_tokens();
- total_num_docs += u64::from(segment_reader.max_doc());
- }
- let average_fieldnorm = total_num_tokens as Score / total_num_docs as Score;
-
- if terms.len() == 1 {
- let term_doc_freq = searcher.doc_freq(&terms[0])?;
- Ok(Bm25Weight::for_one_term(
- term_doc_freq,
- total_num_docs,
- average_fieldnorm,
- ))
- } else {
- let mut idf_sum: Score = 0.0;
- for term in terms {
- let term_doc_freq = searcher.doc_freq(term)?;
- idf_sum += idf(term_doc_freq, total_num_docs);
- }
- let idf_explain = Explanation::new("idf", idf_sum);
- Ok(Bm25Weight::new(idf_explain, average_fieldnorm))
- }
- }
-
- pub fn for_one_term(
- term_doc_freq: u64,
- total_num_docs: u64,
- avg_fieldnorm: Score,
- ) -> Bm25Weight {
- let idf = idf(term_doc_freq, total_num_docs);
- let mut idf_explain =
- Explanation::new("idf, computed as log(1 + (N - n + 0.5) / (n + 0.5))", idf);
- idf_explain.add_const(
- "n, number of docs containing this term",
- term_doc_freq as Score,
- );
- idf_explain.add_const("N, total number of docs", total_num_docs as Score);
- Bm25Weight::new(idf_explain, avg_fieldnorm)
- }
-
- pub(crate) fn new(idf_explain: Explanation, average_fieldnorm: Score) -> Bm25Weight {
- let weight = idf_explain.value() * (1.0 + K1);
- Bm25Weight {
- idf_explain,
- weight,
- cache: compute_tf_cache(average_fieldnorm),
- average_fieldnorm,
- }
- }
-
- #[inline]
- pub fn score(&self, fieldnorm_id: u8, term_freq: Score) -> Score {
- self.weight * self.tf_factor(fieldnorm_id, term_freq)
- }
-
- #[inline]
- pub(crate) fn tf_factor(&self, fieldnorm_id: u8, term_freq: Score) -> Score {
- let norm = self.cache[fieldnorm_id as usize];
- term_freq / (term_freq + norm)
- }
-}
diff --git a/src/query/field_union.rs b/src/query/field_union.rs
deleted file mode 100644
index f8a4b0a3..00000000
--- a/src/query/field_union.rs
+++ /dev/null
@@ -1,73 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::query::Scorer;
-use tantivy::{DocId, DocSet, Score, TERMINATED};
-
-use super::term_scorer::TermScorerForField;
-
-/// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.
-pub struct FieldUnion {
- pub docsets: Vec<TermScorerForField>,
-}
-
-impl From<Vec<TermScorerForField>> for FieldUnion {
- fn from(docsets: Vec<TermScorerForField>) -> FieldUnion {
- FieldUnion { docsets }
- }
-}
-
-impl DocSet for FieldUnion {
- fn advance(&mut self) -> DocId {
- let current_doc = self.doc();
-
- for scorer in self
- .docsets
- .iter_mut()
- .filter(|scorer| scorer.doc() == current_doc)
- {
- scorer.advance();
- }
-
- self.doc()
- }
-
- fn doc(&self) -> DocId {
- self.docsets
- .iter()
- .map(|scorer| scorer.doc())
- .min()
- .unwrap_or(TERMINATED)
- }
-
- fn size_hint(&self) -> u32 {
- self.docsets
- .iter()
- .map(|docset| docset.size_hint())
- .max()
- .unwrap_or(0u32)
- }
-}
-
-impl Scorer for FieldUnion {
- fn score(&mut self) -> Score {
- let current_doc = self.doc();
- self.docsets
- .iter_mut()
- .filter(|scorer| scorer.doc() == current_doc)
- .map(|scorer| scorer.score())
- .sum()
- }
-}
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 03d58753..3910ab24 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -14,47 +14,25 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{query::weight::Weight, schema::ALL_FIELDS, tokenizer::NormalTokenizer, Result};
+use crate::{schema::ALL_FIELDS, tokenizer::NormalTokenizer, Result};
use std::{
collections::{BTreeMap, HashMap},
sync::Arc,
};
use tantivy::{
+ query::{BooleanQuery, BoostQuery, Occur, TermQuery},
schema::{FieldType, IndexRecordOption, Schema},
tokenizer::{TextAnalyzer, Tokenizer},
};
-use self::bm25::Bm25Weight;
use lalrpop_util::lalrpop_mod;
lalrpop_mod!(pub parser, "/query/parser.rs"); // synthesized by LALRPOP
-mod bm25;
-mod field_union;
-mod term_intersection;
-mod term_scorer;
-mod vec_docset;
-mod weight;
-
static PARSER: once_cell::sync::Lazy<parser::TermsParser> =
once_cell::sync::Lazy::new(parser::TermsParser::new);
const MAX_SIMILAR_TERMS: usize = 10;
-pub struct FieldData {
- tantivy: tantivy::schema::Field,
- scoring: Bm25Weight,
- index_record_option: Option<IndexRecordOption>,
- boost: Option<f32>,
- analyzer: Option<TextAnalyzer>,
-}
-impl std::fmt::Debug for FieldData {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("FieldData")
- .field("tanitvy", &self.tantivy)
- .finish()
- }
-}
-
#[derive(Debug)]
pub enum Term<'a> {
Simple(&'a str),
@@ -101,6 +79,43 @@ impl Query {
pub fn simple_terms(&self) -> &[String] {
&self.terms
}
+
+ fn get_tantivy_analyzer(
+ entry: &tantivy::schema::FieldEntry,
+ tokenizer_manager: &tantivy::tokenizer::TokenizerManager,
+ ) -> Option<TextAnalyzer> {
+ match entry.field_type() {
+ tantivy::schema::FieldType::Str(options) => {
+ options.get_indexing_options().map(|indexing_options| {
+ let tokenizer_name = indexing_options.tokenizer();
+ tokenizer_manager
+ .get(tokenizer_name)
+ .expect("Unknown tokenizer")
+ })
+ }
+ _ => None,
+ }
+ }
+
+ fn process_tantivy_term(
+ term: &str,
+ analyzer: Option<TextAnalyzer>,
+ tantivy_field: tantivy::schema::Field,
+ ) -> impl Iterator<Item = tantivy::Term> {
+ match analyzer {
+ None => vec![tantivy::Term::from_field_text(tantivy_field, term)].into_iter(),
+ Some(tokenizer) => {
+ let mut terms: Vec<tantivy::Term> = Vec::new();
+ let mut token_stream = tokenizer.token_stream(term);
+ token_stream.process(&mut |token| {
+ let term = tantivy::Term::from_field_text(tantivy_field, &token.text);
+ terms.push(term);
+ });
+
+ terms.into_iter()
+ }
+ }
+ }
}
impl tantivy::query::Query for Query {
@@ -109,54 +124,53 @@ impl tantivy::query::Query for Query {
searcher: &tantivy::Searcher,
scoring_enabled: bool,
) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
- let terms = self.terms.clone();
+ let schema = searcher.schema();
+ let tokenizer_manager = searcher.index().tokenizers();
- let fields: Vec<_> = ALL_FIELDS
+ let queries = self
+ .terms
.iter()
- .filter(|field| field.is_searchable())
- .map(|field| {
- let tantivy_field = searcher.schema().get_field(field.as_str()).unwrap();
- let analyzer = searcher.index().tokenizer_for_field(tantivy_field).ok();
-
- let tantivy_terms: Vec<_> = terms
- .iter()
- .flat_map(|term_text| {
- let terms = analyzer
- .as_ref()
- .map(|analyzer| {
- let mut terms = Vec::new();
-
- let mut stream = analyzer.token_stream(term_text);
-
- while let Some(token) = stream.next() {
- terms.push(token.text.clone());
- }
-
- terms
+ .map(|term| {
+ let one_term = schema
+ .fields()
+ .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())
+ .into_iter()
+ .map(|(field, entry)| {
+ let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);
+ let processed_terms = Query::process_tantivy_term(term, analyzer, field);
+
+ let processed_queries = processed_terms
+ .map(|term| {
+ (
+ Occur::Should,
+ Box::new(TermQuery::new(
+ term,
+ IndexRecordOption::WithFreqsAndPositions,
+ ))
+ as Box<dyn tantivy::query::Query>,
+ )
})
- .unwrap_or_else(|| vec![term_text.clone()]);
-
- terms
- .into_iter()
- .map(|term| tantivy::Term::from_field_text(tantivy_field, &term))
+ .collect();
+ let boost = ALL_FIELDS[field.field_id() as usize].boost().unwrap_or(1.0);
+
+ (
+ Occur::Should,
+ Box::new(BoostQuery::new(
+ Box::new(BooleanQuery::new(processed_queries)),
+ boost,
+ )) as Box<dyn tantivy::query::Query>,
+ )
})
.collect();
- FieldData {
- tantivy: tantivy_field,
- index_record_option: searcher
- .schema()
- .get_field_entry(tantivy_field)
- .field_type()
- .get_index_record_option(),
- boost: field.boost(),
- scoring: Bm25Weight::for_terms(searcher, &tantivy_terms).unwrap(),
- analyzer,
- }
+ Box::new(BooleanQuery::new(one_term)) as Box<dyn tantivy::query::Query>
})
+ .map(|term_query| (Occur::Must, term_query))
.collect();
- Ok(Box::new(Weight::new(terms, fields, scoring_enabled)))
+ let query = Box::new(BooleanQuery::new(queries));
+
+ query.weight(searcher, scoring_enabled)
}
fn query_terms(&self, terms: &mut BTreeMap<tantivy::Term, bool>) {
diff --git a/src/query/term_intersection.rs b/src/query/term_intersection.rs
deleted file mode 100644
index c9f96f8e..00000000
--- a/src/query/term_intersection.rs
+++ /dev/null
@@ -1,520 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use itertools::Itertools;
-use tantivy::query::{EmptyScorer, Scorer};
-use tantivy::{DocId, DocSet, Postings, Score, TERMINATED};
-
-use super::field_union::FieldUnion;
-
-const MAX_WIDTH: u32 = 25;
-const LAMBDA: f32 = 0.75;
-const GAMMA: f32 = 0.25;
-
-pub fn intersect_terms(mut scorers: Vec<FieldUnion>) -> Box<dyn Scorer> {
- if scorers.is_empty() {
- return Box::new(EmptyScorer);
- }
- scorers.sort_by_key(|scorer| scorer.size_hint());
- let doc = go_to_first_doc(&mut scorers[..]);
- if doc == TERMINATED {
- return Box::new(EmptyScorer);
- }
- Box::new(TermIntersection { docsets: scorers })
-}
-
-pub struct TermIntersection {
- docsets: Vec<FieldUnion>,
-}
-
-fn go_to_first_doc<TDocSet: DocSet>(docsets: &mut [TDocSet]) -> DocId {
- assert!(!docsets.is_empty());
- let mut candidate = docsets.iter().map(TDocSet::doc).max().unwrap();
- 'outer: loop {
- for docset in docsets.iter_mut() {
- let seek_doc = docset.seek(candidate);
- if seek_doc > candidate {
- candidate = docset.doc();
- continue 'outer;
- }
- }
- return candidate;
- }
-}
-
-impl DocSet for TermIntersection {
- fn advance(&mut self) -> DocId {
- let mut candidate = self.docsets[0].advance();
-
- 'outer: loop {
- for docset in self.docsets.iter_mut() {
- let seek_doc = docset.seek(candidate);
- if seek_doc > candidate {
- candidate = self.docsets[0].seek(seek_doc);
- continue 'outer;
- }
- }
- debug_assert!(self.docsets.iter().all(|docset| docset.doc() == candidate));
- return candidate;
- }
- }
-
- fn seek(&mut self, target: DocId) -> DocId {
- self.docsets[0].seek(target);
- let doc = go_to_first_doc(&mut self.docsets);
- debug_assert!(self.docsets.iter().all(|docset| docset.doc() == doc));
- debug_assert!(doc >= target);
- doc
- }
-
- fn doc(&self) -> DocId {
- self.docsets[0].doc()
- }
-
- fn size_hint(&self) -> u32 {
- self.docsets[0].size_hint()
- }
-}
-
-/// modified bm25 score based on term proximity.
-/// more details in the papers "Viewing Term Proximity from a Different Perspective"
-/// and "A Short Note on Proximity-based Scoring of Documents with Multiple Fields"
-impl Scorer for TermIntersection {
- fn score(&mut self) -> Score {
- if self.doc() == TERMINATED {
- return 0.0;
- }
-
- let num_docsets = self.docsets.len();
-
- let postings = self
- .docsets
- .iter_mut()
- .map(|docset| {
- docset
- .docsets
- .iter_mut()
- .map(|docset| (&mut docset.postings, &docset.similarity_weight))
- .collect_vec()
- })
- .collect_vec();
-
- debug_assert_eq!(postings.len(), num_docsets);
-
- let mut fields = Vec::new();
- for term in postings {
- for (i, field) in term.into_iter().enumerate() {
- if i >= fields.len() {
- fields.push(Vec::new());
- }
- fields[i].push(field);
- }
- }
-
- debug_assert!(fields.iter().all(|terms| terms.len() == num_docsets));
-
- let mut positions: Vec<Vec<u32>> = Vec::new();
- for _ in 0..fields.get(0).map(|t| t.len()).unwrap_or(0) {
- positions.push(Vec::new());
- }
-
- fields
- .into_iter()
- .enumerate()
- .map(|(field_id, terms)| {
- let mut weights = Vec::new();
- for (i, (term, weight)) in terms.into_iter().enumerate() {
- if term.doc() == TERMINATED {
- for pos in &mut positions[i..] {
- if !pos.is_empty() {
- pos.clear();
- }
- }
- break;
- }
- weights.push(weight);
- term.positions(positions.get_mut(i).unwrap());
- }
-
- if positions[0].is_empty() {
- // field has no matches in any/all of the terms.
- return 0.0;
- }
-
- let spans: Vec<Span> = SpansIterator::new(positions.clone()).collect();
-
- positions
- .iter()
- .filter(|pos| !pos.is_empty())
- .zip_eq(weights.iter())
- .enumerate()
- .map(|(term_id, (_pos, weight))| {
- let rc = spans
- .iter()
- .filter(|span| span.has_term(term_id as u32))
- .map(|span| span.relevance_contribution())
- .sum::<f32>();
-
- weight.score(field_id.try_into().unwrap(), rc)
- })
- .sum::<Score>()
- })
- .sum::<Score>()
- }
-}
-
-struct SpansIterator {
- num_terms: usize,
- hits: HitsIterator,
- max_width: u32,
- current_hit: Option<Hit>,
- spillover_hit: Option<Hit>,
-}
-
-impl SpansIterator {
- pub fn new(positions: Vec<Vec<u32>>) -> Self {
- Self::new_with_max_width(positions, MAX_WIDTH)
- }
-
- fn new_with_max_width(positions: Vec<Vec<u32>>, max_width: u32) -> Self {
- let num_terms = positions.len();
- let hits = HitsIterator::new(positions);
-
- Self {
- hits,
- num_terms,
- max_width,
- current_hit: None,
- spillover_hit: None,
- }
- }
-}
-
-impl Iterator for SpansIterator {
- type Item = Span;
-
- fn next(&mut self) -> Option<Self::Item> {
- let mut current_span = Span {
- max_width: self.max_width,
- ..Default::default()
- };
- let mut term_seen_before = vec![None; self.num_terms];
-
- if let Some(spillover) = self.spillover_hit.take() {
- term_seen_before[spillover.term_id as usize] = Some(spillover.position);
- current_span.terms.push(SpanTermInfo {
- term_id: spillover.term_id,
- position: spillover.position,
- });
- }
-
- loop {
- match (self.current_hit.take(), self.hits.next()) {
- (None, Some(next_hit)) => self.current_hit = Some(next_hit),
- (Some(current_hit), None) => {
- term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);
-
- current_span.terms.push(SpanTermInfo {
- term_id: current_hit.term_id,
- position: current_hit.position,
- });
-
- self.current_hit = None;
-
- return Some(current_span);
- }
- (Some(current_hit), Some(next_hit)) => {
- term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);
-
- let dist = next_hit.position - current_hit.position + 1;
- if dist > self.max_width || current_hit.term_id == next_hit.term_id {
- // case 1 and 2 from paper
- current_span.terms.push(SpanTermInfo {
- term_id: current_hit.term_id,
- position: current_hit.position,
- });
-
- self.current_hit = Some(next_hit);
-
- return Some(current_span);
- } else if let Some(prev_hit) = term_seen_before[next_hit.term_id as usize] {
- // case 3 from paper
- let dist_prev = current_hit.position - prev_hit + 1;
- let dist_next = next_hit.position - current_hit.position + 1;
-
- if dist_prev < dist_next {
- current_span.terms.push(SpanTermInfo {
- term_id: current_hit.term_id,
- position: current_hit.position,
- });
- } else {
- self.spillover_hit = Some(current_hit);
- }
- self.current_hit = Some(next_hit);
- return Some(current_span);
- } else {
- // case 4
- current_span.terms.push(SpanTermInfo {
- term_id: current_hit.term_id,
- position: current_hit.position,
- });
- self.current_hit = Some(next_hit);
- }
- }
- (None, None) => return None,
- }
- }
- }
-}
-
-#[derive(Debug)]
-struct SpanTermInfo {
- term_id: u32,
- position: u32,
-}
-
-#[derive(Debug)]
-struct Span {
- max_width: u32,
- terms: Vec<SpanTermInfo>,
-}
-
-impl Default for Span {
- fn default() -> Self {
- Self {
- max_width: MAX_WIDTH,
- terms: Default::default(),
- }
- }
-}
-
-impl Span {
- fn width(&self) -> u32 {
- match (self.terms.first(), self.terms.last()) {
- (Some(first), Some(last)) if first.term_id != last.term_id => {
- (last.position - first.position + 1).min(self.max_width)
- }
- _ => self.max_width,
- }
- }
-
- fn num_terms(&self) -> u32 {
- self.terms.len() as u32
- }
-
- fn has_term(&self, term_id: u32) -> bool {
- self.terms.iter().any(|term| term.term_id == term_id)
- }
-
- fn relevance_contribution(&self) -> f32 {
- (self.num_terms() as f32).powf(LAMBDA) / (self.width() as f32).powf(GAMMA)
- }
-}
-
-#[derive(PartialEq, Debug)]
-struct Hit {
- position: u32,
- term_id: u32,
-}
-
-struct HitsIterator {
- positions: Vec<Vec<u32>>,
- cursors: Vec<usize>,
-}
-
-impl HitsIterator {
- fn new(positions: Vec<Vec<u32>>) -> Self {
- let cursors = positions.iter().map(|_| 0).collect();
-
- Self { positions, cursors }
- }
-}
-
-impl Iterator for HitsIterator {
- type Item = Hit;
-
- fn next(&mut self) -> Option<Self::Item> {
- let mut smallest_idx = None;
-
- for idx in 0..self.positions.len() {
- let cursor = self.cursors[idx];
- if cursor >= self.positions[idx].len() {
- continue;
- }
-
- smallest_idx = match smallest_idx {
- Some(s_idx) => {
- let tmp: &Vec<u32> = &self.positions[s_idx];
- if self.positions[idx][cursor] < tmp[self.cursors[s_idx]] {
- Some(idx)
- } else {
- smallest_idx
- }
- }
- None => Some(idx),
- };
- }
-
- smallest_idx.map(|idx| {
- let res = self.positions[idx][self.cursors[idx]];
- self.cursors[idx] += 1;
-
- Hit {
- position: res,
- term_id: idx as u32,
- }
- })
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn hits_iterator() {
- let positions = vec![vec![0, 2, 6], vec![1, 4], vec![3, 5]];
-
- let mut iter = HitsIterator::new(positions);
-
- assert_eq!(
- iter.next(),
- Some(Hit {
- position: 0,
- term_id: 0,
- })
- );
- assert_eq!(
- iter.next(),
- Some(Hit {
- position: 1,
- term_id: 1,
- })
- );
- assert_eq!(
- iter.next(),
- Some(Hit {
- position: 2,
- term_id: 0,
- })
- );
- assert_eq!(
- iter.next(),
- Some(Hit {
- position: 3,
- term_id: 2,
- })
- );
- assert_eq!(
- iter.next(),
- Some(Hit {
- position: 4,
- term_id: 1,
- })
- );
- assert_eq!(
- iter.next(),
- Some(Hit {
- position: 5,
- term_id: 2,
- })
- );
- assert_eq!(
- iter.next(),
- Some(Hit {
- position: 6,
- term_id: 0,
- })
- );
- assert_eq!(iter.next(), None);
- }
-
- #[test]
- fn span_info() {
- let span = Span {
- terms: vec![
- SpanTermInfo {
- term_id: 1,
- position: 0,
- },
- SpanTermInfo {
- term_id: 2,
- position: 1,
- },
- SpanTermInfo {
- term_id: 3,
- position: 3,
- },
- ],
- ..Default::default()
- };
-
- assert_eq!(span.width(), 4);
- assert_eq!(span.num_terms(), 3);
- assert!(span.has_term(1));
- assert!(span.has_term(2));
- assert!(span.has_term(3));
- assert!(!span.has_term(4));
-
- let span = Span {
- terms: vec![SpanTermInfo {
- term_id: 1,
- position: 0,
- }],
- ..Default::default()
- };
-
- assert_eq!(span.width(), MAX_WIDTH);
-
- let span = Span {
- terms: vec![
- SpanTermInfo {
- term_id: 1,
- position: 0,
- },
- SpanTermInfo {
- term_id: 1,
- position: MAX_WIDTH + 10,
- },
- ],
- ..Default::default()
- };
-
- assert_eq!(span.width(), MAX_WIDTH);
- }
-
- #[test]
- fn hits_to_span() {
- // example from paper
- // positions:
- // sea: 5, 29
- // thousand: 7, 10
- // years: 8, 11
-
- let positions = vec![vec![5, 29], vec![7, 10], vec![8, 11]];
-
- let spans: Vec<Span> = SpansIterator::new_with_max_width(positions, 10).collect();
-
- assert_eq!(spans.len(), 3);
- assert_eq!(spans[0].width(), 4);
- assert_eq!(spans[1].width(), 2);
- assert_eq!(spans[2].width(), 10);
-
- assert_eq!(spans[0].num_terms(), 3);
- assert_eq!(spans[1].num_terms(), 2);
- assert_eq!(spans[2].num_terms(), 1);
- }
-}
diff --git a/src/query/term_scorer.rs b/src/query/term_scorer.rs
deleted file mode 100644
index a2e2dd44..00000000
--- a/src/query/term_scorer.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::query::bm25::Bm25Weight;
-
-use tantivy::fieldnorm::FieldNormReader;
-use tantivy::postings::{Postings, SegmentPostings};
-use tantivy::DocSet;
-use tantivy::{DocId, Score};
-
-#[derive(Clone)]
-pub struct TermScorerForField {
- pub postings: SegmentPostings,
- fieldnorm_reader: FieldNormReader,
- pub similarity_weight: Bm25Weight,
-}
-
-impl TermScorerForField {
- pub fn new(
- postings: SegmentPostings,
- fieldnorm_reader: FieldNormReader,
- similarity_weight: Bm25Weight,
- ) -> TermScorerForField {
- TermScorerForField {
- postings,
- fieldnorm_reader,
- similarity_weight,
- }
- }
-
- pub fn term_freq(&self) -> u32 {
- self.postings.term_freq()
- }
-
- pub fn fieldnorm_id(&self) -> u8 {
- self.fieldnorm_reader.fieldnorm_id(self.doc())
- }
-}
-
-impl DocSet for TermScorerForField {
- fn advance(&mut self) -> DocId {
- self.postings.advance()
- }
-
- fn seek(&mut self, target: DocId) -> DocId {
- self.postings.seek(target)
- }
-
- fn doc(&self) -> DocId {
- self.postings.doc()
- }
-
- fn size_hint(&self) -> u32 {
- self.postings.size_hint()
- }
-}
-
-impl tantivy::query::Scorer for TermScorerForField {
- fn score(&mut self) -> Score {
- let fieldnorm_id = self.fieldnorm_id();
- let term_freq = self.term_freq();
- self.similarity_weight
- .score(fieldnorm_id, term_freq as Score)
- }
-}
diff --git a/src/query/vec_docset.rs b/src/query/vec_docset.rs
deleted file mode 100644
index 418c136d..00000000
--- a/src/query/vec_docset.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-// taken from tantivy source for tests.
-// source: https://github.com/quickwit-oss/tantivy/blob/main/src/query/vec_docset.rs
-#![allow(dead_code)]
-
-use tantivy::{DocId, DocSet, TERMINATED};
-
-/// Simulate a `Postings` objects from a `VecPostings`.
-/// `VecPostings` only exist for testing purposes.
-///
-/// Term frequencies always return 1.
-/// No positions are returned.
-pub struct VecDocSet {
- doc_ids: Vec<DocId>,
- cursor: usize,
-}
-
-impl From<Vec<DocId>> for VecDocSet {
- fn from(doc_ids: Vec<DocId>) -> VecDocSet {
- VecDocSet { doc_ids, cursor: 0 }
- }
-}
-
-impl DocSet for VecDocSet {
- fn advance(&mut self) -> DocId {
- self.cursor += 1;
- if self.cursor >= self.doc_ids.len() {
- self.cursor = self.doc_ids.len();
- return TERMINATED;
- }
- self.doc()
- }
-
- fn doc(&self) -> DocId {
- if self.cursor == self.doc_ids.len() {
- return TERMINATED;
- }
- self.doc_ids[self.cursor]
- }
-
- fn size_hint(&self) -> u32 {
- self.doc_ids.len() as u32
- }
-}
-
-#[cfg(test)]
-pub mod tests {
-
- use super::*;
-
- #[test]
- pub fn test_vec_postings() {
- let doc_ids: Vec<DocId> = (0u32..1024u32).map(|e| e * 3).collect();
- let mut postings = VecDocSet::from(doc_ids);
- assert_eq!(postings.doc(), 0u32);
- assert_eq!(postings.advance(), 3u32);
- assert_eq!(postings.doc(), 3u32);
- assert_eq!(postings.seek(14u32), 15u32);
- assert_eq!(postings.doc(), 15u32);
- assert_eq!(postings.seek(300u32), 300u32);
- assert_eq!(postings.doc(), 300u32);
- assert_eq!(postings.seek(6000u32), TERMINATED);
- }
-
- #[test]
- pub fn test_fill_buffer() {
- let doc_ids: Vec<DocId> = (1u32..210u32).collect();
- let mut postings = VecDocSet::from(doc_ids);
- let mut buffer = vec![1000u32; 100];
- assert_eq!(postings.fill_buffer(&mut buffer[..]), 100);
- for i in 0u32..100u32 {
- assert_eq!(buffer[i as usize], i + 1);
- }
- assert_eq!(postings.fill_buffer(&mut buffer[..]), 100);
- for i in 0u32..100u32 {
- assert_eq!(buffer[i as usize], i + 101);
- }
- assert_eq!(postings.fill_buffer(&mut buffer[..]), 9);
- }
-}
diff --git a/src/query/weight.rs b/src/query/weight.rs
deleted file mode 100644
index 5165b491..00000000
--- a/src/query/weight.rs
+++ /dev/null
@@ -1,114 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use super::field_union::FieldUnion;
-use super::term_intersection::intersect_terms;
-use super::term_scorer::TermScorerForField;
-use super::FieldData;
-use tantivy::fieldnorm::FieldNormReader;
-use tantivy::postings::SegmentPostings;
-use tantivy::query::Explanation;
-use tantivy::{DocId, Score, SegmentReader};
-
-pub struct Weight {
- terms: Vec<String>,
- fields: Vec<FieldData>,
- scoring_enabled: bool,
-}
-
-impl tantivy::query::Weight for Weight {
- fn scorer(
- &self,
- reader: &SegmentReader,
- boost: Score,
- ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
- // each term has a union over the fields
- // e.g. if the term both occurs in the title and body
- let mut term_scorers: Vec<FieldUnion> = Vec::new();
-
- for term_text in &self.terms {
- let mut field_scorers: Vec<TermScorerForField> = Vec::new();
-
- for field_data in &self.fields {
- let inverted_index = reader.inverted_index(field_data.tantivy)?;
-
- let processed_terms: Vec<String> = field_data
- .analyzer
- .as_ref()
- .map(|analyzer| {
- let mut terms = Vec::new();
-
- let mut stream = analyzer.token_stream(term_text);
- while let Some(token) = stream.next() {
- terms.push(token.text.clone());
- }
-
- terms
- })
- .unwrap_or_else(|| vec![term_text.clone()]);
-
- for term in processed_terms {
- let fieldnorm_reader_opt = if self.scoring_enabled {
- reader.get_fieldnorms_reader(field_data.tantivy).ok()
- } else {
- None
- };
- let fieldnorm_reader = fieldnorm_reader_opt
- .unwrap_or_else(|| FieldNormReader::constant(reader.max_doc(), 1));
- let similarity_weight = field_data
- .scoring
- .boost_by(field_data.boost.unwrap_or(1.0) * boost);
-
- let term = tantivy::Term::from_field_text(field_data.tantivy, &term);
-
- let postings_opt: Option<SegmentPostings> = field_data
- .index_record_option
- .and_then(|index_record_option| {
- inverted_index
- .read_postings(&term, index_record_option)
- .unwrap()
- });
-
- let posting = postings_opt.unwrap_or_else(SegmentPostings::empty);
-
- field_scorers.push(TermScorerForField::new(
- posting,
- fieldnorm_reader,
- similarity_weight,
- ));
- }
- }
-
- term_scorers.push(FieldUnion::from(field_scorers));
- }
-
- Ok(intersect_terms(term_scorers))
- }
-
- fn explain(&self, _reader: &SegmentReader, _doc: DocId) -> tantivy::Result<Explanation> {
- unimplemented!();
- }
-}
-
-impl Weight {
- pub fn new(terms: Vec<String>, fields: Vec<FieldData>, scoring_enabled: bool) -> Weight {
- Weight {
- terms,
- fields,
- scoring_enabled,
- }
- }
-}
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rs
index 17ddd856..aef24c7f 100644
--- a/src/ranking/initial.rs
+++ b/src/ranking/initial.rs
@@ -97,9 +97,9 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
let hours_since_update = (self.current_timestamp - update_timestamp).max(0.000001) / 3600.0;
(3.0 * score)
- + (4000.0 * centrality)
+ + (6000.0 * centrality)
+ (1.0 * is_homepage)
- + (10.0 / (fetch_time_ms + 1.0))
- + (2000.0 * time_to_score(hours_since_update))
+ + (0.5 / (fetch_time_ms + 1.0))
+ + (1500.0 * time_to_score(hours_since_update))
}
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 965490a6..5b2f82cb 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -192,7 +192,7 @@ mod tests {
),
"https://www.old.com",
vec![],
- 0.1,
+ 0.092,
500,
))
.expect("failed to parse webpage");
diff --git a/src/schema.rs b/src/schema.rs
index 438236d0..12c9c467 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -154,7 +154,8 @@ impl Field {
Field::Host => Some(3.0),
Field::DomainIfHomepage => Some(50.0),
Field::StemmedCleanBody | Field::StemmedTitle => Some(0.1),
- Field::Title | Field::CleanBody => Some(1.5),
+ Field::CleanBody => Some(2.5),
+ Field::Title => Some(10.0),
Field::AllBody => Some(0.01),
Field::StemmedAllBody => Some(0.001),
Field::BacklinkText
diff --git a/templates/search.html b/templates/search.html
index fa76e7f9..9d346716 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -35,7 +35,7 @@
<img src="/favicons/{{ item.domain|urlencode }}" width="13px" />
</div>
<div>
- <a class="hover:no-underline text-gray-600" href="{{ item.url|urlencode }}">{{
+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{
item.pretty_url
}}</a>
</div>