web search done right
Remove term proximity ranking (#20)
Mikkel Denker22468e434aaa
diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex bb6e6431..e2c03061 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,7 +1,11 @@warc_paths_file = "warc.paths"centrality_store_path = "./data/centrality"+limit_warc_files = 10[warc_source]-type = "Local"-folder = "./data/warc_files"-names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]\ No newline at end of file+# type = "Local"+# folder = "./data/warc_files"+# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]+type = "HTTP"+base_url = "https://data.commoncrawl.org/"+warc_paths_file = "warc.paths"diff --git a/justfile b/justfileindex 800ca07e..f2a197ee 100644--- a/justfile+++ b/justfile@@ -19,7 +19,7 @@wget -nc -P data/warc_files https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-05/segments/1642320299852.23/warc/CC-MAIN-20220116093137-20220116123137-00049.warc.gzrm -rf data/index/cargo run --release -- indexer local configs/indexer/local.toml- mv data/index/CC-MAIN-20220116093137-20220116123137-00049.warc.gz/* data/index/+ mv data/index/CC-MAIN-*/* data/index/just frontend@entity:diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex bcda8372..75e3b4f8 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -190,6 +190,7 @@ impl Indexer {warc_paths.into_iter()+ .take(config.limit_warc_files.unwrap_or(usize::MAX)).map(|path| Job {config: job_config.clone(),warc_path: path,diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 094ceaf0..b732bcd5 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -708,84 +708,6 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.example.com");}- #[test]- fn proximity_ranking() {- let mut index = InvertedIndex::temporary().expect("Unable to open index");- let query = Query::parse("termA termB", index.schema()).expect("Failed to parse query");- let ranker = Ranker::new(query.clone());-- index- .insert(Webpage::new(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT} termA termB d d d d d d d d d- </body>- </html>- "#- ),- "https://www.first.com",- vec![],- 1.0,- 500,- ))- .expect("failed to parse webpage");- index- .insert(Webpage::new(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT} termA d d d d d d d d d termB- </body>- </html>- "#- ),- "https://www.third.com",- vec![],- 1.0,- 500,- ))- .expect("failed to parse webpage");- index- .insert(Webpage::new(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT} termA d d d d termB d d d d d- </body>- </html>- "#- ),- "https://www.second.com",- vec![],- 1.0,- 500,- ))- .expect("failed to parse webpage");- index.commit().expect("failed to commit index");-- let result = index- .search(&query, ranker.collector())- .expect("Search failed");- assert_eq!(result.num_docs, 3);- assert_eq!(result.documents.len(), 3);- assert_eq!(result.documents[0].url, "https://www.first.com");- assert_eq!(result.documents[1].url, "https://www.second.com");- assert_eq!(result.documents[2].url, "https://www.third.com");- }-#[test]fn fetch_time_ranking() {let mut index = InvertedIndex::temporary().expect("Unable to open index");diff --git a/src/lib.rs b/src/lib.rsindex 86b90356..787cd41f 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -60,6 +60,7 @@ pub struct IndexingMasterConfig {#[derive(Debug, Deserialize, Clone)]pub struct IndexingLocalConfig {+ limit_warc_files: Option<usize>,warc_source: WarcSource,centrality_store_path: String,}diff --git a/src/query/bm25.rs b/src/query/bm25.rsdeleted file mode 100644index a88ab939..00000000--- a/src/query/bm25.rs+++ /dev/null@@ -1,142 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{Deserialize, Serialize};--use tantivy::fieldnorm::FieldNormReader;-use tantivy::query::Explanation;-use tantivy::{Score, Searcher, Term};--const K1: Score = 1.2;-const B: Score = 0.75;--pub(crate) fn idf(doc_freq: u64, doc_count: u64) -> Score {- assert!(doc_count >= doc_freq, "{} >= {}", doc_count, doc_freq);- let x = ((doc_count - doc_freq) as Score + 0.5) / (doc_freq as Score + 0.5);- (1.0 + x).ln()-}--fn cached_tf_component(fieldnorm: u32, average_fieldnorm: Score) -> Score {- K1 * (1.0 - B + B * fieldnorm as Score / average_fieldnorm)-}--fn compute_tf_cache(average_fieldnorm: Score) -> [Score; 256] {- let mut cache: [Score; 256] = [0.0; 256];- for (fieldnorm_id, cache_mut) in cache.iter_mut().enumerate() {- let fieldnorm = FieldNormReader::id_to_fieldnorm(fieldnorm_id as u8);- *cache_mut = cached_tf_component(fieldnorm, average_fieldnorm);- }- cache-}--#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]-pub struct Bm25Params {- pub idf: Score,- pub avg_fieldnorm: Score,-}--#[derive(Clone)]-pub struct Bm25Weight {- idf_explain: Explanation,- weight: Score,- cache: [Score; 256],- average_fieldnorm: Score,-}--impl Bm25Weight {- pub fn boost_by(&self, boost: Score) -> Bm25Weight {- Bm25Weight {- idf_explain: self.idf_explain.clone(),- weight: self.weight * boost,- cache: self.cache,- average_fieldnorm: self.average_fieldnorm,- }- }-- pub fn for_terms(searcher: &Searcher, terms: &[Term]) -> tantivy::Result<Bm25Weight> {- assert!(!terms.is_empty(), "Bm25 requires at least one term");- let field = terms[0].field();- for term in &terms[1..] {- assert_eq!(- term.field(),- field,- "All terms must belong to the same field."- );- }-- let mut total_num_tokens = 0u64;- let mut total_num_docs = 0u64;- for segment_reader in searcher.segment_readers() {- let inverted_index = segment_reader.inverted_index(field)?;- total_num_tokens += inverted_index.total_num_tokens();- total_num_docs += u64::from(segment_reader.max_doc());- }- let average_fieldnorm = total_num_tokens as Score / total_num_docs as Score;-- if terms.len() == 1 {- let term_doc_freq = searcher.doc_freq(&terms[0])?;- Ok(Bm25Weight::for_one_term(- term_doc_freq,- total_num_docs,- average_fieldnorm,- ))- } else {- let mut idf_sum: Score = 0.0;- for term in terms {- let term_doc_freq = searcher.doc_freq(term)?;- idf_sum += idf(term_doc_freq, total_num_docs);- }- let idf_explain = Explanation::new("idf", idf_sum);- Ok(Bm25Weight::new(idf_explain, average_fieldnorm))- }- }-- pub fn for_one_term(- term_doc_freq: u64,- total_num_docs: u64,- avg_fieldnorm: Score,- ) -> Bm25Weight {- let idf = idf(term_doc_freq, total_num_docs);- let mut idf_explain =- Explanation::new("idf, computed as log(1 + (N - n + 0.5) / (n + 0.5))", idf);- idf_explain.add_const(- "n, number of docs containing this term",- term_doc_freq as Score,- );- idf_explain.add_const("N, total number of docs", total_num_docs as Score);- Bm25Weight::new(idf_explain, avg_fieldnorm)- }-- pub(crate) fn new(idf_explain: Explanation, average_fieldnorm: Score) -> Bm25Weight {- let weight = idf_explain.value() * (1.0 + K1);- Bm25Weight {- idf_explain,- weight,- cache: compute_tf_cache(average_fieldnorm),- average_fieldnorm,- }- }-- #[inline]- pub fn score(&self, fieldnorm_id: u8, term_freq: Score) -> Score {- self.weight * self.tf_factor(fieldnorm_id, term_freq)- }-- #[inline]- pub(crate) fn tf_factor(&self, fieldnorm_id: u8, term_freq: Score) -> Score {- let norm = self.cache[fieldnorm_id as usize];- term_freq / (term_freq + norm)- }-}diff --git a/src/query/field_union.rs b/src/query/field_union.rsdeleted file mode 100644index f8a4b0a3..00000000--- a/src/query/field_union.rs+++ /dev/null@@ -1,73 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::query::Scorer;-use tantivy::{DocId, DocSet, Score, TERMINATED};--use super::term_scorer::TermScorerForField;--/// Creates a `DocSet` that iterate through the union of two or more `DocSet`s.-pub struct FieldUnion {- pub docsets: Vec<TermScorerForField>,-}--impl From<Vec<TermScorerForField>> for FieldUnion {- fn from(docsets: Vec<TermScorerForField>) -> FieldUnion {- FieldUnion { docsets }- }-}--impl DocSet for FieldUnion {- fn advance(&mut self) -> DocId {- let current_doc = self.doc();-- for scorer in self- .docsets- .iter_mut()- .filter(|scorer| scorer.doc() == current_doc)- {- scorer.advance();- }-- self.doc()- }-- fn doc(&self) -> DocId {- self.docsets- .iter()- .map(|scorer| scorer.doc())- .min()- .unwrap_or(TERMINATED)- }-- fn size_hint(&self) -> u32 {- self.docsets- .iter()- .map(|docset| docset.size_hint())- .max()- .unwrap_or(0u32)- }-}--impl Scorer for FieldUnion {- fn score(&mut self) -> Score {- let current_doc = self.doc();- self.docsets- .iter_mut()- .filter(|scorer| scorer.doc() == current_doc)- .map(|scorer| scorer.score())- .sum()- }-}diff --git a/src/query/mod.rs b/src/query/mod.rsindex 03d58753..3910ab24 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -14,47 +14,25 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{query::weight::Weight, schema::ALL_FIELDS, tokenizer::NormalTokenizer, Result};+use crate::{schema::ALL_FIELDS, tokenizer::NormalTokenizer, Result};use std::{collections::{BTreeMap, HashMap},sync::Arc,};use tantivy::{+ query::{BooleanQuery, BoostQuery, Occur, TermQuery},schema::{FieldType, IndexRecordOption, Schema},tokenizer::{TextAnalyzer, Tokenizer},};-use self::bm25::Bm25Weight;use lalrpop_util::lalrpop_mod;lalrpop_mod!(pub parser, "/query/parser.rs"); // synthesized by LALRPOP-mod bm25;-mod field_union;-mod term_intersection;-mod term_scorer;-mod vec_docset;-mod weight;-static PARSER: once_cell::sync::Lazy<parser::TermsParser> =once_cell::sync::Lazy::new(parser::TermsParser::new);const MAX_SIMILAR_TERMS: usize = 10;-pub struct FieldData {- tantivy: tantivy::schema::Field,- scoring: Bm25Weight,- index_record_option: Option<IndexRecordOption>,- boost: Option<f32>,- analyzer: Option<TextAnalyzer>,-}-impl std::fmt::Debug for FieldData {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.debug_struct("FieldData")- .field("tanitvy", &self.tantivy)- .finish()- }-}-#[derive(Debug)]pub enum Term<'a> {Simple(&'a str),@@ -101,6 +79,43 @@ impl Query {pub fn simple_terms(&self) -> &[String] {&self.terms}++ fn get_tantivy_analyzer(+ entry: &tantivy::schema::FieldEntry,+ tokenizer_manager: &tantivy::tokenizer::TokenizerManager,+ ) -> Option<TextAnalyzer> {+ match entry.field_type() {+ tantivy::schema::FieldType::Str(options) => {+ options.get_indexing_options().map(|indexing_options| {+ let tokenizer_name = indexing_options.tokenizer();+ tokenizer_manager+ .get(tokenizer_name)+ .expect("Unknown tokenizer")+ })+ }+ _ => None,+ }+ }++ fn process_tantivy_term(+ term: &str,+ analyzer: Option<TextAnalyzer>,+ tantivy_field: tantivy::schema::Field,+ ) -> impl Iterator<Item = tantivy::Term> {+ match analyzer {+ None => vec![tantivy::Term::from_field_text(tantivy_field, term)].into_iter(),+ Some(tokenizer) => {+ let mut terms: Vec<tantivy::Term> = Vec::new();+ let mut token_stream = tokenizer.token_stream(term);+ token_stream.process(&mut |token| {+ let term = tantivy::Term::from_field_text(tantivy_field, &token.text);+ terms.push(term);+ });++ terms.into_iter()+ }+ }+ }}impl tantivy::query::Query for Query {@@ -109,54 +124,53 @@ impl tantivy::query::Query for Query {searcher: &tantivy::Searcher,scoring_enabled: bool,) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {- let terms = self.terms.clone();+ let schema = searcher.schema();+ let tokenizer_manager = searcher.index().tokenizers();- let fields: Vec<_> = ALL_FIELDS+ let queries = self+ .terms.iter()- .filter(|field| field.is_searchable())- .map(|field| {- let tantivy_field = searcher.schema().get_field(field.as_str()).unwrap();- let analyzer = searcher.index().tokenizer_for_field(tantivy_field).ok();-- let tantivy_terms: Vec<_> = terms- .iter()- .flat_map(|term_text| {- let terms = analyzer- .as_ref()- .map(|analyzer| {- let mut terms = Vec::new();-- let mut stream = analyzer.token_stream(term_text);-- while let Some(token) = stream.next() {- terms.push(token.text.clone());- }-- terms+ .map(|term| {+ let one_term = schema+ .fields()+ .filter(|(field, _)| ALL_FIELDS[field.field_id() as usize].is_searchable())+ .into_iter()+ .map(|(field, entry)| {+ let analyzer = Query::get_tantivy_analyzer(entry, tokenizer_manager);+ let processed_terms = Query::process_tantivy_term(term, analyzer, field);++ let processed_queries = processed_terms+ .map(|term| {+ (+ Occur::Should,+ Box::new(TermQuery::new(+ term,+ IndexRecordOption::WithFreqsAndPositions,+ ))+ as Box<dyn tantivy::query::Query>,+ )})- .unwrap_or_else(|| vec![term_text.clone()]);-- terms- .into_iter()- .map(|term| tantivy::Term::from_field_text(tantivy_field, &term))+ .collect();+ let boost = ALL_FIELDS[field.field_id() as usize].boost().unwrap_or(1.0);++ (+ Occur::Should,+ Box::new(BoostQuery::new(+ Box::new(BooleanQuery::new(processed_queries)),+ boost,+ )) as Box<dyn tantivy::query::Query>,+ )}).collect();- FieldData {- tantivy: tantivy_field,- index_record_option: searcher- .schema()- .get_field_entry(tantivy_field)- .field_type()- .get_index_record_option(),- boost: field.boost(),- scoring: Bm25Weight::for_terms(searcher, &tantivy_terms).unwrap(),- analyzer,- }+ Box::new(BooleanQuery::new(one_term)) as Box<dyn tantivy::query::Query>})+ .map(|term_query| (Occur::Must, term_query)).collect();- Ok(Box::new(Weight::new(terms, fields, scoring_enabled)))+ let query = Box::new(BooleanQuery::new(queries));++ query.weight(searcher, scoring_enabled)}fn query_terms(&self, terms: &mut BTreeMap<tantivy::Term, bool>) {diff --git a/src/query/term_intersection.rs b/src/query/term_intersection.rsdeleted file mode 100644index c9f96f8e..00000000--- a/src/query/term_intersection.rs+++ /dev/null@@ -1,520 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use itertools::Itertools;-use tantivy::query::{EmptyScorer, Scorer};-use tantivy::{DocId, DocSet, Postings, Score, TERMINATED};--use super::field_union::FieldUnion;--const MAX_WIDTH: u32 = 25;-const LAMBDA: f32 = 0.75;-const GAMMA: f32 = 0.25;--pub fn intersect_terms(mut scorers: Vec<FieldUnion>) -> Box<dyn Scorer> {- if scorers.is_empty() {- return Box::new(EmptyScorer);- }- scorers.sort_by_key(|scorer| scorer.size_hint());- let doc = go_to_first_doc(&mut scorers[..]);- if doc == TERMINATED {- return Box::new(EmptyScorer);- }- Box::new(TermIntersection { docsets: scorers })-}--pub struct TermIntersection {- docsets: Vec<FieldUnion>,-}--fn go_to_first_doc<TDocSet: DocSet>(docsets: &mut [TDocSet]) -> DocId {- assert!(!docsets.is_empty());- let mut candidate = docsets.iter().map(TDocSet::doc).max().unwrap();- 'outer: loop {- for docset in docsets.iter_mut() {- let seek_doc = docset.seek(candidate);- if seek_doc > candidate {- candidate = docset.doc();- continue 'outer;- }- }- return candidate;- }-}--impl DocSet for TermIntersection {- fn advance(&mut self) -> DocId {- let mut candidate = self.docsets[0].advance();-- 'outer: loop {- for docset in self.docsets.iter_mut() {- let seek_doc = docset.seek(candidate);- if seek_doc > candidate {- candidate = self.docsets[0].seek(seek_doc);- continue 'outer;- }- }- debug_assert!(self.docsets.iter().all(|docset| docset.doc() == candidate));- return candidate;- }- }-- fn seek(&mut self, target: DocId) -> DocId {- self.docsets[0].seek(target);- let doc = go_to_first_doc(&mut self.docsets);- debug_assert!(self.docsets.iter().all(|docset| docset.doc() == doc));- debug_assert!(doc >= target);- doc- }-- fn doc(&self) -> DocId {- self.docsets[0].doc()- }-- fn size_hint(&self) -> u32 {- self.docsets[0].size_hint()- }-}--/// modified bm25 score based on term proximity.-/// more details in the papers "Viewing Term Proximity from a Different Perspective"-/// and "A Short Note on Proximity-based Scoring of Documents with Multiple Fields"-impl Scorer for TermIntersection {- fn score(&mut self) -> Score {- if self.doc() == TERMINATED {- return 0.0;- }-- let num_docsets = self.docsets.len();-- let postings = self- .docsets- .iter_mut()- .map(|docset| {- docset- .docsets- .iter_mut()- .map(|docset| (&mut docset.postings, &docset.similarity_weight))- .collect_vec()- })- .collect_vec();-- debug_assert_eq!(postings.len(), num_docsets);-- let mut fields = Vec::new();- for term in postings {- for (i, field) in term.into_iter().enumerate() {- if i >= fields.len() {- fields.push(Vec::new());- }- fields[i].push(field);- }- }-- debug_assert!(fields.iter().all(|terms| terms.len() == num_docsets));-- let mut positions: Vec<Vec<u32>> = Vec::new();- for _ in 0..fields.get(0).map(|t| t.len()).unwrap_or(0) {- positions.push(Vec::new());- }-- fields- .into_iter()- .enumerate()- .map(|(field_id, terms)| {- let mut weights = Vec::new();- for (i, (term, weight)) in terms.into_iter().enumerate() {- if term.doc() == TERMINATED {- for pos in &mut positions[i..] {- if !pos.is_empty() {- pos.clear();- }- }- break;- }- weights.push(weight);- term.positions(positions.get_mut(i).unwrap());- }-- if positions[0].is_empty() {- // field has no matches in any/all of the terms.- return 0.0;- }-- let spans: Vec<Span> = SpansIterator::new(positions.clone()).collect();-- positions- .iter()- .filter(|pos| !pos.is_empty())- .zip_eq(weights.iter())- .enumerate()- .map(|(term_id, (_pos, weight))| {- let rc = spans- .iter()- .filter(|span| span.has_term(term_id as u32))- .map(|span| span.relevance_contribution())- .sum::<f32>();-- weight.score(field_id.try_into().unwrap(), rc)- })- .sum::<Score>()- })- .sum::<Score>()- }-}--struct SpansIterator {- num_terms: usize,- hits: HitsIterator,- max_width: u32,- current_hit: Option<Hit>,- spillover_hit: Option<Hit>,-}--impl SpansIterator {- pub fn new(positions: Vec<Vec<u32>>) -> Self {- Self::new_with_max_width(positions, MAX_WIDTH)- }-- fn new_with_max_width(positions: Vec<Vec<u32>>, max_width: u32) -> Self {- let num_terms = positions.len();- let hits = HitsIterator::new(positions);-- Self {- hits,- num_terms,- max_width,- current_hit: None,- spillover_hit: None,- }- }-}--impl Iterator for SpansIterator {- type Item = Span;-- fn next(&mut self) -> Option<Self::Item> {- let mut current_span = Span {- max_width: self.max_width,- ..Default::default()- };- let mut term_seen_before = vec![None; self.num_terms];-- if let Some(spillover) = self.spillover_hit.take() {- term_seen_before[spillover.term_id as usize] = Some(spillover.position);- current_span.terms.push(SpanTermInfo {- term_id: spillover.term_id,- position: spillover.position,- });- }-- loop {- match (self.current_hit.take(), self.hits.next()) {- (None, Some(next_hit)) => self.current_hit = Some(next_hit),- (Some(current_hit), None) => {- term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);-- current_span.terms.push(SpanTermInfo {- term_id: current_hit.term_id,- position: current_hit.position,- });-- self.current_hit = None;-- return Some(current_span);- }- (Some(current_hit), Some(next_hit)) => {- term_seen_before[current_hit.term_id as usize] = Some(current_hit.position);-- let dist = next_hit.position - current_hit.position + 1;- if dist > self.max_width || current_hit.term_id == next_hit.term_id {- // case 1 and 2 from paper- current_span.terms.push(SpanTermInfo {- term_id: current_hit.term_id,- position: current_hit.position,- });-- self.current_hit = Some(next_hit);-- return Some(current_span);- } else if let Some(prev_hit) = term_seen_before[next_hit.term_id as usize] {- // case 3 from paper- let dist_prev = current_hit.position - prev_hit + 1;- let dist_next = next_hit.position - current_hit.position + 1;-- if dist_prev < dist_next {- current_span.terms.push(SpanTermInfo {- term_id: current_hit.term_id,- position: current_hit.position,- });- } else {- self.spillover_hit = Some(current_hit);- }- self.current_hit = Some(next_hit);- return Some(current_span);- } else {- // case 4- current_span.terms.push(SpanTermInfo {- term_id: current_hit.term_id,- position: current_hit.position,- });- self.current_hit = Some(next_hit);- }- }- (None, None) => return None,- }- }- }-}--#[derive(Debug)]-struct SpanTermInfo {- term_id: u32,- position: u32,-}--#[derive(Debug)]-struct Span {- max_width: u32,- terms: Vec<SpanTermInfo>,-}--impl Default for Span {- fn default() -> Self {- Self {- max_width: MAX_WIDTH,- terms: Default::default(),- }- }-}--impl Span {- fn width(&self) -> u32 {- match (self.terms.first(), self.terms.last()) {- (Some(first), Some(last)) if first.term_id != last.term_id => {- (last.position - first.position + 1).min(self.max_width)- }- _ => self.max_width,- }- }-- fn num_terms(&self) -> u32 {- self.terms.len() as u32- }-- fn has_term(&self, term_id: u32) -> bool {- self.terms.iter().any(|term| term.term_id == term_id)- }-- fn relevance_contribution(&self) -> f32 {- (self.num_terms() as f32).powf(LAMBDA) / (self.width() as f32).powf(GAMMA)- }-}--#[derive(PartialEq, Debug)]-struct Hit {- position: u32,- term_id: u32,-}--struct HitsIterator {- positions: Vec<Vec<u32>>,- cursors: Vec<usize>,-}--impl HitsIterator {- fn new(positions: Vec<Vec<u32>>) -> Self {- let cursors = positions.iter().map(|_| 0).collect();-- Self { positions, cursors }- }-}--impl Iterator for HitsIterator {- type Item = Hit;-- fn next(&mut self) -> Option<Self::Item> {- let mut smallest_idx = None;-- for idx in 0..self.positions.len() {- let cursor = self.cursors[idx];- if cursor >= self.positions[idx].len() {- continue;- }-- smallest_idx = match smallest_idx {- Some(s_idx) => {- let tmp: &Vec<u32> = &self.positions[s_idx];- if self.positions[idx][cursor] < tmp[self.cursors[s_idx]] {- Some(idx)- } else {- smallest_idx- }- }- None => Some(idx),- };- }-- smallest_idx.map(|idx| {- let res = self.positions[idx][self.cursors[idx]];- self.cursors[idx] += 1;-- Hit {- position: res,- term_id: idx as u32,- }- })- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn hits_iterator() {- let positions = vec![vec![0, 2, 6], vec![1, 4], vec![3, 5]];-- let mut iter = HitsIterator::new(positions);-- assert_eq!(- iter.next(),- Some(Hit {- position: 0,- term_id: 0,- })- );- assert_eq!(- iter.next(),- Some(Hit {- position: 1,- term_id: 1,- })- );- assert_eq!(- iter.next(),- Some(Hit {- position: 2,- term_id: 0,- })- );- assert_eq!(- iter.next(),- Some(Hit {- position: 3,- term_id: 2,- })- );- assert_eq!(- iter.next(),- Some(Hit {- position: 4,- term_id: 1,- })- );- assert_eq!(- iter.next(),- Some(Hit {- position: 5,- term_id: 2,- })- );- assert_eq!(- iter.next(),- Some(Hit {- position: 6,- term_id: 0,- })- );- assert_eq!(iter.next(), None);- }-- #[test]- fn span_info() {- let span = Span {- terms: vec![- SpanTermInfo {- term_id: 1,- position: 0,- },- SpanTermInfo {- term_id: 2,- position: 1,- },- SpanTermInfo {- term_id: 3,- position: 3,- },- ],- ..Default::default()- };-- assert_eq!(span.width(), 4);- assert_eq!(span.num_terms(), 3);- assert!(span.has_term(1));- assert!(span.has_term(2));- assert!(span.has_term(3));- assert!(!span.has_term(4));-- let span = Span {- terms: vec![SpanTermInfo {- term_id: 1,- position: 0,- }],- ..Default::default()- };-- assert_eq!(span.width(), MAX_WIDTH);-- let span = Span {- terms: vec![- SpanTermInfo {- term_id: 1,- position: 0,- },- SpanTermInfo {- term_id: 1,- position: MAX_WIDTH + 10,- },- ],- ..Default::default()- };-- assert_eq!(span.width(), MAX_WIDTH);- }-- #[test]- fn hits_to_span() {- // example from paper- // positions:- // sea: 5, 29- // thousand: 7, 10- // years: 8, 11-- let positions = vec![vec![5, 29], vec![7, 10], vec![8, 11]];-- let spans: Vec<Span> = SpansIterator::new_with_max_width(positions, 10).collect();-- assert_eq!(spans.len(), 3);- assert_eq!(spans[0].width(), 4);- assert_eq!(spans[1].width(), 2);- assert_eq!(spans[2].width(), 10);-- assert_eq!(spans[0].num_terms(), 3);- assert_eq!(spans[1].num_terms(), 2);- assert_eq!(spans[2].num_terms(), 1);- }-}diff --git a/src/query/term_scorer.rs b/src/query/term_scorer.rsdeleted file mode 100644index a2e2dd44..00000000--- a/src/query/term_scorer.rs+++ /dev/null@@ -1,77 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::query::bm25::Bm25Weight;--use tantivy::fieldnorm::FieldNormReader;-use tantivy::postings::{Postings, SegmentPostings};-use tantivy::DocSet;-use tantivy::{DocId, Score};--#[derive(Clone)]-pub struct TermScorerForField {- pub postings: SegmentPostings,- fieldnorm_reader: FieldNormReader,- pub similarity_weight: Bm25Weight,-}--impl TermScorerForField {- pub fn new(- postings: SegmentPostings,- fieldnorm_reader: FieldNormReader,- similarity_weight: Bm25Weight,- ) -> TermScorerForField {- TermScorerForField {- postings,- fieldnorm_reader,- similarity_weight,- }- }-- pub fn term_freq(&self) -> u32 {- self.postings.term_freq()- }-- pub fn fieldnorm_id(&self) -> u8 {- self.fieldnorm_reader.fieldnorm_id(self.doc())- }-}--impl DocSet for TermScorerForField {- fn advance(&mut self) -> DocId {- self.postings.advance()- }-- fn seek(&mut self, target: DocId) -> DocId {- self.postings.seek(target)- }-- fn doc(&self) -> DocId {- self.postings.doc()- }-- fn size_hint(&self) -> u32 {- self.postings.size_hint()- }-}--impl tantivy::query::Scorer for TermScorerForField {- fn score(&mut self) -> Score {- let fieldnorm_id = self.fieldnorm_id();- let term_freq = self.term_freq();- self.similarity_weight- .score(fieldnorm_id, term_freq as Score)- }-}diff --git a/src/query/vec_docset.rs b/src/query/vec_docset.rsdeleted file mode 100644index 418c136d..00000000--- a/src/query/vec_docset.rs+++ /dev/null@@ -1,79 +0,0 @@-// taken from tantivy source for tests.-// source: https://github.com/quickwit-oss/tantivy/blob/main/src/query/vec_docset.rs-#![allow(dead_code)]--use tantivy::{DocId, DocSet, TERMINATED};--/// Simulate a `Postings` objects from a `VecPostings`.-/// `VecPostings` only exist for testing purposes.-///-/// Term frequencies always return 1.-/// No positions are returned.-pub struct VecDocSet {- doc_ids: Vec<DocId>,- cursor: usize,-}--impl From<Vec<DocId>> for VecDocSet {- fn from(doc_ids: Vec<DocId>) -> VecDocSet {- VecDocSet { doc_ids, cursor: 0 }- }-}--impl DocSet for VecDocSet {- fn advance(&mut self) -> DocId {- self.cursor += 1;- if self.cursor >= self.doc_ids.len() {- self.cursor = self.doc_ids.len();- return TERMINATED;- }- self.doc()- }-- fn doc(&self) -> DocId {- if self.cursor == self.doc_ids.len() {- return TERMINATED;- }- self.doc_ids[self.cursor]- }-- fn size_hint(&self) -> u32 {- self.doc_ids.len() as u32- }-}--#[cfg(test)]-pub mod tests {-- use super::*;-- #[test]- pub fn test_vec_postings() {- let doc_ids: Vec<DocId> = (0u32..1024u32).map(|e| e * 3).collect();- let mut postings = VecDocSet::from(doc_ids);- assert_eq!(postings.doc(), 0u32);- assert_eq!(postings.advance(), 3u32);- assert_eq!(postings.doc(), 3u32);- assert_eq!(postings.seek(14u32), 15u32);- assert_eq!(postings.doc(), 15u32);- assert_eq!(postings.seek(300u32), 300u32);- assert_eq!(postings.doc(), 300u32);- assert_eq!(postings.seek(6000u32), TERMINATED);- }-- #[test]- pub fn test_fill_buffer() {- let doc_ids: Vec<DocId> = (1u32..210u32).collect();- let mut postings = VecDocSet::from(doc_ids);- let mut buffer = vec![1000u32; 100];- assert_eq!(postings.fill_buffer(&mut buffer[..]), 100);- for i in 0u32..100u32 {- assert_eq!(buffer[i as usize], i + 1);- }- assert_eq!(postings.fill_buffer(&mut buffer[..]), 100);- for i in 0u32..100u32 {- assert_eq!(buffer[i as usize], i + 101);- }- assert_eq!(postings.fill_buffer(&mut buffer[..]), 9);- }-}diff --git a/src/query/weight.rs b/src/query/weight.rsdeleted file mode 100644index 5165b491..00000000--- a/src/query/weight.rs+++ /dev/null@@ -1,114 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use super::field_union::FieldUnion;-use super::term_intersection::intersect_terms;-use super::term_scorer::TermScorerForField;-use super::FieldData;-use tantivy::fieldnorm::FieldNormReader;-use tantivy::postings::SegmentPostings;-use tantivy::query::Explanation;-use tantivy::{DocId, Score, SegmentReader};--pub struct Weight {- terms: Vec<String>,- fields: Vec<FieldData>,- scoring_enabled: bool,-}--impl tantivy::query::Weight for Weight {- fn scorer(- &self,- reader: &SegmentReader,- boost: Score,- ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {- // each term has a union over the fields- // e.g. if the term both occurs in the title and body- let mut term_scorers: Vec<FieldUnion> = Vec::new();-- for term_text in &self.terms {- let mut field_scorers: Vec<TermScorerForField> = Vec::new();-- for field_data in &self.fields {- let inverted_index = reader.inverted_index(field_data.tantivy)?;-- let processed_terms: Vec<String> = field_data- .analyzer- .as_ref()- .map(|analyzer| {- let mut terms = Vec::new();-- let mut stream = analyzer.token_stream(term_text);- while let Some(token) = stream.next() {- terms.push(token.text.clone());- }-- terms- })- .unwrap_or_else(|| vec![term_text.clone()]);-- for term in processed_terms {- let fieldnorm_reader_opt = if self.scoring_enabled {- reader.get_fieldnorms_reader(field_data.tantivy).ok()- } else {- None- };- let fieldnorm_reader = fieldnorm_reader_opt- .unwrap_or_else(|| FieldNormReader::constant(reader.max_doc(), 1));- let similarity_weight = field_data- .scoring- .boost_by(field_data.boost.unwrap_or(1.0) * boost);-- let term = tantivy::Term::from_field_text(field_data.tantivy, &term);-- let postings_opt: Option<SegmentPostings> = field_data- .index_record_option- .and_then(|index_record_option| {- inverted_index- .read_postings(&term, index_record_option)- .unwrap()- });-- let posting = postings_opt.unwrap_or_else(SegmentPostings::empty);-- field_scorers.push(TermScorerForField::new(- posting,- fieldnorm_reader,- similarity_weight,- ));- }- }-- term_scorers.push(FieldUnion::from(field_scorers));- }-- Ok(intersect_terms(term_scorers))- }-- fn explain(&self, _reader: &SegmentReader, _doc: DocId) -> tantivy::Result<Explanation> {- unimplemented!();- }-}--impl Weight {- pub fn new(terms: Vec<String>, fields: Vec<FieldData>, scoring_enabled: bool) -> Weight {- Weight {- terms,- fields,- scoring_enabled,- }- }-}diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 17ddd856..aef24c7f 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -97,9 +97,9 @@ impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {let hours_since_update = (self.current_timestamp - update_timestamp).max(0.000001) / 3600.0;(3.0 * score)- + (4000.0 * centrality)+ + (6000.0 * centrality)+ (1.0 * is_homepage)- + (10.0 / (fetch_time_ms + 1.0))- + (2000.0 * time_to_score(hours_since_update))+ + (0.5 / (fetch_time_ms + 1.0))+ + (1500.0 * time_to_score(hours_since_update))}}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 965490a6..5b2f82cb 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -192,7 +192,7 @@ mod tests {),"https://www.old.com",vec![],- 0.1,+ 0.092,500,)).expect("failed to parse webpage");diff --git a/src/schema.rs b/src/schema.rsindex 438236d0..12c9c467 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -154,7 +154,8 @@ impl Field {Field::Host => Some(3.0),Field::DomainIfHomepage => Some(50.0),Field::StemmedCleanBody | Field::StemmedTitle => Some(0.1),- Field::Title | Field::CleanBody => Some(1.5),+ Field::CleanBody => Some(2.5),+ Field::Title => Some(10.0),Field::AllBody => Some(0.01),Field::StemmedAllBody => Some(0.001),Field::BacklinkTextdiff --git a/templates/search.html b/templates/search.htmlindex fa76e7f9..9d346716 100644--- a/templates/search.html+++ b/templates/search.html@@ -35,7 +35,7 @@<img src="/favicons/{{ item.domain|urlencode }}" width="13px" /></div><div>- <a class="hover:no-underline text-gray-600" href="{{ item.url|urlencode }}">{{+ <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{item.pretty_url}}</a></div>