web search done right
Use title hash, site hash and url hash in collector
Mikkel Denker8d1fdcc6fbb9
diff --git a/src/collector.rs b/src/collector.rsindex 741ff4c3..63dbd33c 100644--- a/src/collector.rs+++ b/src/collector.rs@@ -14,23 +14,26 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::{collections::HashMap, sync::Arc};+use min_max_heap::MinMaxHeap;+use serde::{Deserialize, Serialize};use tantivy::{collector::{Collector, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector},- fastfield::MultiValuedFastFieldReader,DocId, Score, SegmentOrdinal, SegmentReader,};use crate::{+ fastfield_cache,inverted_index::{DocAddress, WebsitePointer},- prehashed::{combine_u64s, PrehashMap, Prehashed},- schema::{FastField, Field},+ prehashed::{combine_u64s, Prehashed},+ schema::FastField,};-fn adjust_score(num_taken: usize, original_score: f64) -> f64 {- const SCALE: f64 = 14.0;- original_score * (SCALE / (num_taken as f64 + SCALE))-}+// lower scale -> higher penalty+const SITE_SCALE: f64 = 14.0;+const TITLE_SCALE: f64 = 6.0;+const URL_SCALE: f64 = 0.1;#[derive(Clone)]pub struct MaxDocsConsidered {@@ -38,32 +41,35 @@ pub struct MaxDocsConsidered {pub segments: usize,}-pub trait Doc: PartialEq + Eq + PartialOrd + Ord + Clone {+#[derive(Clone, Copy, Debug, Serialize, Deserialize)]+pub struct Hashes {+ site: Prehashed,+ title: Prehashed,+ url: Prehashed,+}++pub trait Doc: Clone {fn score(&self) -> &f64;- fn key(&self) -> &Prehashed;fn id(&self) -> &DocId;-- fn set_score(&mut self, score: f64);- fn set_key(&mut self, key: Prehashed);- fn set_id(&mut self, id: DocId);-- fn update_values_from(&mut self, other: &Self);+ fn hashes(&self) -> Hashes;}pub struct TopDocs {top_n: usize,offset: usize,max_docs: Option<MaxDocsConsidered>,+ fastfield_cache: Arc<fastfield_cache::FastFieldCache>,de_rank_similar: bool,}impl TopDocs {- pub fn with_limit(top_n: usize) -> Self {+ pub fn with_limit(top_n: usize, fastfield_cache: Arc<fastfield_cache::FastFieldCache>) -> Self {Self {top_n,offset: 0,max_docs: None,de_rank_similar: false,+ fastfield_cache,}}@@ -106,20 +112,13 @@ impl Collector for TopDocs {segment_local_id: tantivy::SegmentOrdinal,segment: &tantivy::SegmentReader,) -> tantivy::Result<Self::Child> {- let key_reader = segment.fast_fields().u64s(- segment- .schema()- .get_field(Field::Fast(FastField::SiteHash).name())- .unwrap(),- )?;-let max_docs = self.max_docs.as_ref().map(|max_docs| max_docs.total_docs / max_docs.segments);Ok(TopSegmentCollector {- key_reader,+ fastfield_segment_cache: self.fastfield_cache.get_segment(&segment.segment_id()),max_docs,num_docs_taken: 0,segment_ord: segment_local_id,@@ -149,7 +148,7 @@ impl Collector for TopDocs {.skip(self.offset).map(|doc| WebsitePointer {score: doc.score,- site_hash: doc.key,+ hashes: doc.hashes,address: DocAddress {segment: doc.segment,doc_id: doc.id,@@ -160,13 +159,28 @@ impl Collector for TopDocs {}pub struct TopSegmentCollector {- key_reader: MultiValuedFastFieldReader<u64>,+ fastfield_segment_cache: Arc<fastfield_cache::SegmentCache>,max_docs: Option<usize>,num_docs_taken: usize,segment_ord: SegmentOrdinal,bucket_collector: BucketCollector<SegmentDoc>,}+impl TopSegmentCollector {+ fn get_hash(&self, doc: &DocId, field: &FastField) -> Prehashed {+ let hash = self+ .fastfield_segment_cache+ .get_doc_cache(field)+ .get_u64s(doc)+ .unwrap();++ debug_assert_eq!(hash.len(), 2);++ let hash = [hash[0], hash[1]];+ combine_u64s(hash).into()+ }+}+impl SegmentCollector for TopSegmentCollector {type Fruit = Vec<SegmentDoc>;@@ -178,15 +192,13 @@ impl SegmentCollector for TopSegmentCollector {self.num_docs_taken += 1;}- let mut keys = Vec::new();- self.key_reader.get_vals(doc, &mut keys);- debug_assert_eq!(keys.len(), 2);-- let keys = [keys[0], keys[1]];- let key = combine_u64s(keys);self.bucket_collector.insert(SegmentDoc {- key: Prehashed(key),+ hashes: Hashes {+ site: self.get_hash(&doc, &FastField::SiteHash),+ title: self.get_hash(&doc, &FastField::TitleHash),+ url: self.get_hash(&doc, &FastField::UrlHash),+ },id: doc,segment: self.segment_ord,score: score as f64,@@ -198,254 +210,157 @@ impl SegmentCollector for TopSegmentCollector {}}-pub struct BucketCollector<T: Doc> {- buckets: PrehashMap<Bucket<T>>,- heads: MinMaxHeap<BucketHead>,- top_n: usize,+struct ScoredDoc<T: Doc> {+ doc: T,+ adjusted_score: f64,}-impl<T: Doc> BucketCollector<T> {- pub fn new(top_n: usize) -> Self {- assert!(top_n > 0);-- Self {- top_n,- heads: MinMaxHeap::with_capacity(top_n + 1),- buckets: PrehashMap::new(),- }+impl<T: Doc> PartialOrd for ScoredDoc<T> {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.adjusted_score.partial_cmp(&other.adjusted_score)}+}- pub fn insert(&mut self, doc: T) {- if let Some(bucket) = self.buckets.get_mut(doc.key()) {- bucket.insert(doc);- } else {- let mut bucket = Bucket::new(self.top_n);- bucket.insert(doc.clone());-- self.buckets.insert(doc.key().clone(), bucket);-- self.heads.push(BucketHead {- key: doc.key().clone(),- tweaked_score: *doc.score(),- });- }-- if self.buckets.len() > self.top_n + 1 {- self.prune_buckets()- }+impl<T: Doc> PartialEq for ScoredDoc<T> {+ fn eq(&self, other: &Self) -> bool {+ self.adjusted_score == other.adjusted_score}+}- fn prune_buckets(&mut self) {- self.update_worst_head();- let worst_head = self.heads.pop_min().unwrap();- self.buckets.remove(&worst_head.key);- }-- fn update_worst_head(&mut self) {- loop {- let mut worst_head = self.heads.peek_min_mut().unwrap();- let current_score = self- .buckets- .get(&worst_head.key)- .unwrap()- .get_best()- .unwrap()- .tweaked_score;-- if worst_head.tweaked_score != current_score {- worst_head.tweaked_score = current_score;- } else {- break;- }- }+impl<T: Doc> Ord for ScoredDoc<T> {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)}+}- fn build_heads(&self) -> MinMaxHeap<BucketHead> {- let mut bucket_heads: MinMaxHeap<BucketHead> = MinMaxHeap::with_capacity(self.top_n);-- for (key, bucket) in self.buckets.iter() {- let best_in_bucket = bucket.get_best().unwrap();- if bucket_heads.len() >= self.top_n {- let mut worst_head = bucket_heads.peek_min_mut().unwrap();+impl<T: Doc> Eq for ScoredDoc<T> {}- if best_in_bucket.tweaked_score > worst_head.tweaked_score {- worst_head.key = key.clone();- worst_head.tweaked_score = best_in_bucket.tweaked_score- }- } else {- bucket_heads.push(BucketHead {- key: key.clone(),- tweaked_score: best_in_bucket.tweaked_score,- });- }+impl<T: Doc> From<T> for ScoredDoc<T> {+ fn from(doc: T) -> Self {+ Self {+ adjusted_score: *doc.score(),+ doc,}-- bucket_heads}+}- pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {- let mut res = Vec::new();-- let mut bucket_heads = self.build_heads();+#[derive(Default)]+struct BucketCount {+ buckets: HashMap<Prehashed, usize>,+}- while let Some(mut head) = bucket_heads.pop_max() {- let bucket = self.buckets.get_mut(&head.key).unwrap();+impl BucketCount {+ pub fn adjust_score<T: Doc>(&self, doc: &mut ScoredDoc<T>) {+ let hashes = doc.doc.hashes();- if let Some(mut doc) = bucket.pop_best() {- if de_rank_similar {- doc.set_score(adjust_score(bucket.num_taken - 1, *doc.score()));- }+ let mut adjuster = 1.0;- res.push(doc);+ let taken_sites = self.buckets.get(&hashes.site).unwrap_or(&0);- if let Some(new_best) = bucket.get_best() {- head.tweaked_score = new_best.tweaked_score;- }+ adjuster *= SITE_SCALE / (SITE_SCALE + (*taken_sites as f64));- bucket_heads.push(head);- }+ let taken_urls = self.buckets.get(&hashes.url).unwrap_or(&0);+ adjuster *= URL_SCALE / (URL_SCALE + (*taken_urls as f64));- if res.len() == self.top_n {- break;- }- }-- res- }-}--struct BucketHead {- key: Prehashed,- tweaked_score: f64,-}+ let taken_titles = self.buckets.get(&hashes.title).unwrap_or(&0);+ adjuster *= TITLE_SCALE / (TITLE_SCALE + (*taken_titles as f64));-impl PartialEq for BucketHead {- fn eq(&self, other: &Self) -> bool {- self.tweaked_score == other.tweaked_score+ doc.adjusted_score = *doc.doc.score() * adjuster;}-}-impl Eq for BucketHead {}--impl PartialOrd for BucketHead {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- self.tweaked_score.partial_cmp(&other.tweaked_score)- }-}+ fn update_counts<T: Doc>(&mut self, doc: &ScoredDoc<T>) {+ let hashes = doc.doc.hashes();-impl Ord for BucketHead {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)+ *self.buckets.entry(hashes.site).or_default() += 1;+ *self.buckets.entry(hashes.url).or_default() += 1;+ *self.buckets.entry(hashes.title).or_default() += 1;}}-struct Bucket<T: Doc> {- num_taken: usize,- docs: MinMaxHeap<T>,+pub struct BucketCollector<T: Doc> {+ count: BucketCount,+ documents: MinMaxHeap<ScoredDoc<T>>,top_n: usize,}-impl<T: Doc> Bucket<T> {+impl<T: Doc> BucketCollector<T> {pub fn new(top_n: usize) -> Self {assert!(top_n > 0);Self {top_n,- num_taken: 0,- docs: MinMaxHeap::with_capacity(top_n),+ count: BucketCount::default(),+ documents: MinMaxHeap::with_capacity(top_n + 1),}}- pub fn pop_best(&mut self) -> Option<T> {- let res = self.docs.pop_max();+ pub fn insert(&mut self, doc: T) {+ let scored_doc = doc.into();- self.num_taken += 1;+ self.documents.push(scored_doc);- res+ self.prune_buckets()}- pub fn get_best(&self) -> Option<TweakedDoc<'_>> {- self.docs.peek_max().map(|doc| TweakedDoc {- tweaked_score: adjust_score(self.num_taken, *doc.score()),- _doc_id: doc.id(),- })+ fn prune_buckets(&mut self) {+ while self.documents.len() > self.top_n + 1 {+ self.documents.pop_min().unwrap();+ }}- pub fn insert(&mut self, doc: T) {- if self.docs.len() >= self.top_n {- let mut worst = self.docs.peek_min_mut().unwrap();+ fn update_best_doc(&mut self) {+ if self.documents.len() <= 1 {+ return;+ }- worst.update_values_from(&doc);- } else {- self.docs.push(doc);+ while let Some(mut best_doc) = self.documents.peek_max_mut() {+ let current_score = best_doc.adjusted_score;+ self.count.adjust_score(&mut *best_doc);++ if best_doc.adjusted_score == current_score {+ break;+ }}}-}-struct TweakedDoc<'a> {- tweaked_score: f64,- _doc_id: &'a DocId,+ pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {+ let mut res = Vec::new();++ while let Some(best_doc) = self.documents.pop_max() {+ if de_rank_similar {+ self.count.update_counts(&best_doc);+ self.update_best_doc();+ }+ res.push(best_doc.doc);++ if res.len() == self.top_n {+ break;+ }+ }++ res+ }}#[derive(Debug, Clone)]pub struct SegmentDoc {- key: Prehashed,+ hashes: Hashes,id: DocId,segment: SegmentOrdinal,score: f64,}-impl PartialEq for SegmentDoc {- fn eq(&self, other: &Self) -> bool {- self.score == other.score- }-}--impl Eq for SegmentDoc {}--impl PartialOrd for SegmentDoc {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- self.score.partial_cmp(&other.score)- }-}--impl Ord for SegmentDoc {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)- }-}-impl Doc for SegmentDoc {fn score(&self) -> &f64 {&self.score}- fn key(&self) -> &Prehashed {- &self.key- }-fn id(&self) -> &DocId {&self.id}- fn set_score(&mut self, score: f64) {- self.score = score;- }-- fn set_key(&mut self, key: Prehashed) {- self.key = key;- }-- fn set_id(&mut self, id: DocId) {- self.id = id;- }-- fn update_values_from(&mut self, other: &Self) {- self.set_id(*other.id());- self.set_key(other.key().clone());- self.set_score(*other.score());-- self.segment = other.segment;+ fn hashes(&self) -> Hashes {+ self.hashes}}@@ -530,12 +445,12 @@ wheremod tests {use super::*;- fn test(top_n: usize, docs: &[(u128, DocId, f64)], expected: &[(f64, DocId)]) {+ fn test(top_n: usize, docs: &[(Hashes, DocId, f64)], expected: &[(f64, DocId)]) {let mut collector = BucketCollector::new(top_n);for doc in docs {collector.insert(SegmentDoc {- key: Prehashed(doc.0),+ hashes: doc.0,id: doc.1,score: doc.2,segment: 0,@@ -556,11 +471,51 @@ mod tests {test(3,&[- (1, 123, 1.0),- (2, 124, 2.0),- (3, 125, 3.0),- (4, 126, 4.0),- (5, 127, 5.0),+ (+ Hashes {+ site: 1.into(),+ title: 1.into(),+ url: 1.into(),+ },+ 123,+ 1.0,+ ),+ (+ Hashes {+ site: 2.into(),+ title: 2.into(),+ url: 2.into(),+ },+ 124,+ 2.0,+ ),+ (+ Hashes {+ site: 3.into(),+ title: 3.into(),+ url: 3.into(),+ },+ 125,+ 3.0,+ ),+ (+ Hashes {+ site: 4.into(),+ title: 4.into(),+ url: 4.into(),+ },+ 126,+ 4.0,+ ),+ (+ Hashes {+ site: 5.into(),+ title: 5.into(),+ url: 5.into(),+ },+ 127,+ 5.0,+ ),],&[(5.0, 127), (4.0, 126), (3.0, 125)],);@@ -570,7 +525,35 @@ mod tests {fn less_than_topn() {test(10,- &[(3, 125, 3.0), (4, 126, 4.0), (5, 127, 5.0)],+ &[+ (+ Hashes {+ site: 3.into(),+ title: 3.into(),+ url: 3.into(),+ },+ 125,+ 3.0,+ ),+ (+ Hashes {+ site: 4.into(),+ title: 4.into(),+ url: 4.into(),+ },+ 126,+ 4.0,+ ),+ (+ Hashes {+ site: 5.into(),+ title: 5.into(),+ url: 5.into(),+ },+ 127,+ 5.0,+ ),+ ],&[(5.0, 127), (4.0, 126), (3.0, 125)],);}@@ -579,18 +562,70 @@ mod tests {fn same_key_de_prioritised() {test(10,- &[(1, 125, 3.0), (2, 126, 3.1), (2, 127, 5.0)],&[- (adjust_score(0, 5.0), 127),- (adjust_score(0, 3.0), 125),- (adjust_score(1, 3.1), 126),+ (+ Hashes {+ site: 1.into(),+ title: 1.into(),+ url: 1.into(),+ },+ 125,+ 3.0,+ ),+ (+ Hashes {+ site: 2.into(),+ title: 2.into(),+ url: 2.into(),+ },+ 126,+ 3.1,+ ),+ (+ Hashes {+ site: 2.into(),+ title: 2.into(),+ url: 2.into(),+ },+ 127,+ 5.0,+ ),],+ &[(5.0, 127), (3.0, 125), (3.1, 126)],);test(2,- &[(1, 125, 3.0), (2, 126, 3.1), (2, 127, 5.0)],- &[(adjust_score(0, 5.0), 127), (adjust_score(0, 3.0), 125)],+ &[+ (+ Hashes {+ site: 1.into(),+ title: 1.into(),+ url: 1.into(),+ },+ 125,+ 3.0,+ ),+ (+ Hashes {+ site: 2.into(),+ title: 2.into(),+ url: 2.into(),+ },+ 126,+ 3.1,+ ),+ (+ Hashes {+ site: 2.into(),+ title: 2.into(),+ url: 2.into(),+ },+ 127,+ 5.0,+ ),+ ],+ &[(5.0, 127), (3.0, 125)],);}}diff --git a/src/fastfield_cache.rs b/src/fastfield_cache.rsindex 57b39917..de245512 100644--- a/src/fastfield_cache.rs+++ b/src/fastfield_cache.rs@@ -130,6 +130,13 @@ impl DocCache {_ => None,}}++ pub fn get_u64s(&self, doc: &DocId) -> Option<&[u64]> {+ match self.get(doc) {+ FieldValue::U64s(res) => Some(res),+ _ => None,+ }+ }}pub struct SegmentCache {diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 9d35a2b3..a90752f8 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -23,9 +23,9 @@ use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};+use crate::collector::Hashes;use crate::fastfield_cache::FastFieldCache;use crate::image_store::Image;-use crate::prehashed::Prehashed;use crate::query::Query;use crate::schema::{FastField, Field, TextField, ALL_FIELDS};use crate::snippet;@@ -47,7 +47,7 @@ pub struct InitialSearchResult {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct WebsitePointer {pub score: f64,- pub site_hash: Prehashed,+ pub hashes: Hashes,pub address: DocAddress,}diff --git a/src/prehashed.rs b/src/prehashed.rsindex 3165d756..0863a4f9 100644--- a/src/prehashed.rs+++ b/src/prehashed.rs@@ -47,9 +47,15 @@ impl Hasher for Prehasher {}}-#[derive(Debug, Eq, Clone, Serialize, Deserialize)]+#[derive(Debug, Eq, Clone, Copy, Serialize, Deserialize)]pub struct Prehashed(pub u128);+impl From<u128> for Prehashed {+ fn from(val: u128) -> Self {+ Self(val)+ }+}+impl Hash for Prehashed {fn hash<H: std::hash::Hasher>(&self, state: &mut H) {state.write_u128(self.0);diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex eda19d72..53c1691d 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -93,7 +93,8 @@ impl Ranker {Arc::clone(&self.fastfield_cache),);- let mut collector = TopDocs::with_limit(NUM_RESULTS_PER_PAGE);+ let mut collector =+ TopDocs::with_limit(NUM_RESULTS_PER_PAGE, Arc::clone(&self.fastfield_cache));if self.de_rank_similar {collector = collector.and_de_rank_similar()diff --git a/src/searcher/distributed.rs b/src/searcher/distributed.rsindex 745464eb..7408c27a 100644--- a/src/searcher/distributed.rs+++ b/src/searcher/distributed.rs@@ -22,7 +22,7 @@ use crate::{searcher::{PrettifiedWebsitesResult, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE},};-use std::{cmp::Ordering, net::SocketAddr, time::Instant};+use std::{net::SocketAddr, time::Instant};use futures::stream::FuturesUnordered;use futures::StreamExt;@@ -269,57 +269,15 @@ impl collector::Doc for ScoredWebsitePointer {&self.local_pointer.score}- fn key(&self) -> &crate::prehashed::Prehashed {- &self.local_pointer.site_hash- }-fn id(&self) -> &tantivy::DocId {&self.local_pointer.address.doc_id}- fn set_score(&mut self, score: f64) {- self.local_pointer.score = score;- }-- fn set_key(&mut self, key: crate::prehashed::Prehashed) {- self.local_pointer.site_hash = key;- }-- fn set_id(&mut self, id: tantivy::DocId) {- self.local_pointer.address.doc_id = id;- }-- fn update_values_from(&mut self, other: &Self) {- self.set_score(*other.score());- self.set_key(other.key().clone());- self.set_id(*other.id());-- self.shard = other.shard.clone();+ fn hashes(&self) -> collector::Hashes {+ self.local_pointer.hashes}}-impl PartialOrd for ScoredWebsitePointer {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- self.local_pointer- .score- .partial_cmp(&other.local_pointer.score)- }-}--impl Ord for ScoredWebsitePointer {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.partial_cmp(other).unwrap_or(Ordering::Equal)- }-}--impl PartialEq for ScoredWebsitePointer {- fn eq(&self, other: &Self) -> bool {- self.local_pointer.score == other.local_pointer.score- }-}--impl Eq for ScoredWebsitePointer {}-impl DistributedSearcher {pub fn new(shards: Vec<Shard>) -> Self {Self { shards }diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 59e75976..483eebbb 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -271,13 +271,15 @@ impl Html {pub fn parse_without_text(html: &str, url: &str) -> Self {let root = kuchiki::parse_html().one(html);+ let mut url: Url = url.to_string().into();+ url = url.full_without_id_tags().into();Self {root,all_text: None,clean_text: None,lang: None,- url: url.to_string().into(),+ url,}}@@ -498,7 +500,7 @@ impl Html {}fn pretokenize_url(&self) -> PreTokenizedString {- let url = self.url().full_without_id_tags();+ let url = self.url().full();self.pretokenize_string(url)}