web search done right
Ftr/de similar results (#56)
Mikkel Denkerfb8cfe1f30ae
diff --git a/Cargo.lock b/Cargo.lockindex 0000b770..9aed7b52 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -697,6 +697,7 @@ dependencies = ["maplit","md5","memmap",+ "min-max-heap","once_cell","parse_wiki_text","quick-xml",@@ -1702,6 +1703,12 @@ dependencies = ["unicase",]+[[package]]+name = "min-max-heap"+version = "1.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2687e6cf9c00f48e9284cf9fd15f2ef341d03cc7743abf9df4c5f07fdee50b18"+[[package]]name = "minimal-lexical"version = "0.2.1"diff --git a/Cargo.toml b/Cargo.tomlindex d8599537..b72e7b99 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -55,6 +55,7 @@ tower-http = { version = "0.3.4", features = ["compression-gzip"] }lalrpop-util = { version = "0.19.8", features = ["lexer"] }serde_urlencoded = "0.7.1"rust-stemmers = "1.2.0"+min-max-heap = "1.3.0"[dev-dependencies]criterion = "0.3.6"diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex d4aa3b48..a9bfff1a 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,8 +1,8 @@warc_paths_file = "warc.paths"centrality_store_path = "./data/centrality"-limit_warc_files = 100+limit_warc_files = 25download_images = false-batch_size = 10+batch_size = 3[warc_source]# type = "Local"diff --git a/src/collector.rs b/src/collector.rsnew file mode 100644index 00000000..6e8409b2--- /dev/null+++ b/src/collector.rs@@ -0,0 +1,510 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use min_max_heap::MinMaxHeap;+use tantivy::{+ collector::{Collector, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector},+ fastfield::MultiValuedFastFieldReader,+ DocAddress, DocId, Score, SegmentOrdinal, SegmentReader,+};++use crate::{+ prehashed::{combine_u64s, PrehashMap, Prehashed},+ schema::Field,+};++fn adjust_score(num_taken: usize, original_score: f64) -> f64 {+ const SCALE: f64 = 14.0;+ original_score * (SCALE / (num_taken as f64 + SCALE))+}++pub struct TopDocs {+ top_n: usize,+ offset: usize,+}++impl TopDocs {+ pub fn with_limit(top_n: usize) -> Self {+ Self { top_n, offset: 0 }+ }++ pub fn and_offset(mut self, offset: usize) -> Self {+ self.offset = offset;+ self+ }++ pub fn tweak_score<TScoreSegmentTweaker, TScoreTweaker>(+ self,+ score_tweaker: TScoreTweaker,+ ) -> impl Collector<Fruit = Vec<(f64, DocAddress)>>+ where+ TScoreSegmentTweaker: ScoreSegmentTweaker<f64> + 'static,+ TScoreTweaker: ScoreTweaker<f64, Child = TScoreSegmentTweaker> + Send + Sync,+ {+ TweakedScoreTopCollector::new(score_tweaker, self)+ }+}++impl Collector for TopDocs {+ type Fruit = Vec<(f64, DocAddress)>;++ type Child = TopSegmentCollector;++ fn for_segment(+ &self,+ segment_local_id: tantivy::SegmentOrdinal,+ segment: &tantivy::SegmentReader,+ ) -> tantivy::Result<Self::Child> {+ let key_reader = segment.fast_fields().u64s(+ segment+ .schema()+ .get_field(Field::SiteHash.as_str())+ .unwrap(),+ )?;++ Ok(TopSegmentCollector {+ key_reader,+ segment_ord: segment_local_id,+ bucket_collector: BucketCollector::new(self.top_n + self.offset),+ })+ }++ fn requires_scoring(&self) -> bool {+ true+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> tantivy::Result<Self::Fruit> {+ let mut collector = BucketCollector::new(self.top_n + self.offset);++ for docs in segment_fruits {+ for doc in docs {+ collector.insert(doc);+ }+ }++ Ok(collector+ .into_sorted_vec(true)+ .into_iter()+ .skip(self.offset)+ .map(|doc| {+ (+ doc.score,+ DocAddress {+ segment_ord: doc.segment,+ doc_id: doc.id,+ },+ )+ })+ .collect())+ }+}++pub struct TopSegmentCollector {+ key_reader: MultiValuedFastFieldReader<u64>,+ segment_ord: SegmentOrdinal,+ bucket_collector: BucketCollector,+}++impl SegmentCollector for TopSegmentCollector {+ type Fruit = Vec<Doc>;++ fn collect(&mut self, doc: DocId, score: Score) {+ let mut keys = Vec::new();+ self.key_reader.get_vals(doc, &mut keys);+ debug_assert_eq!(keys.len(), 2);++ let keys = [keys[0], keys[1]];+ let key = combine_u64s(keys);++ self.bucket_collector.insert(Doc {+ key: Prehashed(key),+ id: doc,+ segment: self.segment_ord,+ score: score as f64,+ })+ }++ fn harvest(self) -> Self::Fruit {+ self.bucket_collector.into_sorted_vec(false)+ }+}++struct BucketCollector {+ buckets: PrehashMap<Bucket>,+ heads: MinMaxHeap<BucketHead>,+ top_n: usize,+}++impl BucketCollector {+ pub fn new(top_n: usize) -> Self {+ assert!(top_n > 0);++ Self {+ top_n,+ heads: MinMaxHeap::with_capacity(top_n * 2 + 1),+ buckets: PrehashMap::new(),+ }+ }++ pub fn insert(&mut self, doc: Doc) {+ if let Some(bucket) = self.buckets.get_mut(&doc.key) {+ bucket.insert(doc);+ } else {+ let mut bucket = Bucket::new(self.top_n);+ bucket.insert(doc.clone());++ self.buckets.insert(doc.key.clone(), bucket);++ self.heads.push(BucketHead {+ key: doc.key,+ tweaked_score: doc.score,+ });+ }++ if self.buckets.len() > (self.top_n * 2) + 1 {+ self.prune_buckets()+ }+ }++ fn prune_buckets(&mut self) {+ self.update_worst_head();+ let worst_head = self.heads.pop_min().unwrap();+ self.buckets.remove(&worst_head.key);+ }++ fn update_worst_head(&mut self) {+ loop {+ let mut worst_head = self.heads.peek_min_mut().unwrap();+ let current_score = self+ .buckets+ .get(&worst_head.key)+ .unwrap()+ .get_best()+ .unwrap()+ .tweaked_score;++ if worst_head.tweaked_score != current_score {+ worst_head.tweaked_score = current_score;+ } else {+ break;+ }+ }+ }++ fn build_heads(&self) -> MinMaxHeap<BucketHead> {+ let mut bucket_heads: MinMaxHeap<BucketHead> = MinMaxHeap::with_capacity(self.top_n);++ for (key, bucket) in self.buckets.iter() {+ let best_in_bucket = bucket.get_best().unwrap();+ if bucket_heads.len() >= self.top_n {+ let mut worst_head = bucket_heads.peek_min_mut().unwrap();++ if best_in_bucket.tweaked_score > worst_head.tweaked_score {+ worst_head.key = key.clone();+ worst_head.tweaked_score = best_in_bucket.tweaked_score+ }+ } else {+ bucket_heads.push(BucketHead {+ key: key.clone(),+ tweaked_score: best_in_bucket.tweaked_score,+ });+ }+ }++ bucket_heads+ }++ pub fn into_sorted_vec(mut self, apply_adjust_score: bool) -> Vec<Doc> {+ let mut res = Vec::new();++ let mut bucket_heads = self.build_heads();++ while let Some(mut head) = bucket_heads.pop_max() {+ let bucket = self.buckets.get_mut(&head.key).unwrap();++ if let Some(mut doc) = bucket.pop_best() {+ if apply_adjust_score {+ doc.score = adjust_score(bucket.num_taken - 1, doc.score);+ }++ res.push(doc);++ if let Some(new_best) = bucket.get_best() {+ head.tweaked_score = new_best.tweaked_score;+ }++ bucket_heads.push(head);+ }++ if res.len() == self.top_n {+ break;+ }+ }++ res+ }+}++struct BucketHead {+ key: Prehashed,+ tweaked_score: f64,+}++impl PartialEq for BucketHead {+ fn eq(&self, other: &Self) -> bool {+ self.tweaked_score == other.tweaked_score+ }+}++impl Eq for BucketHead {}++impl PartialOrd for BucketHead {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.tweaked_score.partial_cmp(&other.tweaked_score)+ }+}++impl Ord for BucketHead {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)+ }+}++struct Bucket {+ num_taken: usize,+ docs: MinMaxHeap<Doc>,+ top_n: usize,+}++impl Bucket {+ pub fn new(top_n: usize) -> Self {+ assert!(top_n > 0);++ Self {+ top_n,+ num_taken: 0,+ docs: MinMaxHeap::with_capacity(top_n),+ }+ }++ pub fn pop_best(&mut self) -> Option<Doc> {+ let res = self.docs.pop_max();++ self.num_taken += 1;++ res+ }++ pub fn get_best(&self) -> Option<TweakedDoc<'_>> {+ self.docs.peek_max().map(|doc| TweakedDoc {+ tweaked_score: adjust_score(self.num_taken, doc.score),+ _doc_id: &doc.id,+ })+ }++ pub fn insert(&mut self, doc: Doc) {+ if self.docs.len() >= self.top_n {+ let mut worst = self.docs.peek_min_mut().unwrap();++ worst.id = doc.id;+ worst.score = doc.score;+ } else {+ self.docs.push(doc);+ }+ }+}++struct TweakedDoc<'a> {+ tweaked_score: f64,+ _doc_id: &'a DocId,+}++#[derive(Debug, Clone)]+pub struct Doc {+ key: Prehashed,+ id: DocId,+ segment: SegmentOrdinal,+ score: f64,+}++impl PartialEq for Doc {+ fn eq(&self, other: &Self) -> bool {+ self.score == other.score+ }+}++impl Eq for Doc {}++impl PartialOrd for Doc {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ self.score.partial_cmp(&other.score)+ }+}++impl Ord for Doc {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)+ }+}++pub(crate) struct TweakedScoreTopCollector<TScoreTweaker> {+ score_tweaker: TScoreTweaker,+ collector: TopDocs,+}++impl<TScoreTweaker> TweakedScoreTopCollector<TScoreTweaker> {+ pub fn new(+ score_tweaker: TScoreTweaker,+ collector: TopDocs,+ ) -> TweakedScoreTopCollector<TScoreTweaker> {+ TweakedScoreTopCollector {+ score_tweaker,+ collector,+ }+ }+}++impl<TScoreTweaker> Collector for TweakedScoreTopCollector<TScoreTweaker>+where+ TScoreTweaker: ScoreTweaker<f64> + Send + Sync,+{+ type Fruit = Vec<(f64, DocAddress)>;++ type Child = TopTweakedScoreSegmentCollector<TScoreTweaker::Child>;++ fn for_segment(+ &self,+ segment_local_id: u32,+ segment_reader: &SegmentReader,+ ) -> tantivy::Result<Self::Child> {+ let segment_scorer = self.score_tweaker.segment_tweaker(segment_reader)?;+ let segment_collector = self+ .collector+ .for_segment(segment_local_id, segment_reader)?;+ Ok(TopTweakedScoreSegmentCollector {+ segment_collector,+ segment_scorer,+ })+ }++ fn requires_scoring(&self) -> bool {+ true+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> tantivy::Result<Self::Fruit> {+ self.collector.merge_fruits(segment_fruits)+ }+}++pub struct TopTweakedScoreSegmentCollector<TSegmentScoreTweaker>+where+ TSegmentScoreTweaker: ScoreSegmentTweaker<f64>,+{+ segment_collector: TopSegmentCollector,+ segment_scorer: TSegmentScoreTweaker,+}++impl<TSegmentScoreTweaker> SegmentCollector+ for TopTweakedScoreSegmentCollector<TSegmentScoreTweaker>+where+ TSegmentScoreTweaker: 'static + ScoreSegmentTweaker<f64>,+{+ type Fruit = Vec<Doc>;++ fn collect(&mut self, doc: DocId, score: Score) {+ let score = self.segment_scorer.score(doc, score);+ self.segment_collector.collect(doc, score as f32);+ }++ fn harvest(self) -> Self::Fruit {+ self.segment_collector.harvest()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn test(top_n: usize, docs: &[(u128, DocId, f64)], expected: &[(f64, DocId)]) {+ let mut collector = BucketCollector::new(top_n);++ for doc in docs {+ collector.insert(Doc {+ key: Prehashed(doc.0),+ id: doc.1,+ score: doc.2,+ segment: 0,+ });+ }++ let res: Vec<(f64, DocId)> = collector+ .into_sorted_vec(true)+ .into_iter()+ .map(|doc| (doc.score, doc.id))+ .collect();++ assert_eq!(&res, expected);+ }++ #[test]+ fn all_different() {+ test(+ 3,+ &[+ (1, 123, 1.0),+ (2, 124, 2.0),+ (3, 125, 3.0),+ (4, 126, 4.0),+ (5, 127, 5.0),+ ],+ &[(5.0, 127), (4.0, 126), (3.0, 125)],+ );+ }++ #[test]+ fn less_than_topn() {+ test(+ 10,+ &[(3, 125, 3.0), (4, 126, 4.0), (5, 127, 5.0)],+ &[(5.0, 127), (4.0, 126), (3.0, 125)],+ );+ }++ #[test]+ fn same_key_de_prioritised() {+ test(+ 10,+ &[(1, 125, 3.0), (2, 126, 3.1), (2, 127, 5.0)],+ &[+ (adjust_score(0, 5.0), 127),+ (adjust_score(0, 3.0), 125),+ (adjust_score(1, 3.1), 126),+ ],+ );++ test(+ 2,+ &[(1, 125, 3.0), (2, 126, 3.1), (2, 127, 5.0)],+ &[(adjust_score(0, 5.0), 127), (adjust_score(0, 3.0), 125)],+ );+ }+}diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 77cf2741..d59b1c0f 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -371,6 +371,7 @@ impl From<Document> for RetrievedWebpage {| Field::Domain| Field::DomainIfHomepage| Field::IsHomepage+ | Field::SiteHash| Field::NumTrackers| Field::NumCleanBodyTokens| Field::NumDescriptionTokensdiff --git a/src/lib.rs b/src/lib.rsindex 93e8557c..be985de5 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -34,6 +34,7 @@ pub mod mapreduce;mod autosuggest;mod bangs;+mod collector;mod directory;mod entity_index;mod exponential_backoff;@@ -42,6 +43,7 @@ mod image_downloader;mod image_store;pub mod index;mod kv;+pub mod prehashed;mod query;mod ranking;mod schema;diff --git a/src/prehashed.rs b/src/prehashed.rsnew file mode 100644index 00000000..6d5dd8ec--- /dev/null+++ b/src/prehashed.rs@@ -0,0 +1,132 @@+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ collections::HashMap,+ hash::{BuildHasher, Hash, Hasher},+};++struct PrehashBuilder {}++impl BuildHasher for PrehashBuilder {+ type Hasher = Prehasher;++ fn build_hasher(&self) -> Self::Hasher {+ Prehasher { val: 0 }+ }+}++struct Prehasher {+ val: u128,+}++impl Hasher for Prehasher {+ fn finish(&self) -> u64 {+ self.val as u64+ }++ fn write(&mut self, _bytes: &[u8]) {+ unimplemented!("This hasher only supports u128")+ }++ fn write_u128(&mut self, i: u128) {+ self.val = i;+ }+}++#[derive(Debug, Eq, Clone)]+pub struct Prehashed(pub u128);++impl Hash for Prehashed {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ state.write_u128(self.0);+ }+}++impl PartialEq for Prehashed {+ fn eq(&self, other: &Self) -> bool {+ self.0 == other.0+ }+}++pub fn hash<T: AsRef<[u8]>>(val: T) -> Prehashed {+ let digest = md5::compute(val);+ Prehashed(u128::from_be_bytes(*digest))+}++pub fn split_u128(num: u128) -> [u64; 2] {+ [(num >> 64) as u64, num as u64]+}++pub fn combine_u64s(nums: [u64; 2]) -> u128 {+ ((nums[0] as u128) << 64) | (nums[1] as u128)+}++pub struct PrehashMap<T> {+ map: HashMap<Prehashed, T>,+}++impl<T> PrehashMap<T> {+ pub fn new() -> Self {+ Self {+ map: HashMap::new(),+ }+ }++ pub fn insert(&mut self, key: Prehashed, val: T) {+ self.map.insert(key, val);+ }++ pub fn remove(&mut self, key: &Prehashed) {+ self.map.remove(key);+ }++ pub fn get_mut(&mut self, key: &Prehashed) -> Option<&mut T> {+ self.map.get_mut(key)+ }++ pub fn get(&self, key: &Prehashed) -> Option<&T> {+ self.map.get(key)+ }++ pub fn len(&self) -> usize {+ self.map.len()+ }++ pub fn is_empty(&self) -> bool {+ self.len() == 0+ }++ pub fn iter(&self) -> std::collections::hash_map::Iter<Prehashed, T> {+ self.map.iter()+ }+}++impl<T> Default for PrehashMap<T> {+ fn default() -> Self {+ Self::new()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn split_combine_u128() {+ for num in 0..10000_u128 {+ assert_eq!(combine_u64s(split_u128(num)), num);+ }+ }+}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 92971a81..5c7dc382 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -23,9 +23,10 @@ pub mod signal;use std::sync::Arc;use initial::InitialScoreTweaker;-use tantivy::collector::{Collector, TopDocs};+use tantivy::collector::Collector;use crate::{+ collector::TopDocs,searcher::NUM_RESULTS_PER_PAGE,webpage::region::{Region, RegionCount},};@@ -565,7 +566,8 @@ mod tests {r#"<html><head>- <title>Example website</title>+ <title>Test website</title>+ <meta property="og:description" content="example" /></head><body>test@@ -575,7 +577,7 @@ mod tests {"https://www.centrality.com",),backlinks: vec![],- host_centrality: 1.0002,+ host_centrality: 1.02,fetch_time_ms: 500,page_centrality: 0.0,primary_image: None,diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex 531b35bf..83ff85cf 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -134,7 +134,7 @@ impl Signal {match self {Signal::Bm25 => 1.0,Signal::HostCentrality => 512.0,- Signal::PageCentrality => 1024.0,+ Signal::PageCentrality => 1512.0,Signal::IsHomepage => 0.1,Signal::FetchTimeMs => 0.1,Signal::UpdateTimestamp => 80.0,diff --git a/src/schema.rs b/src/schema.rsindex 85ce550a..c3c2b057 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -48,9 +48,10 @@ pub enum Field {NumTitleTokens,NumCleanBodyTokens,NumDescriptionTokens,+ SiteHash,}-pub static ALL_FIELDS: [Field; 23] = [+pub static ALL_FIELDS: [Field; 24] = [Field::Title,Field::CleanBody,Field::StemmedTitle,@@ -74,6 +75,7 @@ pub static ALL_FIELDS: [Field; 23] = [Field::NumTitleTokens,Field::NumCleanBodyTokens,Field::NumDescriptionTokens,+ Field::SiteHash,];impl Field {@@ -168,6 +170,9 @@ impl Field {.set_fast(Cardinality::SingleValue).set_indexed(),),+ Field::SiteHash => IndexingOption::Numeric(+ NumericOptions::default().set_fast(Cardinality::MultiValues),+ ),}}@@ -196,6 +201,7 @@ impl Field {Field::NumTitleTokens => "num_title_tokens",Field::NumCleanBodyTokens => "num_clean_body_tokens",Field::NumDescriptionTokens => "num_description_tokens",+ Field::SiteHash => "site_hash",}}@@ -212,6 +218,7 @@ impl Field {Field::BacklinkText => Some(4.0),Field::HostCentrality| Field::PageCentrality+ | Field::SiteHash| Field::IsHomepage| Field::PrimaryImage| Field::FetchTimeMs@@ -243,6 +250,7 @@ impl Field {| Field::NumTitleTokens| Field::NumCleanBodyTokens| Field::NumDescriptionTokens+ | Field::SiteHash)}@@ -267,6 +275,7 @@ impl Field {"all_body" => Some(Field::AllBody),"num_trackers" => Some(Field::NumTrackers),"region" => Some(Field::Region),+ "site_hash" => Some(Field::SiteHash),_ => None,}}diff --git a/src/searcher.rs b/src/searcher.rsindex b58c3149..2df572f4 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -214,7 +214,7 @@ mod tests {let searcher = Searcher::new(index, None, None);for p in 0..NUM_PAGES {- let urls = searcher+ let urls: Vec<_> = searcher.search("test", None, None, Some(p)).unwrap().into_websites()@@ -222,9 +222,12 @@ mod tests {.webpages.documents.into_iter()- .map(|page| page.url);+ .map(|page| page.url)+ .collect();- for (i, url) in urls.enumerate() {+ assert!(!urls.is_empty());++ for (i, url) in urls.into_iter().enumerate() {assert_eq!(url,format!("https://www.{}.com", i + (p * NUM_RESULTS_PER_PAGE))diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex b9f1d891..f4fc2a8e 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -13,7 +13,11 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{schema_org::SchemaOrg, tokenizer, Error, Result};+use crate::{+ prehashed::{hash, split_u128},+ schema_org::SchemaOrg,+ tokenizer, Error, Result,+};use chrono::{DateTime, FixedOffset};use itertools::Itertools;use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};@@ -579,6 +583,12 @@ impl Html {Field::NumDescriptionTokens => {doc.add_u64(tantivy_field, description.tokens.len() as u64)}+ Field::SiteHash => {+ let hash = hash(self.url().site()).0;+ let u64s = split_u128(hash);+ doc.add_u64(tantivy_field, u64s[0]);+ doc.add_u64(tantivy_field, u64s[1]);+ }Field::BacklinkText| Field::HostCentrality| Field::PageCentrality