Skip to content

web search done right

Ftr/de similar results (#56)

Mikkel Denkerfb8cfe1f30ae
diff --git a/Cargo.lock b/Cargo.lock
index 0000b770..9aed7b52 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -697,6 +697,7 @@ dependencies = [
"maplit",
"md5",
"memmap",
+ "min-max-heap",
"once_cell",
"parse_wiki_text",
"quick-xml",
@@ -1702,6 +1703,12 @@ dependencies = [
"unicase",
]
+[[package]]
+name = "min-max-heap"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2687e6cf9c00f48e9284cf9fd15f2ef341d03cc7743abf9df4c5f07fdee50b18"
+
[[package]]
name = "minimal-lexical"
version = "0.2.1"
diff --git a/Cargo.toml b/Cargo.toml
index d8599537..b72e7b99 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,6 +55,7 @@ tower-http = { version = "0.3.4", features = ["compression-gzip"] }
lalrpop-util = { version = "0.19.8", features = ["lexer"] }
serde_urlencoded = "0.7.1"
rust-stemmers = "1.2.0"
+min-max-heap = "1.3.0"
[dev-dependencies]
criterion = "0.3.6"
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index d4aa3b48..a9bfff1a 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,8 +1,8 @@
warc_paths_file = "warc.paths"
centrality_store_path = "./data/centrality"
-limit_warc_files = 100
+limit_warc_files = 25
download_images = false
-batch_size = 10
+batch_size = 3
[warc_source]
# type = "Local"
diff --git a/src/collector.rs b/src/collector.rs
new file mode 100644
index 00000000..6e8409b2
--- /dev/null
+++ b/src/collector.rs
@@ -0,0 +1,510 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use min_max_heap::MinMaxHeap;
+use tantivy::{
+ collector::{Collector, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector},
+ fastfield::MultiValuedFastFieldReader,
+ DocAddress, DocId, Score, SegmentOrdinal, SegmentReader,
+};
+
+use crate::{
+ prehashed::{combine_u64s, PrehashMap, Prehashed},
+ schema::Field,
+};
+
+fn adjust_score(num_taken: usize, original_score: f64) -> f64 {
+ const SCALE: f64 = 14.0;
+ original_score * (SCALE / (num_taken as f64 + SCALE))
+}
+
+pub struct TopDocs {
+ top_n: usize,
+ offset: usize,
+}
+
+impl TopDocs {
+ pub fn with_limit(top_n: usize) -> Self {
+ Self { top_n, offset: 0 }
+ }
+
+ pub fn and_offset(mut self, offset: usize) -> Self {
+ self.offset = offset;
+ self
+ }
+
+ pub fn tweak_score<TScoreSegmentTweaker, TScoreTweaker>(
+ self,
+ score_tweaker: TScoreTweaker,
+ ) -> impl Collector<Fruit = Vec<(f64, DocAddress)>>
+ where
+ TScoreSegmentTweaker: ScoreSegmentTweaker<f64> + 'static,
+ TScoreTweaker: ScoreTweaker<f64, Child = TScoreSegmentTweaker> + Send + Sync,
+ {
+ TweakedScoreTopCollector::new(score_tweaker, self)
+ }
+}
+
+impl Collector for TopDocs {
+ type Fruit = Vec<(f64, DocAddress)>;
+
+ type Child = TopSegmentCollector;
+
+ fn for_segment(
+ &self,
+ segment_local_id: tantivy::SegmentOrdinal,
+ segment: &tantivy::SegmentReader,
+ ) -> tantivy::Result<Self::Child> {
+ let key_reader = segment.fast_fields().u64s(
+ segment
+ .schema()
+ .get_field(Field::SiteHash.as_str())
+ .unwrap(),
+ )?;
+
+ Ok(TopSegmentCollector {
+ key_reader,
+ segment_ord: segment_local_id,
+ bucket_collector: BucketCollector::new(self.top_n + self.offset),
+ })
+ }
+
+ fn requires_scoring(&self) -> bool {
+ true
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> tantivy::Result<Self::Fruit> {
+ let mut collector = BucketCollector::new(self.top_n + self.offset);
+
+ for docs in segment_fruits {
+ for doc in docs {
+ collector.insert(doc);
+ }
+ }
+
+ Ok(collector
+ .into_sorted_vec(true)
+ .into_iter()
+ .skip(self.offset)
+ .map(|doc| {
+ (
+ doc.score,
+ DocAddress {
+ segment_ord: doc.segment,
+ doc_id: doc.id,
+ },
+ )
+ })
+ .collect())
+ }
+}
+
+pub struct TopSegmentCollector {
+ key_reader: MultiValuedFastFieldReader<u64>,
+ segment_ord: SegmentOrdinal,
+ bucket_collector: BucketCollector,
+}
+
+impl SegmentCollector for TopSegmentCollector {
+ type Fruit = Vec<Doc>;
+
+ fn collect(&mut self, doc: DocId, score: Score) {
+ let mut keys = Vec::new();
+ self.key_reader.get_vals(doc, &mut keys);
+ debug_assert_eq!(keys.len(), 2);
+
+ let keys = [keys[0], keys[1]];
+ let key = combine_u64s(keys);
+
+ self.bucket_collector.insert(Doc {
+ key: Prehashed(key),
+ id: doc,
+ segment: self.segment_ord,
+ score: score as f64,
+ })
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ self.bucket_collector.into_sorted_vec(false)
+ }
+}
+
+struct BucketCollector {
+ buckets: PrehashMap<Bucket>,
+ heads: MinMaxHeap<BucketHead>,
+ top_n: usize,
+}
+
+impl BucketCollector {
+ pub fn new(top_n: usize) -> Self {
+ assert!(top_n > 0);
+
+ Self {
+ top_n,
+ heads: MinMaxHeap::with_capacity(top_n * 2 + 1),
+ buckets: PrehashMap::new(),
+ }
+ }
+
+ pub fn insert(&mut self, doc: Doc) {
+ if let Some(bucket) = self.buckets.get_mut(&doc.key) {
+ bucket.insert(doc);
+ } else {
+ let mut bucket = Bucket::new(self.top_n);
+ bucket.insert(doc.clone());
+
+ self.buckets.insert(doc.key.clone(), bucket);
+
+ self.heads.push(BucketHead {
+ key: doc.key,
+ tweaked_score: doc.score,
+ });
+ }
+
+ if self.buckets.len() > (self.top_n * 2) + 1 {
+ self.prune_buckets()
+ }
+ }
+
+ fn prune_buckets(&mut self) {
+ self.update_worst_head();
+ let worst_head = self.heads.pop_min().unwrap();
+ self.buckets.remove(&worst_head.key);
+ }
+
+ fn update_worst_head(&mut self) {
+ loop {
+ let mut worst_head = self.heads.peek_min_mut().unwrap();
+ let current_score = self
+ .buckets
+ .get(&worst_head.key)
+ .unwrap()
+ .get_best()
+ .unwrap()
+ .tweaked_score;
+
+ if worst_head.tweaked_score != current_score {
+ worst_head.tweaked_score = current_score;
+ } else {
+ break;
+ }
+ }
+ }
+
+ fn build_heads(&self) -> MinMaxHeap<BucketHead> {
+ let mut bucket_heads: MinMaxHeap<BucketHead> = MinMaxHeap::with_capacity(self.top_n);
+
+ for (key, bucket) in self.buckets.iter() {
+ let best_in_bucket = bucket.get_best().unwrap();
+ if bucket_heads.len() >= self.top_n {
+ let mut worst_head = bucket_heads.peek_min_mut().unwrap();
+
+ if best_in_bucket.tweaked_score > worst_head.tweaked_score {
+ worst_head.key = key.clone();
+ worst_head.tweaked_score = best_in_bucket.tweaked_score
+ }
+ } else {
+ bucket_heads.push(BucketHead {
+ key: key.clone(),
+ tweaked_score: best_in_bucket.tweaked_score,
+ });
+ }
+ }
+
+ bucket_heads
+ }
+
+ pub fn into_sorted_vec(mut self, apply_adjust_score: bool) -> Vec<Doc> {
+ let mut res = Vec::new();
+
+ let mut bucket_heads = self.build_heads();
+
+ while let Some(mut head) = bucket_heads.pop_max() {
+ let bucket = self.buckets.get_mut(&head.key).unwrap();
+
+ if let Some(mut doc) = bucket.pop_best() {
+ if apply_adjust_score {
+ doc.score = adjust_score(bucket.num_taken - 1, doc.score);
+ }
+
+ res.push(doc);
+
+ if let Some(new_best) = bucket.get_best() {
+ head.tweaked_score = new_best.tweaked_score;
+ }
+
+ bucket_heads.push(head);
+ }
+
+ if res.len() == self.top_n {
+ break;
+ }
+ }
+
+ res
+ }
+}
+
+struct BucketHead {
+ key: Prehashed,
+ tweaked_score: f64,
+}
+
+impl PartialEq for BucketHead {
+ fn eq(&self, other: &Self) -> bool {
+ self.tweaked_score == other.tweaked_score
+ }
+}
+
+impl Eq for BucketHead {}
+
+impl PartialOrd for BucketHead {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.tweaked_score.partial_cmp(&other.tweaked_score)
+ }
+}
+
+impl Ord for BucketHead {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
+ }
+}
+
+struct Bucket {
+ num_taken: usize,
+ docs: MinMaxHeap<Doc>,
+ top_n: usize,
+}
+
+impl Bucket {
+ pub fn new(top_n: usize) -> Self {
+ assert!(top_n > 0);
+
+ Self {
+ top_n,
+ num_taken: 0,
+ docs: MinMaxHeap::with_capacity(top_n),
+ }
+ }
+
+ pub fn pop_best(&mut self) -> Option<Doc> {
+ let res = self.docs.pop_max();
+
+ self.num_taken += 1;
+
+ res
+ }
+
+ pub fn get_best(&self) -> Option<TweakedDoc<'_>> {
+ self.docs.peek_max().map(|doc| TweakedDoc {
+ tweaked_score: adjust_score(self.num_taken, doc.score),
+ _doc_id: &doc.id,
+ })
+ }
+
+ pub fn insert(&mut self, doc: Doc) {
+ if self.docs.len() >= self.top_n {
+ let mut worst = self.docs.peek_min_mut().unwrap();
+
+ worst.id = doc.id;
+ worst.score = doc.score;
+ } else {
+ self.docs.push(doc);
+ }
+ }
+}
+
+struct TweakedDoc<'a> {
+ tweaked_score: f64,
+ _doc_id: &'a DocId,
+}
+
+#[derive(Debug, Clone)]
+pub struct Doc {
+ key: Prehashed,
+ id: DocId,
+ segment: SegmentOrdinal,
+ score: f64,
+}
+
+impl PartialEq for Doc {
+ fn eq(&self, other: &Self) -> bool {
+ self.score == other.score
+ }
+}
+
+impl Eq for Doc {}
+
+impl PartialOrd for Doc {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ self.score.partial_cmp(&other.score)
+ }
+}
+
+impl Ord for Doc {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.partial_cmp(other).unwrap_or(std::cmp::Ordering::Equal)
+ }
+}
+
+pub(crate) struct TweakedScoreTopCollector<TScoreTweaker> {
+ score_tweaker: TScoreTweaker,
+ collector: TopDocs,
+}
+
+impl<TScoreTweaker> TweakedScoreTopCollector<TScoreTweaker> {
+ pub fn new(
+ score_tweaker: TScoreTweaker,
+ collector: TopDocs,
+ ) -> TweakedScoreTopCollector<TScoreTweaker> {
+ TweakedScoreTopCollector {
+ score_tweaker,
+ collector,
+ }
+ }
+}
+
+impl<TScoreTweaker> Collector for TweakedScoreTopCollector<TScoreTweaker>
+where
+ TScoreTweaker: ScoreTweaker<f64> + Send + Sync,
+{
+ type Fruit = Vec<(f64, DocAddress)>;
+
+ type Child = TopTweakedScoreSegmentCollector<TScoreTweaker::Child>;
+
+ fn for_segment(
+ &self,
+ segment_local_id: u32,
+ segment_reader: &SegmentReader,
+ ) -> tantivy::Result<Self::Child> {
+ let segment_scorer = self.score_tweaker.segment_tweaker(segment_reader)?;
+ let segment_collector = self
+ .collector
+ .for_segment(segment_local_id, segment_reader)?;
+ Ok(TopTweakedScoreSegmentCollector {
+ segment_collector,
+ segment_scorer,
+ })
+ }
+
+ fn requires_scoring(&self) -> bool {
+ true
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> tantivy::Result<Self::Fruit> {
+ self.collector.merge_fruits(segment_fruits)
+ }
+}
+
+pub struct TopTweakedScoreSegmentCollector<TSegmentScoreTweaker>
+where
+ TSegmentScoreTweaker: ScoreSegmentTweaker<f64>,
+{
+ segment_collector: TopSegmentCollector,
+ segment_scorer: TSegmentScoreTweaker,
+}
+
+impl<TSegmentScoreTweaker> SegmentCollector
+ for TopTweakedScoreSegmentCollector<TSegmentScoreTweaker>
+where
+ TSegmentScoreTweaker: 'static + ScoreSegmentTweaker<f64>,
+{
+ type Fruit = Vec<Doc>;
+
+ fn collect(&mut self, doc: DocId, score: Score) {
+ let score = self.segment_scorer.score(doc, score);
+ self.segment_collector.collect(doc, score as f32);
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ self.segment_collector.harvest()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn test(top_n: usize, docs: &[(u128, DocId, f64)], expected: &[(f64, DocId)]) {
+ let mut collector = BucketCollector::new(top_n);
+
+ for doc in docs {
+ collector.insert(Doc {
+ key: Prehashed(doc.0),
+ id: doc.1,
+ score: doc.2,
+ segment: 0,
+ });
+ }
+
+ let res: Vec<(f64, DocId)> = collector
+ .into_sorted_vec(true)
+ .into_iter()
+ .map(|doc| (doc.score, doc.id))
+ .collect();
+
+ assert_eq!(&res, expected);
+ }
+
+ #[test]
+ fn all_different() {
+ test(
+ 3,
+ &[
+ (1, 123, 1.0),
+ (2, 124, 2.0),
+ (3, 125, 3.0),
+ (4, 126, 4.0),
+ (5, 127, 5.0),
+ ],
+ &[(5.0, 127), (4.0, 126), (3.0, 125)],
+ );
+ }
+
+ #[test]
+ fn less_than_topn() {
+ test(
+ 10,
+ &[(3, 125, 3.0), (4, 126, 4.0), (5, 127, 5.0)],
+ &[(5.0, 127), (4.0, 126), (3.0, 125)],
+ );
+ }
+
+ #[test]
+ fn same_key_de_prioritised() {
+ test(
+ 10,
+ &[(1, 125, 3.0), (2, 126, 3.1), (2, 127, 5.0)],
+ &[
+ (adjust_score(0, 5.0), 127),
+ (adjust_score(0, 3.0), 125),
+ (adjust_score(1, 3.1), 126),
+ ],
+ );
+
+ test(
+ 2,
+ &[(1, 125, 3.0), (2, 126, 3.1), (2, 127, 5.0)],
+ &[(adjust_score(0, 5.0), 127), (adjust_score(0, 3.0), 125)],
+ );
+ }
+}
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 77cf2741..d59b1c0f 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -371,6 +371,7 @@ impl From<Document> for RetrievedWebpage {
| Field::Domain
| Field::DomainIfHomepage
| Field::IsHomepage
+ | Field::SiteHash
| Field::NumTrackers
| Field::NumCleanBodyTokens
| Field::NumDescriptionTokens
diff --git a/src/lib.rs b/src/lib.rs
index 93e8557c..be985de5 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -34,6 +34,7 @@ pub mod mapreduce;
mod autosuggest;
mod bangs;
+mod collector;
mod directory;
mod entity_index;
mod exponential_backoff;
@@ -42,6 +43,7 @@ mod image_downloader;
mod image_store;
pub mod index;
mod kv;
+pub mod prehashed;
mod query;
mod ranking;
mod schema;
diff --git a/src/prehashed.rs b/src/prehashed.rs
new file mode 100644
index 00000000..6d5dd8ec
--- /dev/null
+++ b/src/prehashed.rs
@@ -0,0 +1,132 @@
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ collections::HashMap,
+ hash::{BuildHasher, Hash, Hasher},
+};
+
+struct PrehashBuilder {}
+
+impl BuildHasher for PrehashBuilder {
+ type Hasher = Prehasher;
+
+ fn build_hasher(&self) -> Self::Hasher {
+ Prehasher { val: 0 }
+ }
+}
+
+struct Prehasher {
+ val: u128,
+}
+
+impl Hasher for Prehasher {
+ fn finish(&self) -> u64 {
+ self.val as u64
+ }
+
+ fn write(&mut self, _bytes: &[u8]) {
+ unimplemented!("This hasher only supports u128")
+ }
+
+ fn write_u128(&mut self, i: u128) {
+ self.val = i;
+ }
+}
+
+#[derive(Debug, Eq, Clone)]
+pub struct Prehashed(pub u128);
+
+impl Hash for Prehashed {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ state.write_u128(self.0);
+ }
+}
+
+impl PartialEq for Prehashed {
+ fn eq(&self, other: &Self) -> bool {
+ self.0 == other.0
+ }
+}
+
+pub fn hash<T: AsRef<[u8]>>(val: T) -> Prehashed {
+ let digest = md5::compute(val);
+ Prehashed(u128::from_be_bytes(*digest))
+}
+
+pub fn split_u128(num: u128) -> [u64; 2] {
+ [(num >> 64) as u64, num as u64]
+}
+
+pub fn combine_u64s(nums: [u64; 2]) -> u128 {
+ ((nums[0] as u128) << 64) | (nums[1] as u128)
+}
+
+pub struct PrehashMap<T> {
+ map: HashMap<Prehashed, T>,
+}
+
+impl<T> PrehashMap<T> {
+ pub fn new() -> Self {
+ Self {
+ map: HashMap::new(),
+ }
+ }
+
+ pub fn insert(&mut self, key: Prehashed, val: T) {
+ self.map.insert(key, val);
+ }
+
+ pub fn remove(&mut self, key: &Prehashed) {
+ self.map.remove(key);
+ }
+
+ pub fn get_mut(&mut self, key: &Prehashed) -> Option<&mut T> {
+ self.map.get_mut(key)
+ }
+
+ pub fn get(&self, key: &Prehashed) -> Option<&T> {
+ self.map.get(key)
+ }
+
+ pub fn len(&self) -> usize {
+ self.map.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ pub fn iter(&self) -> std::collections::hash_map::Iter<Prehashed, T> {
+ self.map.iter()
+ }
+}
+
+impl<T> Default for PrehashMap<T> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn split_combine_u128() {
+ for num in 0..10000_u128 {
+ assert_eq!(combine_u64s(split_u128(num)), num);
+ }
+ }
+}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 92971a81..5c7dc382 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -23,9 +23,10 @@ pub mod signal;
use std::sync::Arc;
use initial::InitialScoreTweaker;
-use tantivy::collector::{Collector, TopDocs};
+use tantivy::collector::Collector;
use crate::{
+ collector::TopDocs,
searcher::NUM_RESULTS_PER_PAGE,
webpage::region::{Region, RegionCount},
};
@@ -565,7 +566,8 @@ mod tests {
r#"
<html>
<head>
- <title>Example website</title>
+ <title>Test website</title>
+ <meta property="og:description" content="example" />
</head>
<body>
test
@@ -575,7 +577,7 @@ mod tests {
"https://www.centrality.com",
),
backlinks: vec![],
- host_centrality: 1.0002,
+ host_centrality: 1.02,
fetch_time_ms: 500,
page_centrality: 0.0,
primary_image: None,
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index 531b35bf..83ff85cf 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -134,7 +134,7 @@ impl Signal {
match self {
Signal::Bm25 => 1.0,
Signal::HostCentrality => 512.0,
- Signal::PageCentrality => 1024.0,
+ Signal::PageCentrality => 1512.0,
Signal::IsHomepage => 0.1,
Signal::FetchTimeMs => 0.1,
Signal::UpdateTimestamp => 80.0,
diff --git a/src/schema.rs b/src/schema.rs
index 85ce550a..c3c2b057 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -48,9 +48,10 @@ pub enum Field {
NumTitleTokens,
NumCleanBodyTokens,
NumDescriptionTokens,
+ SiteHash,
}
-pub static ALL_FIELDS: [Field; 23] = [
+pub static ALL_FIELDS: [Field; 24] = [
Field::Title,
Field::CleanBody,
Field::StemmedTitle,
@@ -74,6 +75,7 @@ pub static ALL_FIELDS: [Field; 23] = [
Field::NumTitleTokens,
Field::NumCleanBodyTokens,
Field::NumDescriptionTokens,
+ Field::SiteHash,
];
impl Field {
@@ -168,6 +170,9 @@ impl Field {
.set_fast(Cardinality::SingleValue)
.set_indexed(),
),
+ Field::SiteHash => IndexingOption::Numeric(
+ NumericOptions::default().set_fast(Cardinality::MultiValues),
+ ),
}
}
@@ -196,6 +201,7 @@ impl Field {
Field::NumTitleTokens => "num_title_tokens",
Field::NumCleanBodyTokens => "num_clean_body_tokens",
Field::NumDescriptionTokens => "num_description_tokens",
+ Field::SiteHash => "site_hash",
}
}
@@ -212,6 +218,7 @@ impl Field {
Field::BacklinkText => Some(4.0),
Field::HostCentrality
| Field::PageCentrality
+ | Field::SiteHash
| Field::IsHomepage
| Field::PrimaryImage
| Field::FetchTimeMs
@@ -243,6 +250,7 @@ impl Field {
| Field::NumTitleTokens
| Field::NumCleanBodyTokens
| Field::NumDescriptionTokens
+ | Field::SiteHash
)
}
@@ -267,6 +275,7 @@ impl Field {
"all_body" => Some(Field::AllBody),
"num_trackers" => Some(Field::NumTrackers),
"region" => Some(Field::Region),
+ "site_hash" => Some(Field::SiteHash),
_ => None,
}
}
diff --git a/src/searcher.rs b/src/searcher.rs
index b58c3149..2df572f4 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -214,7 +214,7 @@ mod tests {
let searcher = Searcher::new(index, None, None);
for p in 0..NUM_PAGES {
- let urls = searcher
+ let urls: Vec<_> = searcher
.search("test", None, None, Some(p))
.unwrap()
.into_websites()
@@ -222,9 +222,12 @@ mod tests {
.webpages
.documents
.into_iter()
- .map(|page| page.url);
+ .map(|page| page.url)
+ .collect();
- for (i, url) in urls.enumerate() {
+ assert!(!urls.is_empty());
+
+ for (i, url) in urls.into_iter().enumerate() {
assert_eq!(
url,
format!("https://www.{}.com", i + (p * NUM_RESULTS_PER_PAGE))
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index b9f1d891..f4fc2a8e 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -13,7 +13,11 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{schema_org::SchemaOrg, tokenizer, Error, Result};
+use crate::{
+ prehashed::{hash, split_u128},
+ schema_org::SchemaOrg,
+ tokenizer, Error, Result,
+};
use chrono::{DateTime, FixedOffset};
use itertools::Itertools;
use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
@@ -579,6 +583,12 @@ impl Html {
Field::NumDescriptionTokens => {
doc.add_u64(tantivy_field, description.tokens.len() as u64)
}
+ Field::SiteHash => {
+ let hash = hash(self.url().site()).0;
+ let u64s = split_u128(hash);
+ doc.add_u64(tantivy_field, u64s[0]);
+ doc.add_u64(tantivy_field, u64s[1]);
+ }
Field::BacklinkText
| Field::HostCentrality
| Field::PageCentrality