web search done right
Remove duplicate search results based on simhash
Mikkel Denker848d2e3c8921
diff --git a/Cargo.lock b/Cargo.lockindex 66293c89..2246aff5 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -760,6 +760,7 @@ dependencies = ["once_cell","parse_wiki_text","quick-xml",+ "rand 0.8.5","rayon","regex","reqwest",diff --git a/Cargo.toml b/Cargo.tomlindex 3fd7c9ed..4a4d7c1b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -70,6 +70,7 @@ rusoto_core = { version = "0.48.0", default-features = false, features = ["rustl[dev-dependencies]criterion = "0.3.6"maplit = "1.0.2"+rand = "0.8.5"[build-dependencies]lalrpop = { version = "0.19.8", features = ["lexer"] }diff --git a/src/collector.rs b/src/collector.rsindex 63dbd33c..899bc2a8 100644--- a/src/collector.rs+++ b/src/collector.rs@@ -28,6 +28,7 @@ use crate::{inverted_index::{DocAddress, WebsitePointer},prehashed::{combine_u64s, Prehashed},schema::FastField,+ simhash,};// lower scale -> higher penalty@@ -46,6 +47,7 @@ pub struct Hashes {site: Prehashed,title: Prehashed,url: Prehashed,+ simhash: simhash::HashType,}pub trait Doc: Clone {@@ -198,6 +200,11 @@ impl SegmentCollector for TopSegmentCollector {site: self.get_hash(&doc, &FastField::SiteHash),title: self.get_hash(&doc, &FastField::TitleHash),url: self.get_hash(&doc, &FastField::UrlHash),+ simhash: self+ .fastfield_segment_cache+ .get_doc_cache(&FastField::SimHash)+ .get_u64(&doc)+ .unwrap(),},id: doc,segment: self.segment_ord,@@ -325,8 +332,18 @@ impl<T: Doc> BucketCollector<T> {pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {let mut res = Vec::new();+ let mut simhash = simhash::Table::default();while let Some(best_doc) = self.documents.pop_max() {+ let hashes = best_doc.doc.hashes();++ if hashes.simhash != 0 {+ if simhash.contains(&hashes.simhash) {+ continue;+ }+ simhash.insert(hashes.simhash);+ }+if de_rank_similar {self.count.update_counts(&best_doc);self.update_best_doc();@@ -476,6 +493,7 @@ mod tests {site: 1.into(),title: 1.into(),url: 1.into(),+ simhash: 12,},123,1.0,@@ -485,6 +503,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ simhash: 123,},124,2.0,@@ -494,6 +513,7 @@ mod tests {site: 3.into(),title: 3.into(),url: 3.into(),+ simhash: 1234,},125,3.0,@@ -503,6 +523,7 @@ mod tests {site: 4.into(),title: 4.into(),url: 4.into(),+ simhash: 12345,},126,4.0,@@ -512,6 +533,7 @@ mod tests {site: 5.into(),title: 5.into(),url: 5.into(),+ simhash: 123456,},127,5.0,@@ -531,6 +553,7 @@ mod tests {site: 3.into(),title: 3.into(),url: 3.into(),+ simhash: 12,},125,3.0,@@ -540,6 +563,7 @@ mod tests {site: 4.into(),title: 4.into(),url: 4.into(),+ simhash: 123,},126,4.0,@@ -549,6 +573,7 @@ mod tests {site: 5.into(),title: 5.into(),url: 5.into(),+ simhash: 1234,},127,5.0,@@ -568,6 +593,7 @@ mod tests {site: 1.into(),title: 1.into(),url: 1.into(),+ simhash: 12,},125,3.0,@@ -577,6 +603,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ simhash: 123,},126,3.1,@@ -586,6 +613,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ simhash: 1234,},127,5.0,@@ -602,6 +630,7 @@ mod tests {site: 1.into(),title: 1.into(),url: 1.into(),+ simhash: 12,},125,3.0,@@ -611,6 +640,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ simhash: 123,},126,3.1,@@ -620,6 +650,7 @@ mod tests {site: 2.into(),title: 2.into(),url: 2.into(),+ simhash: 1234,},127,5.0,@@ -628,4 +659,44 @@ mod tests {&[(5.0, 127), (3.0, 125)],);}++ #[test]+ fn simhash_dedup() {+ test(+ 10,+ &[+ (+ Hashes {+ site: 1.into(),+ title: 1.into(),+ url: 1.into(),+ simhash: 1234,+ },+ 125,+ 3.0,+ ),+ (+ Hashes {+ site: 2.into(),+ title: 2.into(),+ url: 2.into(),+ simhash: 1234,+ },+ 126,+ 3.1,+ ),+ (+ Hashes {+ site: 3.into(),+ title: 3.into(),+ url: 3.into(),+ simhash: 1,+ },+ 127,+ 5.0,+ ),+ ],+ &[(5.0, 127), (3.1, 126)],+ );+ }}diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 8b79f4a0..1f7c5ede 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -757,6 +757,8 @@ mod tests {);for _ in 0..100 {+ let dedup_s = crate::rand_words(100);+index.insert(Webpage::new(&format!(@@ -766,7 +768,7 @@ mod tests {<title>Website for runners</title></head><body>- {CONTENT}+ {CONTENT} {dedup_s}</body></html>"#@@ -839,10 +841,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.example.com",))@@ -859,10 +862,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.example.com",))@@ -1065,4 +1069,65 @@ mod tests {assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.example.com");}++ #[test]+ fn remove_duplicates() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");+ let query = Query::parse(+ "dr",+ index.schema(),+ index.tokenizers(),+ &SignalAggregator::default(),+ )+ .expect("Failed to parse query");+ let ranker = Ranker::new(+ RegionCount::default(),+ SignalAggregator::default(),+ index.fastfield_cache(),+ );++ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>News website</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.dr.xyz",+ ))+ .expect("failed to insert webpage");++ index+ .insert(Webpage::new(+ &format!(+ r#"+ <html>+ <head>+ <title>News website</title>+ </head>+ <body>+ {CONTENT} dr+ </body>+ </html>+ "#+ ),+ "https://www.dr.dk",+ ))+ .expect("failed to insert webpage");++ index.commit().expect("failed to commit index");++ let result = index+ .search(&query, ranker.collector())+ .expect("Search failed");+ assert_eq!(result.documents.len(), 1);+ assert_eq!(result.documents[0].url, "https://www.dr.dk");+ }}diff --git a/src/lib.rs b/src/lib.rsindex 0eca3671..4cb0486c 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -51,6 +51,7 @@ mod schema;mod schema_org;mod search_prettifier;pub mod searcher;+mod simhash;mod snippet;mod sonic;mod spell;@@ -233,7 +234,6 @@ pub enum Error {pub(crate) type Result<T> = std::result::Result<T, Error>;// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445-#[allow(unused)]fn gen_temp_path() -> PathBuf {use std::sync::atomic::{AtomicUsize, Ordering};use std::time::SystemTime;@@ -259,3 +259,23 @@ fn gen_temp_path() -> PathBuf {std::env::temp_dir().join(format!("pagecache.tmp.{}", salt))}}++#[cfg(test)]+fn rand_words(num_words: usize) -> String {+ use rand::{distributions::Alphanumeric, Rng};+ let mut res = String::new();++ for _ in 0..num_words {+ res.push_str(+ rand::thread_rng()+ .sample_iter(&Alphanumeric)+ .take(30)+ .map(char::from)+ .collect::<String>()+ .as_str(),+ );+ res.push(' ');+ }++ res.trim().to_string()+}diff --git a/src/prehashed.rs b/src/prehashed.rsindex 0863a4f9..a049e72f 100644--- a/src/prehashed.rs+++ b/src/prehashed.rs@@ -1,3 +1,5 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -12,10 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{- collections::HashMap,- hash::{BuildHasher, Hash, Hasher},-};+use std::hash::{BuildHasher, Hash, Hasher};use serde::{Deserialize, Serialize};@@ -81,52 +80,6 @@ pub fn combine_u64s(nums: [u64; 2]) -> u128 {((nums[0] as u128) << 64) | (nums[1] as u128)}-pub struct PrehashMap<T> {- map: HashMap<Prehashed, T>,-}--impl<T> PrehashMap<T> {- pub fn new() -> Self {- Self {- map: HashMap::new(),- }- }-- pub fn insert(&mut self, key: Prehashed, val: T) {- self.map.insert(key, val);- }-- pub fn remove(&mut self, key: &Prehashed) {- self.map.remove(key);- }-- pub fn get_mut(&mut self, key: &Prehashed) -> Option<&mut T> {- self.map.get_mut(key)- }-- pub fn get(&self, key: &Prehashed) -> Option<&T> {- self.map.get(key)- }-- pub fn len(&self) -> usize {- self.map.len()- }-- pub fn is_empty(&self) -> bool {- self.len() == 0- }-- pub fn iter(&self) -> std::collections::hash_map::Iter<Prehashed, T> {- self.map.iter()- }-}--impl<T> Default for PrehashMap<T> {- fn default() -> Self {- Self::new()- }-}-#[cfg(test)]mod tests {use super::*;diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex 5fbe6a1f..4b5023d4 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -427,11 +427,12 @@ mod tests {<title>Website A</title></head><body>- {CONTENT}+ {CONTENT} {}example example example</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.a.com",),@@ -454,10 +455,11 @@ mod tests {<title>Website B</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.b.com",),@@ -669,11 +671,12 @@ mod tests {<title>Website A</title></head><body>- {CONTENT}+ {CONTENT} {}example example example</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.a.com/this/is/a/pattern",),@@ -696,10 +699,11 @@ mod tests {<title>Website B</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.b.com/this/is/b/pattern",),@@ -722,10 +726,11 @@ mod tests {<title>Website B</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.c.com/this/is/c/pattern",),@@ -802,11 +807,12 @@ mod tests {<title>Website A</title></head><body>- {CONTENT}+ {CONTENT} {}example example example</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.a.com/this/is/a/pattern",),@@ -835,10 +841,11 @@ mod tests {<title>Website B</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.b.com/this/is/b/pattern",),@@ -867,10 +874,11 @@ mod tests {<title>Website B</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.c.com/this/is/c/pattern",),diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 8c167a28..df15de30 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -136,11 +136,12 @@ mod tests {<title>Website A</title></head><body>- {CONTENT}+ {CONTENT} {}example example example</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.a.com",),@@ -163,10 +164,11 @@ mod tests {<title>Website B</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.b.com",),@@ -212,11 +214,12 @@ mod tests {<title>Website A</title></head><body>- {CONTENT}+ {CONTENT} {}example example example</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.a.com",),@@ -239,10 +242,11 @@ mod tests {<title>Website B</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.b.com",),@@ -288,10 +292,11 @@ mod tests {<title>DR Homepage</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.dr.dk",),@@ -314,10 +319,11 @@ mod tests {<title>Subsite dr</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.dr.dk/whatever",),@@ -339,9 +345,10 @@ mod tests {<head><title>Website B</title></head>- dr and some other text {CONTENT} dk+ dr and some other text {CONTENT} dk {}</html>- "#+ "#,+ crate::rand_words(100)),"https://www.b.com",),@@ -767,10 +774,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT} termA termB d d d d d d d d d+ {CONTENT} termA termB d d d d d d d d d {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.first.com",),@@ -793,10 +801,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT} termA d d d d d d d d d termB+ {CONTENT} termA d d d d d d d d d termB {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.third.com",),@@ -819,10 +828,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT} termA d d d d termB d d d d d+ {CONTENT} termA d d d d termB d d d d d {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.second.com",),@@ -872,10 +882,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.first.com",),@@ -898,10 +909,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.second.com",),diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex fe38f808..2326bb33 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -33,7 +33,7 @@ use crate::{use crate::ranking::goggles::ast::{RawAlteration, Target};-#[derive(Debug, Clone, PartialEq, Eq, Hash)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum Signal {Bm25,HostCentrality,diff --git a/src/ranking/site_rankings.rs b/src/ranking/site_rankings.rsindex 36cc9cc3..5a624ab7 100644--- a/src/ranking/site_rankings.rs+++ b/src/ranking/site_rankings.rs@@ -137,10 +137,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.first.com",),@@ -170,10 +171,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.second.com",),@@ -203,10 +205,11 @@ mod tests {<title>Test website</title></head><body>- {CONTENT}+ {CONTENT} {}</body></html>- "#+ "#,+ crate::rand_words(100)),"https://www.third.com",),diff --git a/src/schema.rs b/src/schema.rsindex e545d3a3..3ea4606a 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -65,6 +65,7 @@ pub enum FastField {DomainHash,PreComputedScore,HostNodeID,+ SimHash,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -73,7 +74,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 34] = [+pub static ALL_FIELDS: [Field; 35] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -109,6 +110,7 @@ pub static ALL_FIELDS: [Field; 34] = [Field::Fast(FastField::DomainHash),Field::Fast(FastField::PreComputedScore),Field::Fast(FastField::HostNodeID),+ Field::Fast(FastField::SimHash),];impl Field {@@ -256,6 +258,12 @@ impl Field {.set_indexed().set_stored(),),+ Field::Fast(FastField::SimHash) => IndexingOption::Integer(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed()+ .set_stored(),+ ),}}@@ -297,6 +305,7 @@ impl Field {Field::Fast(FastField::UrlHash) => "url_hash",Field::Fast(FastField::DomainHash) => "domain_hash",Field::Fast(FastField::HostNodeID) => "host_node_id",+ Field::Fast(FastField::SimHash) => "sim_hash",}}@@ -366,6 +375,7 @@ impl Field {"domain_hash" => Some(Field::Fast(FastField::DomainHash)),"title_hash" => Some(Field::Fast(FastField::TitleHash)),"host_node_id" => Some(Field::Fast(FastField::HostNodeID)),+ "sim_hash" => Some(Field::Fast(FastField::SimHash)),_ => None,}}@@ -434,6 +444,7 @@ impl FastField {FastField::DomainHash => DataType::U64s,FastField::PreComputedScore => DataType::F64,FastField::HostNodeID => DataType::U64,+ FastField::SimHash => DataType::U64,}}}diff --git a/src/simhash.rs b/src/simhash.rsnew file mode 100644index 00000000..7e47aace--- /dev/null+++ b/src/simhash.rs@@ -0,0 +1,194 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ collections::{hash_map::DefaultHasher, HashMap},+ hash::{Hash, Hasher},+};++use crate::tokenizer::Tokenizer;++pub type HashType = u64;++fn hash_token(token: &tantivy::tokenizer::Token) -> HashType {+ let mut hasher = DefaultHasher::default();+ token.text.as_str().hash(&mut hasher);+ hasher.finish()+}++pub fn hash(text: &str) -> HashType {+ let tokenizer = Tokenizer::default();++ let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, text);++ let mut v = [0i64; HashType::BITS as usize];++ while let Some(token) = stream.next() {+ let h = hash_token(token);++ for (i, item) in v.iter_mut().enumerate() {+ let bit = (h >> i) & 1;+ if bit == 1 {+ *item = item.saturating_add(1);+ } else {+ *item = item.saturating_sub(1);+ }+ }+ }++ let mut simhash: HashType = 0;+ for (i, item) in v.iter().enumerate() {+ if *item > 0 {+ simhash |= 1 << i;+ }+ }++ simhash+}++fn hamming_distance(x: HashType, y: HashType) -> u32 {+ (x ^ y).count_ones()+}++#[derive(PartialEq, Eq, Hash)]+struct Prefix(HashType);++struct Block {+ hashes: HashMap<Prefix, Vec<HashType>>,+ mask: HashType,+}++impl Block {+ fn new(block_idx: usize) -> Self {+ Self {+ hashes: Default::default(),+ mask: (HashType::MAX << (HashType::BITS as usize - BLOCK_SIZE))+ >> (BLOCK_SIZE * block_idx),+ }+ }+}++impl Block {+ fn insert(&mut self, hash: HashType) {+ let prefix = Prefix(hash & self.mask);++ self.hashes.entry(prefix).or_default().push(hash)+ }++ /// true iff. `Block` has indexed a hash that is within `K` distance away from the query hash.+ fn contains(&self, hash: &HashType) -> bool {+ let prefix = Prefix(*hash & self.mask);++ match self.hashes.get(&prefix) {+ Some(candidates) => candidates+ .iter()+ .any(|candidate| hamming_distance(*hash, *candidate) as usize <= K),+ None => false,+ }+ }+}++// TODO: When `generic_const_exprs` becomes stable, re-write this to use const-generics+const K: usize = 3;+const NUM_BLOCKS: usize = K + 1;+const BLOCK_SIZE: usize = HashType::BITS as usize / NUM_BLOCKS;++pub struct Table {+ blocks: [Block; NUM_BLOCKS],+}++impl Default for Table {+ fn default() -> Self {+ Self {+ blocks: std::array::from_fn(Block::new),+ }+ }+}++impl Table {+ pub fn insert(&mut self, hash: HashType) {+ for block in &mut self.blocks {+ block.insert(hash);+ }+ }++ /// true iff. `Table` has indexed a hash that is within `K` distance away from the query hash.+ pub fn contains(&self, hash: &HashType) -> bool {+ self.blocks.iter().any(|block| block.contains(hash))+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ /// Calculate similarity as `f64` of two hashes+ /// 0.0 means no similarity, 1.0 means identical+ pub fn similarity(hash1: HashType, hash2: HashType) -> f64 {+ let distance: f64 = hamming_distance(hash1, hash2) as f64;+ 1.0 - (distance / (HashType::BITS as f64))+ }++ #[test]+ fn simhash_test() {+ assert_eq!(hash("The cat sat on the mat"), 1696787384511938835);+ assert_eq!(hash("The cat sat under the mat"), 1557175861565382659);+ assert_eq!(hash("Why the lucky stiff"), 2343560682201631264);+ }++ #[test]+ fn hamming_distance_test() {+ assert_eq!(+ hamming_distance(0b0000000u64 as HashType, 0b0000000u64 as HashType),+ 0+ );+ assert_eq!(+ hamming_distance(0b1111111u64 as HashType, 0b0000000u64 as HashType),+ 7+ );+ assert_eq!(+ hamming_distance(0b0100101u64 as HashType, 0b1100110u64 as HashType),+ 3+ );+ }++ #[test]+ fn similarity_test() {+ assert_eq!(+ similarity(hash("Stop hammertime"), hash("Stop hammertime")),+ 1.0+ );+ assert!(+ similarity(hash("Hocus pocus"), hash("Hocus pocus pilatus pas"))+ > similarity(hash("This should"), hash("not overlap"))+ );+ }++ #[test]+ fn table() {+ let mut table = Table::default();++ let h1 = 0b0000000u64 as HashType;+ let h2 = 0b0000001u64 as HashType;+ let h3 = 0b1111111u64 as HashType;++ assert!(!table.contains(&h1));++ table.insert(h1);+ assert!(table.contains(&h2));+ assert!(!table.contains(&h3));+ }+}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 561a9d03..fe19a422 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -17,7 +17,7 @@ use crate::{prehashed::{hash, split_u128},schema::{FastField, TextField},schema_org::SchemaOrg,- tokenizer, Error, Result,+ simhash, tokenizer, Error, Result,};use chrono::{DateTime, FixedOffset, Utc};use itertools::Itertools;@@ -700,6 +700,14 @@ impl Html {doc.add_u64(tantivy_field, u64s[0]);doc.add_u64(tantivy_field, u64s[1]);}+ Field::Fast(FastField::SimHash) => {+ let hash = if !clean_text.text.is_empty() {+ simhash::hash(&clean_text.text)+ } else {+ 0+ };+ doc.add_u64(tantivy_field, hash);+ }Field::Text(TextField::BacklinkText)| Field::Fast(FastField::HostCentrality)| Field::Fast(FastField::PageCentrality)