Skip to content

web search done right

Remove duplicate search results based on simhash

Mikkel Denker848d2e3c8921
diff --git a/Cargo.lock b/Cargo.lock
index 66293c89..2246aff5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -760,6 +760,7 @@ dependencies = [
"once_cell",
"parse_wiki_text",
"quick-xml",
+ "rand 0.8.5",
"rayon",
"regex",
"reqwest",
diff --git a/Cargo.toml b/Cargo.toml
index 3fd7c9ed..4a4d7c1b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -70,6 +70,7 @@ rusoto_core = { version = "0.48.0", default-features = false, features = ["rustl
[dev-dependencies]
criterion = "0.3.6"
maplit = "1.0.2"
+rand = "0.8.5"
[build-dependencies]
lalrpop = { version = "0.19.8", features = ["lexer"] }
diff --git a/src/collector.rs b/src/collector.rs
index 63dbd33c..899bc2a8 100644
--- a/src/collector.rs
+++ b/src/collector.rs
@@ -28,6 +28,7 @@ use crate::{
inverted_index::{DocAddress, WebsitePointer},
prehashed::{combine_u64s, Prehashed},
schema::FastField,
+ simhash,
};
// lower scale -> higher penalty
@@ -46,6 +47,7 @@ pub struct Hashes {
site: Prehashed,
title: Prehashed,
url: Prehashed,
+ simhash: simhash::HashType,
}
pub trait Doc: Clone {
@@ -198,6 +200,11 @@ impl SegmentCollector for TopSegmentCollector {
site: self.get_hash(&doc, &FastField::SiteHash),
title: self.get_hash(&doc, &FastField::TitleHash),
url: self.get_hash(&doc, &FastField::UrlHash),
+ simhash: self
+ .fastfield_segment_cache
+ .get_doc_cache(&FastField::SimHash)
+ .get_u64(&doc)
+ .unwrap(),
},
id: doc,
segment: self.segment_ord,
@@ -325,8 +332,18 @@ impl<T: Doc> BucketCollector<T> {
pub fn into_sorted_vec(mut self, de_rank_similar: bool) -> Vec<T> {
let mut res = Vec::new();
+ let mut simhash = simhash::Table::default();
while let Some(best_doc) = self.documents.pop_max() {
+ let hashes = best_doc.doc.hashes();
+
+ if hashes.simhash != 0 {
+ if simhash.contains(&hashes.simhash) {
+ continue;
+ }
+ simhash.insert(hashes.simhash);
+ }
+
if de_rank_similar {
self.count.update_counts(&best_doc);
self.update_best_doc();
@@ -476,6 +493,7 @@ mod tests {
site: 1.into(),
title: 1.into(),
url: 1.into(),
+ simhash: 12,
},
123,
1.0,
@@ -485,6 +503,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ simhash: 123,
},
124,
2.0,
@@ -494,6 +513,7 @@ mod tests {
site: 3.into(),
title: 3.into(),
url: 3.into(),
+ simhash: 1234,
},
125,
3.0,
@@ -503,6 +523,7 @@ mod tests {
site: 4.into(),
title: 4.into(),
url: 4.into(),
+ simhash: 12345,
},
126,
4.0,
@@ -512,6 +533,7 @@ mod tests {
site: 5.into(),
title: 5.into(),
url: 5.into(),
+ simhash: 123456,
},
127,
5.0,
@@ -531,6 +553,7 @@ mod tests {
site: 3.into(),
title: 3.into(),
url: 3.into(),
+ simhash: 12,
},
125,
3.0,
@@ -540,6 +563,7 @@ mod tests {
site: 4.into(),
title: 4.into(),
url: 4.into(),
+ simhash: 123,
},
126,
4.0,
@@ -549,6 +573,7 @@ mod tests {
site: 5.into(),
title: 5.into(),
url: 5.into(),
+ simhash: 1234,
},
127,
5.0,
@@ -568,6 +593,7 @@ mod tests {
site: 1.into(),
title: 1.into(),
url: 1.into(),
+ simhash: 12,
},
125,
3.0,
@@ -577,6 +603,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ simhash: 123,
},
126,
3.1,
@@ -586,6 +613,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ simhash: 1234,
},
127,
5.0,
@@ -602,6 +630,7 @@ mod tests {
site: 1.into(),
title: 1.into(),
url: 1.into(),
+ simhash: 12,
},
125,
3.0,
@@ -611,6 +640,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ simhash: 123,
},
126,
3.1,
@@ -620,6 +650,7 @@ mod tests {
site: 2.into(),
title: 2.into(),
url: 2.into(),
+ simhash: 1234,
},
127,
5.0,
@@ -628,4 +659,44 @@ mod tests {
&[(5.0, 127), (3.0, 125)],
);
}
+
+ #[test]
+ fn simhash_dedup() {
+ test(
+ 10,
+ &[
+ (
+ Hashes {
+ site: 1.into(),
+ title: 1.into(),
+ url: 1.into(),
+ simhash: 1234,
+ },
+ 125,
+ 3.0,
+ ),
+ (
+ Hashes {
+ site: 2.into(),
+ title: 2.into(),
+ url: 2.into(),
+ simhash: 1234,
+ },
+ 126,
+ 3.1,
+ ),
+ (
+ Hashes {
+ site: 3.into(),
+ title: 3.into(),
+ url: 3.into(),
+ simhash: 1,
+ },
+ 127,
+ 5.0,
+ ),
+ ],
+ &[(5.0, 127), (3.1, 126)],
+ );
+ }
}
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 8b79f4a0..1f7c5ede 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -757,6 +757,8 @@ mod tests {
);
for _ in 0..100 {
+ let dedup_s = crate::rand_words(100);
+
index
.insert(Webpage::new(
&format!(
@@ -766,7 +768,7 @@ mod tests {
<title>Website for runners</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {dedup_s}
</body>
</html>
"#
@@ -839,10 +841,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.example.com",
))
@@ -859,10 +862,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.example.com",
))
@@ -1065,4 +1069,65 @@ mod tests {
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
}
+
+ #[test]
+ fn remove_duplicates() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let query = Query::parse(
+ "dr",
+ index.schema(),
+ index.tokenizers(),
+ &SignalAggregator::default(),
+ )
+ .expect("Failed to parse query");
+ let ranker = Ranker::new(
+ RegionCount::default(),
+ SignalAggregator::default(),
+ index.fastfield_cache(),
+ );
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>News website</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.dr.xyz",
+ ))
+ .expect("failed to insert webpage");
+
+ index
+ .insert(Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>News website</title>
+ </head>
+ <body>
+ {CONTENT} dr
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.dr.dk",
+ ))
+ .expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+
+ let result = index
+ .search(&query, ranker.collector())
+ .expect("Search failed");
+ assert_eq!(result.documents.len(), 1);
+ assert_eq!(result.documents[0].url, "https://www.dr.dk");
+ }
}
diff --git a/src/lib.rs b/src/lib.rs
index 0eca3671..4cb0486c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -51,6 +51,7 @@ mod schema;
mod schema_org;
mod search_prettifier;
pub mod searcher;
+mod simhash;
mod snippet;
mod sonic;
mod spell;
@@ -233,7 +234,6 @@ pub enum Error {
pub(crate) type Result<T> = std::result::Result<T, Error>;
// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445
-#[allow(unused)]
fn gen_temp_path() -> PathBuf {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::SystemTime;
@@ -259,3 +259,23 @@ fn gen_temp_path() -> PathBuf {
std::env::temp_dir().join(format!("pagecache.tmp.{}", salt))
}
}
+
+#[cfg(test)]
+fn rand_words(num_words: usize) -> String {
+ use rand::{distributions::Alphanumeric, Rng};
+ let mut res = String::new();
+
+ for _ in 0..num_words {
+ res.push_str(
+ rand::thread_rng()
+ .sample_iter(&Alphanumeric)
+ .take(30)
+ .map(char::from)
+ .collect::<String>()
+ .as_str(),
+ );
+ res.push(' ');
+ }
+
+ res.trim().to_string()
+}
diff --git a/src/prehashed.rs b/src/prehashed.rs
index 0863a4f9..a049e72f 100644
--- a/src/prehashed.rs
+++ b/src/prehashed.rs
@@ -1,3 +1,5 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -12,10 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{
- collections::HashMap,
- hash::{BuildHasher, Hash, Hasher},
-};
+use std::hash::{BuildHasher, Hash, Hasher};
use serde::{Deserialize, Serialize};
@@ -81,52 +80,6 @@ pub fn combine_u64s(nums: [u64; 2]) -> u128 {
((nums[0] as u128) << 64) | (nums[1] as u128)
}
-pub struct PrehashMap<T> {
- map: HashMap<Prehashed, T>,
-}
-
-impl<T> PrehashMap<T> {
- pub fn new() -> Self {
- Self {
- map: HashMap::new(),
- }
- }
-
- pub fn insert(&mut self, key: Prehashed, val: T) {
- self.map.insert(key, val);
- }
-
- pub fn remove(&mut self, key: &Prehashed) {
- self.map.remove(key);
- }
-
- pub fn get_mut(&mut self, key: &Prehashed) -> Option<&mut T> {
- self.map.get_mut(key)
- }
-
- pub fn get(&self, key: &Prehashed) -> Option<&T> {
- self.map.get(key)
- }
-
- pub fn len(&self) -> usize {
- self.map.len()
- }
-
- pub fn is_empty(&self) -> bool {
- self.len() == 0
- }
-
- pub fn iter(&self) -> std::collections::hash_map::Iter<Prehashed, T> {
- self.map.iter()
- }
-}
-
-impl<T> Default for PrehashMap<T> {
- fn default() -> Self {
- Self::new()
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 5fbe6a1f..4b5023d4 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -427,11 +427,12 @@ mod tests {
<title>Website A</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
example example example
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.a.com",
),
@@ -454,10 +455,11 @@ mod tests {
<title>Website B</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.b.com",
),
@@ -669,11 +671,12 @@ mod tests {
<title>Website A</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
example example example
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.a.com/this/is/a/pattern",
),
@@ -696,10 +699,11 @@ mod tests {
<title>Website B</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.b.com/this/is/b/pattern",
),
@@ -722,10 +726,11 @@ mod tests {
<title>Website B</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.c.com/this/is/c/pattern",
),
@@ -802,11 +807,12 @@ mod tests {
<title>Website A</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
example example example
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.a.com/this/is/a/pattern",
),
@@ -835,10 +841,11 @@ mod tests {
<title>Website B</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.b.com/this/is/b/pattern",
),
@@ -867,10 +874,11 @@ mod tests {
<title>Website B</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.c.com/this/is/c/pattern",
),
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 8c167a28..df15de30 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -136,11 +136,12 @@ mod tests {
<title>Website A</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
example example example
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.a.com",
),
@@ -163,10 +164,11 @@ mod tests {
<title>Website B</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.b.com",
),
@@ -212,11 +214,12 @@ mod tests {
<title>Website A</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
example example example
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.a.com",
),
@@ -239,10 +242,11 @@ mod tests {
<title>Website B</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.b.com",
),
@@ -288,10 +292,11 @@ mod tests {
<title>DR Homepage</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.dr.dk",
),
@@ -314,10 +319,11 @@ mod tests {
<title>Subsite dr</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.dr.dk/whatever",
),
@@ -339,9 +345,10 @@ mod tests {
<head>
<title>Website B</title>
</head>
- dr and some other text {CONTENT} dk
+ dr and some other text {CONTENT} dk {}
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.b.com",
),
@@ -767,10 +774,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT} termA termB d d d d d d d d d
+ {CONTENT} termA termB d d d d d d d d d {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.first.com",
),
@@ -793,10 +801,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT} termA d d d d d d d d d termB
+ {CONTENT} termA d d d d d d d d d termB {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.third.com",
),
@@ -819,10 +828,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT} termA d d d d termB d d d d d
+ {CONTENT} termA d d d d termB d d d d d {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.second.com",
),
@@ -872,10 +882,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.first.com",
),
@@ -898,10 +909,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.second.com",
),
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index fe38f808..2326bb33 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -33,7 +33,7 @@ use crate::{
use crate::ranking::goggles::ast::{RawAlteration, Target};
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Signal {
Bm25,
HostCentrality,
diff --git a/src/ranking/site_rankings.rs b/src/ranking/site_rankings.rs
index 36cc9cc3..5a624ab7 100644
--- a/src/ranking/site_rankings.rs
+++ b/src/ranking/site_rankings.rs
@@ -137,10 +137,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.first.com",
),
@@ -170,10 +171,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.second.com",
),
@@ -203,10 +205,11 @@ mod tests {
<title>Test website</title>
</head>
<body>
- {CONTENT}
+ {CONTENT} {}
</body>
</html>
- "#
+ "#,
+ crate::rand_words(100)
),
"https://www.third.com",
),
diff --git a/src/schema.rs b/src/schema.rs
index e545d3a3..3ea4606a 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -65,6 +65,7 @@ pub enum FastField {
DomainHash,
PreComputedScore,
HostNodeID,
+ SimHash,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -73,7 +74,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 34] = [
+pub static ALL_FIELDS: [Field; 35] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -109,6 +110,7 @@ pub static ALL_FIELDS: [Field; 34] = [
Field::Fast(FastField::DomainHash),
Field::Fast(FastField::PreComputedScore),
Field::Fast(FastField::HostNodeID),
+ Field::Fast(FastField::SimHash),
];
impl Field {
@@ -256,6 +258,12 @@ impl Field {
.set_indexed()
.set_stored(),
),
+ Field::Fast(FastField::SimHash) => IndexingOption::Integer(
+ NumericOptions::default()
+ .set_fast(Cardinality::SingleValue)
+ .set_indexed()
+ .set_stored(),
+ ),
}
}
@@ -297,6 +305,7 @@ impl Field {
Field::Fast(FastField::UrlHash) => "url_hash",
Field::Fast(FastField::DomainHash) => "domain_hash",
Field::Fast(FastField::HostNodeID) => "host_node_id",
+ Field::Fast(FastField::SimHash) => "sim_hash",
}
}
@@ -366,6 +375,7 @@ impl Field {
"domain_hash" => Some(Field::Fast(FastField::DomainHash)),
"title_hash" => Some(Field::Fast(FastField::TitleHash)),
"host_node_id" => Some(Field::Fast(FastField::HostNodeID)),
+ "sim_hash" => Some(Field::Fast(FastField::SimHash)),
_ => None,
}
}
@@ -434,6 +444,7 @@ impl FastField {
FastField::DomainHash => DataType::U64s,
FastField::PreComputedScore => DataType::F64,
FastField::HostNodeID => DataType::U64,
+ FastField::SimHash => DataType::U64,
}
}
}
diff --git a/src/simhash.rs b/src/simhash.rs
new file mode 100644
index 00000000..7e47aace
--- /dev/null
+++ b/src/simhash.rs
@@ -0,0 +1,194 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ collections::{hash_map::DefaultHasher, HashMap},
+ hash::{Hash, Hasher},
+};
+
+use crate::tokenizer::Tokenizer;
+
+pub type HashType = u64;
+
+fn hash_token(token: &tantivy::tokenizer::Token) -> HashType {
+ let mut hasher = DefaultHasher::default();
+ token.text.as_str().hash(&mut hasher);
+ hasher.finish()
+}
+
+pub fn hash(text: &str) -> HashType {
+ let tokenizer = Tokenizer::default();
+
+ let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, text);
+
+ let mut v = [0i64; HashType::BITS as usize];
+
+ while let Some(token) = stream.next() {
+ let h = hash_token(token);
+
+ for (i, item) in v.iter_mut().enumerate() {
+ let bit = (h >> i) & 1;
+ if bit == 1 {
+ *item = item.saturating_add(1);
+ } else {
+ *item = item.saturating_sub(1);
+ }
+ }
+ }
+
+ let mut simhash: HashType = 0;
+ for (i, item) in v.iter().enumerate() {
+ if *item > 0 {
+ simhash |= 1 << i;
+ }
+ }
+
+ simhash
+}
+
+fn hamming_distance(x: HashType, y: HashType) -> u32 {
+ (x ^ y).count_ones()
+}
+
+#[derive(PartialEq, Eq, Hash)]
+struct Prefix(HashType);
+
+struct Block {
+ hashes: HashMap<Prefix, Vec<HashType>>,
+ mask: HashType,
+}
+
+impl Block {
+ fn new(block_idx: usize) -> Self {
+ Self {
+ hashes: Default::default(),
+ mask: (HashType::MAX << (HashType::BITS as usize - BLOCK_SIZE))
+ >> (BLOCK_SIZE * block_idx),
+ }
+ }
+}
+
+impl Block {
+ fn insert(&mut self, hash: HashType) {
+ let prefix = Prefix(hash & self.mask);
+
+ self.hashes.entry(prefix).or_default().push(hash)
+ }
+
+ /// true iff. `Block` has indexed a hash that is within `K` distance away from the query hash.
+ fn contains(&self, hash: &HashType) -> bool {
+ let prefix = Prefix(*hash & self.mask);
+
+ match self.hashes.get(&prefix) {
+ Some(candidates) => candidates
+ .iter()
+ .any(|candidate| hamming_distance(*hash, *candidate) as usize <= K),
+ None => false,
+ }
+ }
+}
+
+// TODO: When `generic_const_exprs` becomes stable, re-write this to use const-generics
+const K: usize = 3;
+const NUM_BLOCKS: usize = K + 1;
+const BLOCK_SIZE: usize = HashType::BITS as usize / NUM_BLOCKS;
+
+pub struct Table {
+ blocks: [Block; NUM_BLOCKS],
+}
+
+impl Default for Table {
+ fn default() -> Self {
+ Self {
+ blocks: std::array::from_fn(Block::new),
+ }
+ }
+}
+
+impl Table {
+ pub fn insert(&mut self, hash: HashType) {
+ for block in &mut self.blocks {
+ block.insert(hash);
+ }
+ }
+
+ /// true iff. `Table` has indexed a hash that is within `K` distance away from the query hash.
+ pub fn contains(&self, hash: &HashType) -> bool {
+ self.blocks.iter().any(|block| block.contains(hash))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Calculate similarity as `f64` of two hashes
+ /// 0.0 means no similarity, 1.0 means identical
+ pub fn similarity(hash1: HashType, hash2: HashType) -> f64 {
+ let distance: f64 = hamming_distance(hash1, hash2) as f64;
+ 1.0 - (distance / (HashType::BITS as f64))
+ }
+
+ #[test]
+ fn simhash_test() {
+ assert_eq!(hash("The cat sat on the mat"), 1696787384511938835);
+ assert_eq!(hash("The cat sat under the mat"), 1557175861565382659);
+ assert_eq!(hash("Why the lucky stiff"), 2343560682201631264);
+ }
+
+ #[test]
+ fn hamming_distance_test() {
+ assert_eq!(
+ hamming_distance(0b0000000u64 as HashType, 0b0000000u64 as HashType),
+ 0
+ );
+ assert_eq!(
+ hamming_distance(0b1111111u64 as HashType, 0b0000000u64 as HashType),
+ 7
+ );
+ assert_eq!(
+ hamming_distance(0b0100101u64 as HashType, 0b1100110u64 as HashType),
+ 3
+ );
+ }
+
+ #[test]
+ fn similarity_test() {
+ assert_eq!(
+ similarity(hash("Stop hammertime"), hash("Stop hammertime")),
+ 1.0
+ );
+ assert!(
+ similarity(hash("Hocus pocus"), hash("Hocus pocus pilatus pas"))
+ > similarity(hash("This should"), hash("not overlap"))
+ );
+ }
+
+ #[test]
+ fn table() {
+ let mut table = Table::default();
+
+ let h1 = 0b0000000u64 as HashType;
+ let h2 = 0b0000001u64 as HashType;
+ let h3 = 0b1111111u64 as HashType;
+
+ assert!(!table.contains(&h1));
+
+ table.insert(h1);
+ assert!(table.contains(&h2));
+ assert!(!table.contains(&h3));
+ }
+}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 561a9d03..fe19a422 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -17,7 +17,7 @@ use crate::{
prehashed::{hash, split_u128},
schema::{FastField, TextField},
schema_org::SchemaOrg,
- tokenizer, Error, Result,
+ simhash, tokenizer, Error, Result,
};
use chrono::{DateTime, FixedOffset, Utc};
use itertools::Itertools;
@@ -700,6 +700,14 @@ impl Html {
doc.add_u64(tantivy_field, u64s[0]);
doc.add_u64(tantivy_field, u64s[1]);
}
+ Field::Fast(FastField::SimHash) => {
+ let hash = if !clean_text.text.is_empty() {
+ simhash::hash(&clean_text.text)
+ } else {
+ 0
+ };
+ doc.add_u64(tantivy_field, hash);
+ }
Field::Text(TextField::BacklinkText)
| Field::Fast(FastField::HostCentrality)
| Field::Fast(FastField::PageCentrality)