Skip to content

web search done right

way faster bitvec similarity calculations by approximating intersection for sites with many inbound links with hyperloglog

Mikkel Denker63118f1a226b
diff --git a/Cargo.lock b/Cargo.lock
index 6ddbc6f5..9646e570 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4234,6 +4234,7 @@ dependencies = [
"eventsource-stream",
"fend-core",
"flate2",
+ "fnv",
"fst",
"futures",
"half 2.3.1",
diff --git a/Cargo.toml b/Cargo.toml
index ecc9f3ec..93615be6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -35,6 +35,7 @@ encoding_rs = "0.8.31"
eventsource-stream = "0.2.3"
fend-core = "1.2.2"
flate2 = "1.0.28"
+fnv = "1.0.3"
fst = {version = "0.4.7", features = ["levenshtein"]}
futures = "0.3.21"
half = {version = "2.2.1", features = ["serde"]}
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 1f93fb5d..624ded77 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -45,6 +45,7 @@ encoding_rs = {workspace = true}
eventsource-stream = {workspace = true}
fend-core = {workspace = true}
flate2 = {workspace = true}
+fnv = {workspace = true}
fst = {workspace = true}
futures = {workspace = true}
half = {workspace = true}
diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rs
index 6349fc41..cb59e606 100644
--- a/core/src/hyperloglog.rs
+++ b/core/src/hyperloglog.rs
@@ -18,6 +18,12 @@
// It seemed a bit excessive to add another dependency just for hyperloglog
// but huge credits to the authors of that crate.
+use std::{
+ collections::hash_map::DefaultHasher,
+ hash::{Hash as _, Hasher as _},
+ marker::PhantomData,
+};
+
use serde::{Deserialize, Serialize};
pub(crate) const THRESHOLD_DATA_OFFSET: usize = 4;
@@ -4295,25 +4301,51 @@ const ONE_OVER_POWER_OF_TWO: [f64; 256] = [
1.727233711018889e-77,
];
+pub trait HyperLogLogHasher: std::fmt::Debug + Clone + Copy + Send + Sync + 'static {
+ fn hash(item: u64) -> u64;
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct FastHasher;
+
+impl HyperLogLogHasher for FastHasher {
+ #[inline]
+ fn hash(item: u64) -> u64 {
+ item.wrapping_mul(11400714819323198549)
+ }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct StableHasher;
+
+impl HyperLogLogHasher for StableHasher {
+ #[inline]
+ fn hash(item: u64) -> u64 {
+ let mut hasher = DefaultHasher::new();
+ item.hash(&mut hasher);
+ hasher.finish()
+ }
+}
+
#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct HyperLogLog<const N: usize> {
+pub struct HyperLogLog<const N: usize, H = FastHasher>
+where
+ H: HyperLogLogHasher,
+{
registers: Vec<u8>,
+ _hasher: PhantomData<H>,
}
-impl<const N: usize> Default for HyperLogLog<N> {
+impl<H: HyperLogLogHasher, const N: usize> Default for HyperLogLog<N, H> {
fn default() -> Self {
Self {
registers: vec![0; N],
+ _hasher: PhantomData,
}
}
}
-impl<const N: usize> HyperLogLog<N> {
- #[inline]
- fn hash(item: u64) -> u64 {
- item.wrapping_mul(11400714819323198549)
- }
-
+impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {
#[inline]
fn am(&self) -> f64 {
let m = self.registers.len();
@@ -4336,7 +4368,7 @@ impl<const N: usize> HyperLogLog<N> {
pub fn add(&mut self, item: u64) {
let b = self.b();
- let hash = Self::hash(item) as usize;
+ let hash = H::hash(item) as usize;
let j = hash >> (64 - b);
let w = hash << b;
diff --git a/core/src/intmap.rs b/core/src/intmap.rs
index 71dd3750..3a7cdcce 100644
--- a/core/src/intmap.rs
+++ b/core/src/intmap.rs
@@ -16,12 +16,11 @@
use serde::{Deserialize, Serialize};
-pub trait Key: PartialEq + Copy + std::fmt::Debug {
+pub trait Key: PartialOrd + Ord + PartialEq + Copy + std::fmt::Debug {
const BIG_PRIME: Self;
fn wrapping_mul(self, rhs: Self) -> Self;
- fn bit_and(self, rhs: Self) -> Self;
- fn from_usize(val: usize) -> Self;
+ fn modulus_usize(self, rhs: usize) -> usize;
fn as_usize(self) -> usize;
}
@@ -32,12 +31,8 @@ impl Key for u64 {
self.wrapping_mul(rhs)
}
- fn bit_and(self, rhs: Self) -> Self {
- self & rhs
- }
-
- fn from_usize(val: usize) -> Self {
- val as Self
+ fn modulus_usize(self, rhs: usize) -> usize {
+ self as usize % rhs
}
fn as_usize(self) -> usize {
@@ -76,14 +71,11 @@ impl<K: Key, V> IntMap<K, V> {
}
fn bin_idx(&self, key: &K) -> usize {
- let mask = K::from_usize(self.bins.len() - 1);
- let key = key.wrapping_mul(K::BIG_PRIME);
-
- key.bit_and(mask).as_usize()
+ key.modulus_usize(self.bins.len())
}
pub fn insert(&mut self, key: K, value: V) {
- if self.len >= (self.bins.len() as f64 * 1.2) as usize {
+ if self.len >= (self.bins.len() as f64 * 1.5) as usize {
self.grow();
}
@@ -94,6 +86,7 @@ impl<K: Key, V> IntMap<K, V> {
bin[idx] = (key, value);
} else {
bin.push((key, value));
+ bin.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
self.len += 1;
}
}
@@ -113,14 +106,18 @@ impl<K: Key, V> IntMap<K, V> {
self.bins[bin_idx].push((key, value));
}
}
+
+ for bin in &mut self.bins {
+ bin.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
+ }
}
pub fn get(&self, key: &K) -> Option<&V> {
let bin = self.bin_idx(key);
- self.bins[bin]
- .iter()
- .find(|(stored_key, _)| stored_key == key)
- .map(|(_, val)| val)
+ match self.bins[bin].binary_search_by(|(stored_key, _)| stored_key.cmp(key)) {
+ Ok(idx) => Some(&self.bins[bin][idx].1),
+ Err(_) => None,
+ }
}
pub fn get_mut(&mut self, key: &K) -> Option<&mut V> {
@@ -193,10 +190,6 @@ impl<K: Key> IntSet<K> {
pub fn contains(&self, item: &K) -> bool {
self.map.contains_key(item)
}
-
- pub fn into_iter(self) -> impl Iterator<Item = K> {
- self.map.into_iter().map(|(k, _)| k)
- }
}
impl<K: Key> std::iter::FromIterator<K> for IntSet<K> {
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rs
index f50effe7..6e577ef7 100644
--- a/core/src/ranking/bitvec_similarity.rs
+++ b/core/src/ranking/bitvec_similarity.rs
@@ -17,10 +17,10 @@
use itertools::Itertools;
use serde::{Deserialize, Serialize};
-use crate::hyperloglog::HyperLogLog;
+use crate::hyperloglog::{self, HyperLogLog};
-const THRESHOLD_SIM_ESTIMATE: f64 = 0.1;
-const HYPERLOGLOG_REGISTERS: usize = 128;
+const HYPERLOGLOG_REGISTERS: usize = 2048;
+const MIN_POSTING_SIZE_HYPERLOGLOG: usize = 1024000;
#[derive(Serialize, Deserialize, Debug, Clone)]
struct VeryJankyBloomFilter {
@@ -55,11 +55,6 @@ impl VeryJankyBloomFilter {
}
}
-#[derive(Clone)]
-struct ScratchSpace {
- hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS>,
-}
-
#[derive(Serialize, Deserialize, Debug, Clone)]
struct Posting {
ranks: Vec<u64>,
@@ -102,44 +97,56 @@ impl Posting {
j += 1;
}
std::cmp::Ordering::Less => {
- match self.skip_pointers.get(i / self.skip_size + 1).copied() {
- Some(skip_a) => {
- if skip_a < b {
- i += self.skip_size - i % self.skip_size;
- } else {
- i += 1;
- }
- }
- None => {
- i += 1;
- }
- }
+ i = self.next_skip_index(i, b);
}
std::cmp::Ordering::Greater => {
- match other.skip_pointers.get(j / other.skip_size + 1).copied() {
- Some(skip_b) => {
- if skip_b < a {
- j += other.skip_size - j % other.skip_size;
- } else {
- j += 1;
- }
- }
- None => {
- j += 1;
- }
- }
+ j = other.next_skip_index(j, a);
}
}
}
count
}
+
+ fn next_skip_index(&self, current_index: usize, target: u64) -> usize {
+ let mut index = current_index;
+
+ while (index / self.skip_size) + 1 < self.skip_pointers.len() {
+ let skip_index = (index / self.skip_size) + 1;
+ let skip_value = self.skip_pointers[skip_index];
+
+ if skip_value >= target {
+ break;
+ }
+
+ index = skip_index * self.skip_size;
+
+ if index >= self.ranks.len() {
+ return self.ranks.len();
+ }
+ }
+
+ if index == current_index {
+ index += 1;
+ }
+
+ std::cmp::min(index, self.ranks.len())
+ }
+
+ fn len(&self) -> usize {
+ self.ranks.len()
+ }
+}
+
+#[derive(Clone)]
+struct ScratchSpace {
+ hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS, hyperloglog::StableHasher>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BitVec {
bloom: VeryJankyBloomFilter,
- hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS>,
+ hyperloglog: Option<HyperLogLog<HYPERLOGLOG_REGISTERS, hyperloglog::StableHasher>>,
posting: Posting,
sqrt_len: f64,
}
@@ -152,11 +159,18 @@ impl BitVec {
let len = ranks.len();
let mut bloom = VeryJankyBloomFilter::new(2);
- let mut hyperloglog = HyperLogLog::default();
+ let mut hyperloglog = if ranks.len() > MIN_POSTING_SIZE_HYPERLOGLOG {
+ Some(HyperLogLog::default())
+ } else {
+ None
+ };
for rank in &ranks {
bloom.insert(*rank);
- hyperloglog.add(*rank);
+
+ if let Some(hyperloglog) = &mut hyperloglog {
+ hyperloglog.add(*rank);
+ }
}
let posting = Posting::new(ranks);
@@ -178,22 +192,21 @@ impl BitVec {
return 0.0;
}
- self.hyperloglog
- .merge_into(&other.hyperloglog, &mut scratchspace.hyperloglog);
-
- let union_est = scratchspace.hyperloglog.size();
-
- let intersect_est = (self.posting.ranks.len() + other.posting.ranks.len())
- .checked_sub(union_est)
- .unwrap_or_default();
+ let intersect = match (self.hyperloglog.as_ref(), other.hyperloglog.as_ref()) {
+ (Some(a), Some(b)) => {
+ a.merge_into(b, &mut scratchspace.hyperloglog);
- let sim_est = (intersect_est as f64) / (self.sqrt_len * other.sqrt_len);
+ let union_est = scratchspace.hyperloglog.size() as u64;
- if sim_est < THRESHOLD_SIM_ESTIMATE {
- return 0.0;
- }
+ (self.posting.len() as u64 + other.posting.len() as u64)
+ .checked_sub(union_est)
+ .unwrap_or_default()
+ .min(self.posting.len() as u64)
+ .min(other.posting.len() as u64) as f64
+ }
+ _ => self.posting.intersection_size(&other.posting) as f64,
+ };
- let intersect = self.posting.intersection_size(&other.posting) as f64;
intersect / (self.sqrt_len * other.sqrt_len)
}
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 13571032..fb4208d7 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -22,11 +22,12 @@ use std::{
};
use dashmap::DashMap;
+use fnv::FnvHashMap as HashMap;
+use fnv::FnvHashSet as HashSet;
use rayon::prelude::ParallelIterator;
use serde::{Deserialize, Serialize};
use crate::{
- intmap::{IntMap, IntSet},
webgraph::{NodeID, Webgraph},
Result,
};
@@ -38,8 +39,8 @@ pub struct Scorer {
similarity: bitvec_similarity::BitVecSimilarity,
liked: Vec<NodeScorer>,
disliked: Vec<NodeScorer>,
- vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,
- cache: IntMap<NodeID, f64>,
+ vectors: Arc<HashMap<NodeID, bitvec_similarity::BitVec>>,
+ cache: HashMap<NodeID, f64>,
normalized: bool,
}
@@ -126,14 +127,14 @@ impl Scorer {
#[derive(Serialize, Deserialize, Default)]
pub struct InboundSimilarity {
- vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,
+ vectors: Arc<HashMap<NodeID, bitvec_similarity::BitVec>>,
}
impl InboundSimilarity {
pub fn build(graph: &Webgraph) -> Self {
- let mut vectors = IntMap::new();
+ let mut vectors = HashMap::default();
- let adjacency: DashMap<NodeID, IntSet<NodeID>> = DashMap::new();
+ let adjacency: DashMap<NodeID, HashSet<NodeID>> = DashMap::new();
graph.par_edges().for_each(|edge| {
adjacency.entry(edge.to).or_default().insert(edge.from);
@@ -174,7 +175,7 @@ impl InboundSimilarity {
liked,
disliked,
vectors: self.vectors.clone(),
- cache: IntMap::new(),
+ cache: HashMap::default(),
normalized,
}
}
diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rs
index a94576d4..5dc83060 100644
--- a/core/src/searcher/api.rs
+++ b/core/src/searcher/api.rs
@@ -88,6 +88,7 @@ pub struct ApiSearcher {
#[cfg(not(feature = "libtorch"))]
pub struct ApiSearcher {
distributed_searcher: DistributedSearcher,
+ live_searcher: LiveSearcher,
lambda_model: Option<Arc<LambdaMART>>,
bangs: Bangs,
collector_config: CollectorConfig,
@@ -126,7 +127,8 @@ impl ApiSearcher {
config: ApiConfig,
) -> Self {
Self {
- distributed_searcher: DistributedSearcher::new(cluster),
+ distributed_searcher: DistributedSearcher::new(Arc::clone(&cluster)),
+ live_searcher: LiveSearcher::new(Arc::clone(&cluster)),
lambda_model: lambda_model.map(Arc::new),
bangs,
collector_config: config.collector,
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 428f4e29..eb11fc7d 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -79,17 +79,13 @@ impl intmap::Key for NodeID {
NodeID(self.0.wrapping_mul(rhs.0))
}
- fn bit_and(self, rhs: Self) -> Self {
- NodeID(self.0 & rhs.0)
- }
-
- fn from_usize(val: usize) -> Self {
- NodeID(val as u128)
- }
-
fn as_usize(self) -> usize {
self.0 as usize
}
+
+ fn modulus_usize(self, rhs: usize) -> usize {
+ (self.0 % (rhs as u128)) as usize
+ }
}
pub trait EdgeLabel
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 5f167346..0c71f3ae 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -1435,6 +1435,44 @@ impl Html {
.collect()
}
+ pub fn likely_has_ads(&self) -> bool {
+ for script in self.scripts() {
+ if let Some(url) = script
+ .attributes
+ .get("src")
+ .and_then(|url| Url::parse(url).ok())
+ {
+ if let Some(domain) = url.root_domain() {
+ return matches!(
+ domain,
+ "doubleclick.net" | "googlesyndication.com" | "amazon-adsystem.com"
+ );
+ }
+ }
+ }
+
+ false
+ }
+
+ pub fn likely_has_paywall(&self) -> bool {
+ self.schema_org()
+ .into_iter()
+ .filter(|item| {
+ item.types_contains("NewsArticle")
+ || item.types_contains("Article")
+ || item.types_contains("BlogPosting")
+ || item.types_contains("WebPage")
+ || item.types_contains("WebPageElement")
+ })
+ .any(|item| {
+ item.properties
+ .get("isAccessibleForFree")
+ .and_then(|value| value.clone().one().and_then(|v| v.try_into_string()))
+ .map(|value| value.parse().ok().unwrap_or(false))
+ .unwrap_or(false)
+ })
+ }
+
fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {
self.metadata()
.into_iter()
diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelte
index d74da549..43221259 100644
--- a/frontend/src/routes/search/Result.svelte
+++ b/frontend/src/routes/search/Result.svelte
@@ -1,5 +1,6 @@
<script lang="ts">
import AdjustVertical from '~icons/heroicons/adjustments-vertical';
+ import LockClosed from '~icons/heroicons/lock-closed';
import type { DisplayedWebpage } from '$lib/api';
import { createEventDispatcher } from 'svelte';
import { clearSummary, summariesStore } from '$lib/stores';
@@ -56,6 +57,9 @@
<div class="line-clamp-3">
<div class="inline">
<span id="snippet-text" class="snippet-text">
+ <span>
+ <LockClosed class="inline-block w-4 h-4 mr-1 text-neutral" />
+ </span>
{#if webpage.snippet.date}
<span class="text-neutral">
{webpage.snippet.date} -