web search done right
way faster bitvec similarity calculations by approximating intersection for sites with many inbound links with hyperloglog
Mikkel Denker63118f1a226b
diff --git a/Cargo.lock b/Cargo.lockindex 6ddbc6f5..9646e570 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4234,6 +4234,7 @@ dependencies = ["eventsource-stream","fend-core","flate2",+ "fnv","fst","futures","half 2.3.1",diff --git a/Cargo.toml b/Cargo.tomlindex ecc9f3ec..93615be6 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -35,6 +35,7 @@ encoding_rs = "0.8.31"eventsource-stream = "0.2.3"fend-core = "1.2.2"flate2 = "1.0.28"+fnv = "1.0.3"fst = {version = "0.4.7", features = ["levenshtein"]}futures = "0.3.21"half = {version = "2.2.1", features = ["serde"]}diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 1f93fb5d..624ded77 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -45,6 +45,7 @@ encoding_rs = {workspace = true}eventsource-stream = {workspace = true}fend-core = {workspace = true}flate2 = {workspace = true}+fnv = {workspace = true}fst = {workspace = true}futures = {workspace = true}half = {workspace = true}diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rsindex 6349fc41..cb59e606 100644--- a/core/src/hyperloglog.rs+++ b/core/src/hyperloglog.rs@@ -18,6 +18,12 @@// It seemed a bit excessive to add another dependency just for hyperloglog// but huge credits to the authors of that crate.+use std::{+ collections::hash_map::DefaultHasher,+ hash::{Hash as _, Hasher as _},+ marker::PhantomData,+};+use serde::{Deserialize, Serialize};pub(crate) const THRESHOLD_DATA_OFFSET: usize = 4;@@ -4295,25 +4301,51 @@ const ONE_OVER_POWER_OF_TWO: [f64; 256] = [1.727233711018889e-77,];+pub trait HyperLogLogHasher: std::fmt::Debug + Clone + Copy + Send + Sync + 'static {+ fn hash(item: u64) -> u64;+}++#[derive(Clone, Copy, Debug)]+pub struct FastHasher;++impl HyperLogLogHasher for FastHasher {+ #[inline]+ fn hash(item: u64) -> u64 {+ item.wrapping_mul(11400714819323198549)+ }+}++#[derive(Clone, Copy, Debug)]+pub struct StableHasher;++impl HyperLogLogHasher for StableHasher {+ #[inline]+ fn hash(item: u64) -> u64 {+ let mut hasher = DefaultHasher::new();+ item.hash(&mut hasher);+ hasher.finish()+ }+}+#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct HyperLogLog<const N: usize> {+pub struct HyperLogLog<const N: usize, H = FastHasher>+where+ H: HyperLogLogHasher,+{registers: Vec<u8>,+ _hasher: PhantomData<H>,}-impl<const N: usize> Default for HyperLogLog<N> {+impl<H: HyperLogLogHasher, const N: usize> Default for HyperLogLog<N, H> {fn default() -> Self {Self {registers: vec![0; N],+ _hasher: PhantomData,}}}-impl<const N: usize> HyperLogLog<N> {- #[inline]- fn hash(item: u64) -> u64 {- item.wrapping_mul(11400714819323198549)- }-+impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {#[inline]fn am(&self) -> f64 {let m = self.registers.len();@@ -4336,7 +4368,7 @@ impl<const N: usize> HyperLogLog<N> {pub fn add(&mut self, item: u64) {let b = self.b();- let hash = Self::hash(item) as usize;+ let hash = H::hash(item) as usize;let j = hash >> (64 - b);let w = hash << b;diff --git a/core/src/intmap.rs b/core/src/intmap.rsindex 71dd3750..3a7cdcce 100644--- a/core/src/intmap.rs+++ b/core/src/intmap.rs@@ -16,12 +16,11 @@use serde::{Deserialize, Serialize};-pub trait Key: PartialEq + Copy + std::fmt::Debug {+pub trait Key: PartialOrd + Ord + PartialEq + Copy + std::fmt::Debug {const BIG_PRIME: Self;fn wrapping_mul(self, rhs: Self) -> Self;- fn bit_and(self, rhs: Self) -> Self;- fn from_usize(val: usize) -> Self;+ fn modulus_usize(self, rhs: usize) -> usize;fn as_usize(self) -> usize;}@@ -32,12 +31,8 @@ impl Key for u64 {self.wrapping_mul(rhs)}- fn bit_and(self, rhs: Self) -> Self {- self & rhs- }-- fn from_usize(val: usize) -> Self {- val as Self+ fn modulus_usize(self, rhs: usize) -> usize {+ self as usize % rhs}fn as_usize(self) -> usize {@@ -76,14 +71,11 @@ impl<K: Key, V> IntMap<K, V> {}fn bin_idx(&self, key: &K) -> usize {- let mask = K::from_usize(self.bins.len() - 1);- let key = key.wrapping_mul(K::BIG_PRIME);-- key.bit_and(mask).as_usize()+ key.modulus_usize(self.bins.len())}pub fn insert(&mut self, key: K, value: V) {- if self.len >= (self.bins.len() as f64 * 1.2) as usize {+ if self.len >= (self.bins.len() as f64 * 1.5) as usize {self.grow();}@@ -94,6 +86,7 @@ impl<K: Key, V> IntMap<K, V> {bin[idx] = (key, value);} else {bin.push((key, value));+ bin.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));self.len += 1;}}@@ -113,14 +106,18 @@ impl<K: Key, V> IntMap<K, V> {self.bins[bin_idx].push((key, value));}}++ for bin in &mut self.bins {+ bin.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));+ }}pub fn get(&self, key: &K) -> Option<&V> {let bin = self.bin_idx(key);- self.bins[bin]- .iter()- .find(|(stored_key, _)| stored_key == key)- .map(|(_, val)| val)+ match self.bins[bin].binary_search_by(|(stored_key, _)| stored_key.cmp(key)) {+ Ok(idx) => Some(&self.bins[bin][idx].1),+ Err(_) => None,+ }}pub fn get_mut(&mut self, key: &K) -> Option<&mut V> {@@ -193,10 +190,6 @@ impl<K: Key> IntSet<K> {pub fn contains(&self, item: &K) -> bool {self.map.contains_key(item)}-- pub fn into_iter(self) -> impl Iterator<Item = K> {- self.map.into_iter().map(|(k, _)| k)- }}impl<K: Key> std::iter::FromIterator<K> for IntSet<K> {diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rsindex f50effe7..6e577ef7 100644--- a/core/src/ranking/bitvec_similarity.rs+++ b/core/src/ranking/bitvec_similarity.rs@@ -17,10 +17,10 @@use itertools::Itertools;use serde::{Deserialize, Serialize};-use crate::hyperloglog::HyperLogLog;+use crate::hyperloglog::{self, HyperLogLog};-const THRESHOLD_SIM_ESTIMATE: f64 = 0.1;-const HYPERLOGLOG_REGISTERS: usize = 128;+const HYPERLOGLOG_REGISTERS: usize = 2048;+const MIN_POSTING_SIZE_HYPERLOGLOG: usize = 1024000;#[derive(Serialize, Deserialize, Debug, Clone)]struct VeryJankyBloomFilter {@@ -55,11 +55,6 @@ impl VeryJankyBloomFilter {}}-#[derive(Clone)]-struct ScratchSpace {- hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS>,-}-#[derive(Serialize, Deserialize, Debug, Clone)]struct Posting {ranks: Vec<u64>,@@ -102,44 +97,56 @@ impl Posting {j += 1;}std::cmp::Ordering::Less => {- match self.skip_pointers.get(i / self.skip_size + 1).copied() {- Some(skip_a) => {- if skip_a < b {- i += self.skip_size - i % self.skip_size;- } else {- i += 1;- }- }- None => {- i += 1;- }- }+ i = self.next_skip_index(i, b);}std::cmp::Ordering::Greater => {- match other.skip_pointers.get(j / other.skip_size + 1).copied() {- Some(skip_b) => {- if skip_b < a {- j += other.skip_size - j % other.skip_size;- } else {- j += 1;- }- }- None => {- j += 1;- }- }+ j = other.next_skip_index(j, a);}}}count}++ fn next_skip_index(&self, current_index: usize, target: u64) -> usize {+ let mut index = current_index;++ while (index / self.skip_size) + 1 < self.skip_pointers.len() {+ let skip_index = (index / self.skip_size) + 1;+ let skip_value = self.skip_pointers[skip_index];++ if skip_value >= target {+ break;+ }++ index = skip_index * self.skip_size;++ if index >= self.ranks.len() {+ return self.ranks.len();+ }+ }++ if index == current_index {+ index += 1;+ }++ std::cmp::min(index, self.ranks.len())+ }++ fn len(&self) -> usize {+ self.ranks.len()+ }+}++#[derive(Clone)]+struct ScratchSpace {+ hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS, hyperloglog::StableHasher>,}#[derive(Serialize, Deserialize, Debug, Clone)]pub struct BitVec {bloom: VeryJankyBloomFilter,- hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS>,+ hyperloglog: Option<HyperLogLog<HYPERLOGLOG_REGISTERS, hyperloglog::StableHasher>>,posting: Posting,sqrt_len: f64,}@@ -152,11 +159,18 @@ impl BitVec {let len = ranks.len();let mut bloom = VeryJankyBloomFilter::new(2);- let mut hyperloglog = HyperLogLog::default();+ let mut hyperloglog = if ranks.len() > MIN_POSTING_SIZE_HYPERLOGLOG {+ Some(HyperLogLog::default())+ } else {+ None+ };for rank in &ranks {bloom.insert(*rank);- hyperloglog.add(*rank);++ if let Some(hyperloglog) = &mut hyperloglog {+ hyperloglog.add(*rank);+ }}let posting = Posting::new(ranks);@@ -178,22 +192,21 @@ impl BitVec {return 0.0;}- self.hyperloglog- .merge_into(&other.hyperloglog, &mut scratchspace.hyperloglog);-- let union_est = scratchspace.hyperloglog.size();-- let intersect_est = (self.posting.ranks.len() + other.posting.ranks.len())- .checked_sub(union_est)- .unwrap_or_default();+ let intersect = match (self.hyperloglog.as_ref(), other.hyperloglog.as_ref()) {+ (Some(a), Some(b)) => {+ a.merge_into(b, &mut scratchspace.hyperloglog);- let sim_est = (intersect_est as f64) / (self.sqrt_len * other.sqrt_len);+ let union_est = scratchspace.hyperloglog.size() as u64;- if sim_est < THRESHOLD_SIM_ESTIMATE {- return 0.0;- }+ (self.posting.len() as u64 + other.posting.len() as u64)+ .checked_sub(union_est)+ .unwrap_or_default()+ .min(self.posting.len() as u64)+ .min(other.posting.len() as u64) as f64+ }+ _ => self.posting.intersection_size(&other.posting) as f64,+ };- let intersect = self.posting.intersection_size(&other.posting) as f64;intersect / (self.sqrt_len * other.sqrt_len)}}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 13571032..fb4208d7 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -22,11 +22,12 @@ use std::{};use dashmap::DashMap;+use fnv::FnvHashMap as HashMap;+use fnv::FnvHashSet as HashSet;use rayon::prelude::ParallelIterator;use serde::{Deserialize, Serialize};use crate::{- intmap::{IntMap, IntSet},webgraph::{NodeID, Webgraph},Result,};@@ -38,8 +39,8 @@ pub struct Scorer {similarity: bitvec_similarity::BitVecSimilarity,liked: Vec<NodeScorer>,disliked: Vec<NodeScorer>,- vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,- cache: IntMap<NodeID, f64>,+ vectors: Arc<HashMap<NodeID, bitvec_similarity::BitVec>>,+ cache: HashMap<NodeID, f64>,normalized: bool,}@@ -126,14 +127,14 @@ impl Scorer {#[derive(Serialize, Deserialize, Default)]pub struct InboundSimilarity {- vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,+ vectors: Arc<HashMap<NodeID, bitvec_similarity::BitVec>>,}impl InboundSimilarity {pub fn build(graph: &Webgraph) -> Self {- let mut vectors = IntMap::new();+ let mut vectors = HashMap::default();- let adjacency: DashMap<NodeID, IntSet<NodeID>> = DashMap::new();+ let adjacency: DashMap<NodeID, HashSet<NodeID>> = DashMap::new();graph.par_edges().for_each(|edge| {adjacency.entry(edge.to).or_default().insert(edge.from);@@ -174,7 +175,7 @@ impl InboundSimilarity {liked,disliked,vectors: self.vectors.clone(),- cache: IntMap::new(),+ cache: HashMap::default(),normalized,}}diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex a94576d4..5dc83060 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -88,6 +88,7 @@ pub struct ApiSearcher {#[cfg(not(feature = "libtorch"))]pub struct ApiSearcher {distributed_searcher: DistributedSearcher,+ live_searcher: LiveSearcher,lambda_model: Option<Arc<LambdaMART>>,bangs: Bangs,collector_config: CollectorConfig,@@ -126,7 +127,8 @@ impl ApiSearcher {config: ApiConfig,) -> Self {Self {- distributed_searcher: DistributedSearcher::new(cluster),+ distributed_searcher: DistributedSearcher::new(Arc::clone(&cluster)),+ live_searcher: LiveSearcher::new(Arc::clone(&cluster)),lambda_model: lambda_model.map(Arc::new),bangs,collector_config: config.collector,diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 428f4e29..eb11fc7d 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -79,17 +79,13 @@ impl intmap::Key for NodeID {NodeID(self.0.wrapping_mul(rhs.0))}- fn bit_and(self, rhs: Self) -> Self {- NodeID(self.0 & rhs.0)- }-- fn from_usize(val: usize) -> Self {- NodeID(val as u128)- }-fn as_usize(self) -> usize {self.0 as usize}++ fn modulus_usize(self, rhs: usize) -> usize {+ (self.0 % (rhs as u128)) as usize+ }}pub trait EdgeLabeldiff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 5f167346..0c71f3ae 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -1435,6 +1435,44 @@ impl Html {.collect()}+ pub fn likely_has_ads(&self) -> bool {+ for script in self.scripts() {+ if let Some(url) = script+ .attributes+ .get("src")+ .and_then(|url| Url::parse(url).ok())+ {+ if let Some(domain) = url.root_domain() {+ return matches!(+ domain,+ "doubleclick.net" | "googlesyndication.com" | "amazon-adsystem.com"+ );+ }+ }+ }++ false+ }++ pub fn likely_has_paywall(&self) -> bool {+ self.schema_org()+ .into_iter()+ .filter(|item| {+ item.types_contains("NewsArticle")+ || item.types_contains("Article")+ || item.types_contains("BlogPosting")+ || item.types_contains("WebPage")+ || item.types_contains("WebPageElement")+ })+ .any(|item| {+ item.properties+ .get("isAccessibleForFree")+ .and_then(|value| value.clone().one().and_then(|v| v.try_into_string()))+ .map(|value| value.parse().ok().unwrap_or(false))+ .unwrap_or(false)+ })+ }+fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {self.metadata().into_iter()diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelteindex d74da549..43221259 100644--- a/frontend/src/routes/search/Result.svelte+++ b/frontend/src/routes/search/Result.svelte@@ -1,5 +1,6 @@<script lang="ts">import AdjustVertical from '~icons/heroicons/adjustments-vertical';+ import LockClosed from '~icons/heroicons/lock-closed';import type { DisplayedWebpage } from '$lib/api';import { createEventDispatcher } from 'svelte';import { clearSummary, summariesStore } from '$lib/stores';@@ -56,6 +57,9 @@<div class="line-clamp-3"><div class="inline"><span id="snippet-text" class="snippet-text">+ <span>+ <LockClosed class="inline-block w-4 h-4 mr-1 text-neutral" />+ </span>{#if webpage.snippet.date}<span class="text-neutral">{webpage.snippet.date} -