web search done right
Disable webgraph compression. An insane amount of time was spent decompressing (about 80%) when fetching inbound links. This made it impractical to index backlink texts, which of course is very usefull information to have in the index.
Mikkel Denker3b8bdc22b653
diff --git a/Cargo.lock b/Cargo.lockindex 43ee9903..17847337 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -75,17 +75,6 @@ dependencies = ["version_check",]-[[package]]-name = "ahash"-version = "0.8.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"-dependencies = [- "cfg-if",- "once_cell",- "version_check",-]-[[package]]name = "aho-corasick"version = "0.7.20"@@ -1819,16 +1808,7 @@ version = "0.12.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"dependencies = [- "ahash 0.7.6",-]--[[package]]-name = "hashbrown"-version = "0.13.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"-dependencies = [- "ahash 0.8.3",+ "ahash",][[package]]@@ -2439,15 +2419,6 @@ dependencies = ["hashbrown 0.12.3",]-[[package]]-name = "lru"-version = "0.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "03f1160296536f10c833a82dca22267d5486734230d47bf00bf435885814ba1e"-dependencies = [- "hashbrown 0.13.2",-]-[[package]]name = "lz-str"version = "0.2.1"@@ -4360,7 +4331,6 @@ dependencies = ["lalrpop-util","libc","logos",- "lru 0.10.0","lz-str","maplit","md5",@@ -4530,7 +4500,7 @@ dependencies = ["itertools 0.10.5","levenshtein_automata","log",- "lru 0.7.8",+ "lru","lz4_flex","measure_time","memmap2",diff --git a/Cargo.toml b/Cargo.tomlindex 9bf3b0a3..dd136093 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -88,7 +88,6 @@ robotstxt-with-cache = "0.4.0"rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }async-channel = "1.8.0"-lru = "0.10.0"url = { version = "2.4.0", features = ["serde"] }anyhow = { version = "1.0.72", features = ["backtrace"] }bytemuck = { version = "1.13.1", features = ["derive"] }diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 6ade96bb..ca2c2721 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -88,7 +88,6 @@ reqwest-eventsource = { workspace = true }robotstxt-with-cache = { workspace = true }rust-s3 = { workspace = true }async-channel = { workspace = true }-lru = { workspace = true }url = { workspace = true }anyhow = { workspace = true }bytemuck = { workspace = true }diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 8477ea08..1188aa1e 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -17,7 +17,6 @@ use chrono::Utc;use rayon::prelude::*;use std::path::Path;use std::thread;-use url::Url;use itertools::Itertools;use serde::{Deserialize, Serialize};@@ -31,7 +30,7 @@ use crate::mapreduce::{Map, Reduce, Worker};use crate::ranking::centrality_store::IndexerCentralityStore;use crate::ranking::SignalAggregator;use crate::webgraph::{Node, Webgraph, WebgraphBuilder};-use crate::webpage::{Html, Link, Webpage};+use crate::webpage::{Html, Webpage};use crate::{human_website_annotations, Result};pub struct Indexer {}@@ -168,22 +167,14 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}}- let backlinks: Vec<Link> = worker+ let backlink_labels: Vec<String> = worker.webgraph.as_ref().map(|webgraph| {webgraph- .ingoing_edges(Node::from(html.url()))+ .raw_ingoing_edges_with_labels(&Node::from(html.url()).id()).into_iter()- .map(|edge| Link {- source: Url::parse(&("http://".to_string() + edge.from.name.as_str()))- .unwrap(),- destination: Url::parse(- &("http://".to_string() + edge.to.name.as_str()),- )- .unwrap(),- text: edge.label,- })+ .map(|edge| edge.label.loaded().unwrap()).collect()}).unwrap_or_else(Vec::new);@@ -204,7 +195,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {has_page_centrality = true;}- if !backlinks.is_empty() {+ if !backlink_labels.is_empty() {has_backlinks = true;}@@ -227,7 +218,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let mut webpage = Webpage {html,- backlinks,+ backlink_labels,page_centrality,host_centrality,fetch_time_ms,diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 9cd236bd..6863bff6 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -616,7 +616,7 @@ mod tests {use crate::{ranking::{Ranker, SignalAggregator},searcher::SearchQuery,- webpage::{Html, Link},+ webpage::Html,};use super::*;@@ -891,11 +891,7 @@ mod tests {"https://www.b.com",).unwrap(),- backlinks: vec![Link {- source: Url::parse("https://www.a.com/").unwrap(),- destination: Url::parse("https://www.b.com/").unwrap(),- text: "B site is great".to_string(),- }],+ backlink_labels: vec!["B site is great".to_string()],host_centrality: 1.0,page_centrality: 0.0,fetch_time_ms: 500,diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex e3a3ec8c..fc68c310 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -260,7 +260,7 @@ mod tests {),"https://www.a.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -288,7 +288,7 @@ mod tests {),"https://www.b.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.01,page_centrality: 0.0,@@ -386,7 +386,7 @@ mod tests {),"https://www.a.com/this/is/a/pattern",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -413,7 +413,7 @@ mod tests {),"https://www.b.com/this/is/b/pattern",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0001,page_centrality: 0.0,@@ -489,7 +489,7 @@ mod tests {),"https://www.a.com/this/is/a/pattern",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -517,7 +517,7 @@ mod tests {),"https://www.b.com/this/is/b/pattern",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0001,page_centrality: 0.0,@@ -545,7 +545,7 @@ mod tests {),"https://www.c.com/this/is/c/pattern",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0001,page_centrality: 0.0,@@ -648,7 +648,7 @@ mod tests {),"https://www.a.com/this/is/a/pattern",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -676,7 +676,7 @@ mod tests {),"https://www.b.com/this/is/b/pattern",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0001,page_centrality: 0.0,@@ -704,7 +704,7 @@ mod tests {),"https://www.c.com/this/is/c/pattern",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0002,page_centrality: 0.0,@@ -779,7 +779,7 @@ mod tests {),"https://www.a.com/",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -822,7 +822,7 @@ mod tests {),"https://www.b.com/",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0001,page_centrality: 0.0,@@ -956,7 +956,7 @@ mod tests {),"https://chat.stackoverflow.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1030,7 +1030,7 @@ mod tests {),"https://a.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1059,7 +1059,7 @@ mod tests {),"https://b.com/",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1113,7 +1113,7 @@ mod tests {include_str!("../../testcases/schema_org/infinity_war.html"),"https://a.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1171,7 +1171,7 @@ mod tests {),"https://example.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1332,7 +1332,7 @@ mod tests {),"https://example.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1405,7 +1405,7 @@ mod tests {),"https://example.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1465,7 +1465,7 @@ mod tests {),"https://example.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1494,7 +1494,7 @@ mod tests {),"https://example.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -1592,7 +1592,7 @@ mod tests {"#,"https://example.com/",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 9f95bae7..77784573 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -364,7 +364,7 @@ mod tests {"https://e.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,@@ -393,7 +393,7 @@ mod tests {"https://d.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.01,fetch_time_ms: 500,page_centrality: 0.0,diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 96a2fde6..042fb8f3 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -135,12 +135,11 @@ impl Ranker {mod tests {use optics::Optic;- use url::Url;use crate::{index::Index,searcher::{LocalSearcher, SearchQuery},- webpage::{Html, Webpage, Link},+ webpage::{Html, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";@@ -169,7 +168,7 @@ mod tests {),"https://www.a.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,@@ -197,7 +196,7 @@ mod tests {),"https://www.b.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 5.0,fetch_time_ms: 500,page_centrality: 0.0,@@ -244,7 +243,7 @@ mod tests {),"https://www.a.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,@@ -272,7 +271,7 @@ mod tests {),"https://www.b.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 5.0,@@ -318,7 +317,7 @@ mod tests {),"https://www.old.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 499,page_centrality: 0.0,@@ -346,7 +345,7 @@ mod tests {),"https://www.new.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 500,pre_computed_score: 0.0,@@ -388,7 +387,7 @@ mod tests {"#,"https://www.first.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,@@ -425,7 +424,7 @@ mod tests {</html>"#,"https://www.second.com").unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.00003,fetch_time_ms: 500,page_centrality: 0.0,@@ -468,11 +467,7 @@ mod tests {"#,"https://www.first.com",).unwrap(),- backlinks: vec![Link {- source: Url::parse("https://www.second.com").unwrap(),- destination: Url::parse("https://www.first.com").unwrap(),- text: "test this is the best test site".to_string(),- }],+ backlink_labels: vec!["test this is the best test site".to_string()],host_centrality: 0.0,fetch_time_ms: 500,page_centrality: 0.0,@@ -497,7 +492,7 @@ mod tests {"#,"https://www.second.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 0.00003,fetch_time_ms: 500,pre_computed_score: 0.0,@@ -540,7 +535,7 @@ mod tests {"#,"https://www.body.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 20,pre_computed_score: 0.0,@@ -566,7 +561,7 @@ mod tests {"#,"https://www.title.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 20,page_centrality: 0.0,@@ -593,7 +588,7 @@ mod tests {"#,"https://www.centrality.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.02,fetch_time_ms: 500,page_centrality: 0.0,@@ -667,7 +662,7 @@ mod tests {),"https://www.first.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,pre_computed_score: 0.0,fetch_time_ms: 500,@@ -699,7 +694,7 @@ mod tests {),"https://www.third.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 500,pre_computed_score: 0.0,@@ -731,7 +726,7 @@ mod tests {),"https://www.second.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 500,pre_computed_score: 0.0,@@ -785,7 +780,7 @@ mod tests {),"https://www.first.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 0,pre_computed_score: 0.0,@@ -813,7 +808,7 @@ mod tests {),"https://www.second.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 5000,page_centrality: 0.0,@@ -862,7 +857,7 @@ mod tests {),"https://www.first.com/one",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 2,pre_computed_score: 0.0,@@ -890,7 +885,7 @@ mod tests {),"https://www.second.com/one/two",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 1,page_centrality: 0.0,@@ -918,7 +913,7 @@ mod tests {),"https://www.third.com/one/two123",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 0,page_centrality: 0.0,diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex da791350..a8ec071f 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -96,7 +96,7 @@ mod tests {"https://www.first.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 500,pre_computed_score: 0.0,@@ -125,7 +125,7 @@ mod tests {"https://www.second.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 499,page_centrality: 0.0,@@ -154,7 +154,7 @@ mod tests {"https://www.third.com",).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: 1.0,fetch_time_ms: 0,page_centrality: 0.0,diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex a0c64ced..fb5181c2 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -444,7 +444,7 @@ mod tests {&format!("https://www.{i}.com"),).unwrap(),- backlinks: vec![],+ backlink_labels: vec![],host_centrality: (NUM_WEBSITES - i) as f64,fetch_time_ms: 500,page_centrality: 0.0,diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex e65b3df3..2e0688f9 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -19,10 +19,9 @@ use std::collections::{BTreeMap, BinaryHeap, HashSet};use std::fs::File;use std::io::{BufReader, BufWriter, Read, Write};use std::path::Path;-use std::sync::{Arc, Mutex};+use std::sync::Arc;use std::{cmp, fs};-use lru::LruCache;use rayon::prelude::*;use serde::{Deserialize, Serialize};use url::Url;@@ -206,7 +205,7 @@ pub enum Loaded<T> {NotYet,}impl<T> Loaded<T> {- fn loaded(self) -> Option<T> {+ pub fn loaded(self) -> Option<T> {match self {Loaded::Some(t) => Some(t),Loaded::NotYet => None,@@ -214,6 +213,20 @@ impl<T> Loaded<T> {}}+impl<T> std::fmt::Display for Loaded<T>+where+ T: std::fmt::Display,+{+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Loaded::Some(val) => {+ write!(f, "{}", val)+ }+ Loaded::NotYet => Ok(()),+ }+ }+}+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct FullEdge {pub from: Node,@@ -346,7 +359,6 @@ type SegmentID = String;#[derive(Serialize, Deserialize, Default)]struct Meta {comitted_segments: Vec<SegmentID>,- next_node_id: u64,}struct SegmentMergeCandidate {@@ -360,7 +372,6 @@ pub struct Webgraph {segments: Vec<StoredSegment>,executor: Arc<Executor>,id2node: Store<NodeID, Node>,- id2node_cache: Mutex<LruCache<NodeID, Node>>,commit_mode: CommitMode,meta: Meta,}@@ -417,7 +428,6 @@ impl Webgraph {commit_mode,executor: Arc::new(executor),id2node: Store::open(path.as_ref().join("id2node")),- id2node_cache: Mutex::new(LruCache::new(10_000.try_into().unwrap())),meta,}}@@ -495,6 +505,10 @@ impl Webgraph {self.inner_ingoing_edges(node, false)}+ pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge> {+ self.inner_ingoing_edges(node, true)+ }+fn inner_ingoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {self.executor.map(@@ -535,19 +549,7 @@ impl Webgraph {}pub fn id2node(&self, id: &NodeID) -> Option<Node> {- let mut guard = self.id2node_cache.lock().unwrap();-- if let Some(node) = guard.get(id) {- return Some(node.clone());- }-- let node = self.id2node.get(id);-- if let Some(node) = &node {- guard.put(*id, node.clone());- }-- node+ self.id2node.get(id)}pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 5f3bc323..ff81b224 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -47,6 +47,8 @@ whereoptions.set_block_based_table_factory(&block_options);options.set_optimize_filters_for_hits(true);+ options.set_compression_type(rocksdb::DBCompressionType::None);+let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();Self {diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 5fdb1d67..915edc73 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -114,7 +114,7 @@ impl<const N: usize> Preprocessor<N> {pub struct Webpage {pub html: Html,- pub backlinks: Vec<Link>,+ pub backlink_labels: Vec<String>,pub host_centrality: f64,pub page_centrality: f64,pub fetch_time_ms: u64,@@ -130,7 +130,7 @@ impl Webpage {Ok(Self {html,- backlinks: Vec::new(),+ backlink_labels: Vec::new(),host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 0,@@ -180,7 +180,7 @@ impl Webpage {}let backlink_text: String = itertools::intersperse(- self.backlinks.into_iter().map(|link| link.text),+ self.backlink_labels.into_iter().map(|label| label),"\n".to_string(),).collect();@@ -2034,7 +2034,7 @@ mod tests {let webpage = Webpage {html,- backlinks: Vec::new(),+ backlink_labels: Vec::new(),host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,@@ -2068,7 +2068,7 @@ mod tests {.unwrap();let webpage = Webpage {html,- backlinks: Vec::new(),+ backlink_labels: Vec::new(),host_centrality: 0.0,page_centrality: 0.0,fetch_time_ms: 500,