web search done right
a bunch of changes primarily to centrality store to make it store more stuff on disk
Mikkel Denker2191b4734c52
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 1a1dcb13..d6ea0514 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -129,6 +129,7 @@ fn create_inverted_index() -> Result<()> {let worker = indexer::IndexingWorker::new(centrality_path.to_str().unwrap().to_string(),+ None,Some(webgraph_path.to_str().unwrap().to_string()),Some(Path::new(DATA_PATH)diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 6dd0a80b..4f303d1f 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -76,19 +76,22 @@ pub struct Job {}pub struct IndexingWorker {- centrality_store: IndexerCentralityStore,+ host_centrality_store: IndexerCentralityStore,+ page_centrality_store: Option<IndexerCentralityStore>,webgraph: Option<Webgraph>,topics: Option<human_website_annotations::Mapper>,}impl IndexingWorker {pub fn new(- centrality_store_path: String,+ host_centrality_store_path: String,+ page_centrality_store_path: Option<String>,webgraph_path: Option<String>,topics_path: Option<String>,) -> Self {Self {- centrality_store: IndexerCentralityStore::open(centrality_store_path),+ host_centrality_store: IndexerCentralityStore::open(host_centrality_store_path),+ page_centrality_store: page_centrality_store_path.map(IndexerCentralityStore::open),webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).open()),topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),}@@ -123,7 +126,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}){let mut html = Html::parse_without_text(&record.response.body, &record.request.url);- let node_id = worker.centrality_store.node2id.get(+ let node_id = worker.host_centrality_store.node2id.get(&html.url().host_without_specific_subdomains_and_query()@@ -132,7 +135,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {);let host_centrality = node_id- .and_then(|node_id| worker.centrality_store.harmonic.host.get(&node_id))+ .and_then(|node_id| worker.host_centrality_store.harmonic.get(&node_id)).unwrap_or_default();if let Some(host_centrality_threshold) = job.host_centrality_threshold {@@ -170,14 +173,17 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}).unwrap_or_else(Vec::new);- let node_id = worker- .centrality_store- .node2id- .get(&html.url().without_protocol().to_lowercase().into());+ let mut page_centrality = 0.0;- let page_centrality = node_id- .and_then(|node_id| worker.centrality_store.harmonic.full.get(&node_id))- .unwrap_or_default();+ if let Some(store) = worker.page_centrality_store.as_ref() {+ let node_id = store+ .node2id+ .get(&html.url().without_protocol().to_lowercase().into());++ page_centrality = node_id+ .and_then(|node_id| store.harmonic.get(&node_id))+ .unwrap_or_default();+ }let fetch_time_ms = record.metadata.fetch_time_ms as u64;@@ -187,7 +193,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {trace!("text = {:?}", html.clean_text());let node_id = worker- .centrality_store+ .host_centrality_store.node2id.get(&Node::from_url(html.url()).into_host());@@ -342,7 +348,8 @@ impl Indexer {pub fn run_worker(worker_addr: String,- centrality_store_path: String,+ host_centrality_store_path: String,+ page_centrality_store_path: Option<String>,webgraph_path: Option<String>,topics_path: Option<String>,) -> Result<()> {@@ -351,14 +358,19 @@ impl Indexer {.build().unwrap().block_on(async {- IndexingWorker::new(centrality_store_path, webgraph_path, topics_path)- .run::<Job, FrozenIndex>(- worker_addr- .parse::<SocketAddr>()- .expect("Could not parse worker address"),- )- .await- .unwrap();+ IndexingWorker::new(+ host_centrality_store_path,+ page_centrality_store_path,+ webgraph_path,+ topics_path,+ )+ .run::<Job, FrozenIndex>(+ worker_addr+ .parse::<SocketAddr>()+ .expect("Could not parse worker address"),+ )+ .await+ .unwrap();});Ok(())}@@ -369,7 +381,8 @@ impl Indexer {let job_config: JobConfig = config.warc_source.clone().into();let worker = IndexingWorker::new(- config.centrality_store_path.clone(),+ config.host_centrality_store_path.clone(),+ config.page_centrality_store_path.clone(),config.webgraph_path.clone(),config.topics_path.clone(),);diff --git a/core/src/intmap.rs b/core/src/intmap.rsindex 02bbdab6..8fe2566e 100644--- a/core/src/intmap.rs+++ b/core/src/intmap.rs@@ -150,7 +150,7 @@ impl<V> Default for IntMap<V> {}}-#[derive(Default)]+#[derive(Default, Debug)]pub struct IntSet {map: IntMap<()>,}@@ -160,8 +160,8 @@ impl IntSet {Self { map: IntMap::new() }}- pub fn insert(&mut self, key: Key) {- self.map.insert(key, ());+ pub fn insert(&mut self, item: Key) {+ self.map.insert(item, ());}pub fn into_iter(self) -> impl Iterator<Item = Key> {@@ -171,6 +171,22 @@ impl IntSet {pub fn len(&self) -> usize {self.map.len()}++ pub fn contains(&self, item: &Key) -> bool {+ self.map.contains_key(item)+ }+}++impl std::iter::FromIterator<u64> for IntSet {+ fn from_iter<T: IntoIterator<Item = u64>>(iter: T) -> Self {+ let mut set = Self::new();++ for num in iter {+ set.insert(num);+ }++ set+ }}#[cfg(test)]diff --git a/core/src/lib.rs b/core/src/lib.rsindex 78c4eee0..4622ae5d 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -106,7 +106,8 @@ pub struct IndexingLocalConfig {output_path: Option<String>,host_centrality_threshold: Option<f64>,topics_path: Option<String>,- centrality_store_path: String,+ host_centrality_store_path: String,+ page_centrality_store_path: Option<String>,minimum_clean_words: Option<usize>,}diff --git a/core/src/main.rs b/core/src/main.rsindex c79c6575..0b612f3e 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -153,7 +153,8 @@ enum IndexingOptions {},Worker {address: String,- centrality_store_path: String,+ host_centrality_store_path: String,+ page_centrality_store_path: Option<String>,webgraph_path: Option<String>,topics_path: Option<String>,},@@ -190,13 +191,15 @@ fn main() -> Result<()> {}IndexingOptions::Worker {address,- centrality_store_path,+ host_centrality_store_path,+ page_centrality_store_path,webgraph_path,topics_path,} => {entrypoint::Indexer::run_worker(address,- centrality_store_path,+ host_centrality_store_path,+ page_centrality_store_path,webgraph_path,topics_path,)?;diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex af99a2e5..7d718651 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -134,24 +134,7 @@ impl Node2Id {}}-pub struct HarmonicCentralityStore {- pub host: Box<dyn Kv<NodeID, f64>>,- pub full: Box<dyn Kv<NodeID, f64>>,-}--impl HarmonicCentralityStore {- pub fn open<P: AsRef<Path>>(path: P) -> Self {- Self {- host: RocksDbStore::open(path.as_ref().join("host")),- full: RocksDbStore::open(path.as_ref().join("full")),- }- }-- fn flush(&self) {- self.host.flush();- self.full.flush();- }-}+pub type HarmonicCentralityStore = Box<dyn Kv<NodeID, f64> + Send + Sync>;pub struct IndexerCentralityStore {pub harmonic: HarmonicCentralityStore,@@ -161,7 +144,7 @@ pub struct IndexerCentralityStore {impl IndexerCentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {- harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),+ harmonic: RocksDbStore::open(path.as_ref().join("harmonic")),node2id: Node2Id::open(path.as_ref().join("node2id")),}}@@ -210,7 +193,7 @@ pub struct CentralityStore {impl CentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {- harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),+ harmonic: RocksDbStore::open(path.as_ref().join("harmonic")),inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity")).ok().unwrap_or_default(),@@ -219,10 +202,11 @@ impl CentralityStore {}}- fn store_host<P: AsRef<Path>>(+ fn store_harmonic<P: AsRef<Path>>(output_path: P,store: &mut CentralityStore,harmonic_centrality: HarmonicCentrality,+ graph: &Webgraph,) {let csv_file = File::options().write(true)@@ -231,17 +215,23 @@ impl CentralityStore {.open(output_path.as_ref().join("harmonic.csv")).unwrap();- let mut host: Vec<_> = harmonic_centrality.host.into_iter().collect();- host.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));+ let mut harmonic: Vec<_> = harmonic_centrality+ .iter()+ .map(|(node, centrality)| (*node, centrality))+ .take(1_000_000)+ .collect();++ harmonic.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));let mut wtr = csv::Writer::from_writer(csv_file);- for (node, centrality) in host {- let node_id = store.node2id.get(&node).unwrap();- store.harmonic.host.insert(node_id, centrality);+ for (node_id, centrality) in harmonic {+ let node = graph.id2node(&node_id).unwrap();++ store.harmonic.insert(node_id, centrality);wtr.write_record(&[node.name, centrality.to_string()]).unwrap();}wtr.flush().unwrap();- store.harmonic.host.flush();+ store.harmonic.flush();}pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {@@ -254,7 +244,7 @@ impl CentralityStore {store.node2id.batch_put(graph.node_ids());let harmonic_centrality = HarmonicCentrality::calculate(graph);- Self::store_host(&output_path, &mut store, harmonic_centrality);+ Self::store_harmonic(&output_path, &mut store, harmonic_centrality, graph);store.flush();storediff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex bd83cfab..423857e1 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::{- cmp::{Ordering, Reverse},+ cmp::Ordering,collections::BinaryHeap,fs::File,io::{BufReader, BufWriter, Read},@@ -27,7 +27,7 @@ use dashmap::DashMap;use serde::{Deserialize, Serialize};use crate::{- intmap::IntMap,+ intmap::{IntMap, IntSet},webgraph::{NodeID, Webgraph},Result,};@@ -158,30 +158,35 @@ impl InboundSimilarity {) -> Self {let mut vectors = IntMap::new();- let mut top_nodes: BinaryHeap<Reverse<ScoredNode>> =- BinaryHeap::with_capacity(num_top_nodes);+ let mut top_nodes: BinaryHeap<ScoredNode> = BinaryHeap::with_capacity(num_top_nodes);- for (node, centrality) in harmonic.host.iter() {- let scored_node = Reverse(ScoredNode {+ for (node, centrality) in harmonic.iter() {+ let scored_node = ScoredNode {node,score: centrality,- });+ };if top_nodes.len() >= num_top_nodes {if let Some(mut worst) = top_nodes.peek_mut() {- if worst.0.score < scored_node.0.score {+ if worst.score < scored_node.score {*worst = scored_node.clone();}}} else {- top_nodes.push(scored_node.clone());+ top_nodes.push(scored_node);}}- for node_id in top_nodes.into_iter().map(|n| n.0.node) {+ let top_nodes: IntSet = top_nodes.into_iter().map(|n| n.node.0).collect();++ for node_id in graph.nodes() {let mut ranks = Vec::new();for edge in graph.raw_ingoing_edges(&node_id) {+ if !top_nodes.contains(&edge.from.0) {+ continue;+ }+ranks.push(edge.from.0 as usize);}@@ -254,6 +259,7 @@ mod tests {use crate::{gen_temp_path,index::Index,+ kv::rocksdb_store::RocksDbStore,rand_words,ranking::centrality_store::CentralityStore,searcher::{LocalSearcher, SearchQuery},@@ -281,13 +287,11 @@ mod tests {let harmonic = HarmonicCentrality::calculate(&graph);- let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());- for (node, centrality) in harmonic.host {- harmonic_centrality_store- .host- .insert(graph.node2id(&node).unwrap(), centrality);+ let harmonic_centrality_store = RocksDbStore::open(crate::gen_temp_path());+ for (node, centrality) in harmonic.iter() {+ harmonic_centrality_store.insert(*node, centrality);}- harmonic_centrality_store.host.flush();+ harmonic_centrality_store.flush();let inbound =InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);@@ -306,18 +310,17 @@ mod tests {graph.insert(Node::from("b.com"), Node::from("a.com"), String::new());graph.insert(Node::from("c.com"), Node::from("d.com"), String::new());graph.insert(Node::from("b.com"), Node::from("e.com"), String::new());+ graph.insert(Node::from("c.com"), Node::from("b.com"), String::new());graph.commit();let harmonic = HarmonicCentrality::calculate(&graph);- let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());- for (node, centrality) in harmonic.host {- harmonic_centrality_store- .host- .insert(graph.node2id(&node).unwrap(), centrality);+ let harmonic_centrality_store = RocksDbStore::open(crate::gen_temp_path());+ for (node, centrality) in harmonic.iter() {+ harmonic_centrality_store.insert(*node, centrality);}- harmonic_centrality_store.host.flush();+ harmonic_centrality_store.flush();let inbound =InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex 37300a60..a5b9a40d 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::collections::HashMap;+use std::collections::BTreeMap;use bitvec::vec::BitVec;use tracing::info;@@ -23,7 +23,7 @@ use crate::{hyperloglog::HyperLogLog,intmap::{IntMap, IntSet},kahan_sum::KahanSum,- webgraph::{Node, NodeID, Webgraph},+ webgraph::{NodeID, Webgraph},};const HYPERLOGLOG_COUNTERS: usize = 64;@@ -79,11 +79,7 @@ impl JankyBloomFilter {}}-pub struct HarmonicCentrality {- pub host: HashMap<Node, f64>,-}--fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {+fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {let nodes: Vec<_> = graph.nodes().collect();info!("Found {} nodes in the graph", nodes.len());let norm_factor = (nodes.len() - 1) as f64;@@ -205,31 +201,30 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {.into_iter().map(|(node_id, sum)| (node_id, f64::from(sum))).filter(|(_, centrality)| *centrality > 0.0)- .map(|(node_id, centrality)| {- (- graph.id2node(&NodeID::from(node_id)).unwrap(),- centrality / norm_factor,- )- })+ .map(|(node_id, centrality)| (NodeID(node_id), centrality / norm_factor)).collect()}-fn calculate_host(graph: &Webgraph) -> HashMap<Node, f64> {- calculate_centrality(graph)-}+pub struct HarmonicCentrality(BTreeMap<NodeID, f64>);impl HarmonicCentrality {pub fn calculate(graph: &Webgraph) -> Self {- Self {- host: calculate_host(graph),- }+ Self(calculate_centrality(graph))+ }++ pub fn get(&self, node: &NodeID) -> Option<f64> {+ self.0.get(node).copied()+ }++ pub fn iter(&self) -> impl Iterator<Item = (&NodeID, f64)> {+ self.0.iter().map(|(node, centrality)| (node, *centrality))}}#[cfg(test)]mod tests {use super::*;- use crate::webgraph::WebgraphBuilder;+ use crate::webgraph::{Node, WebgraphBuilder};fn test_graph() -> Webgraph {// ┌────┐@@ -260,77 +255,116 @@ mod tests {fn host_harmonic_centrality() {let mut graph = WebgraphBuilder::new_memory().open();- graph.insert(Node::from("A.com/1"), Node::from("A.com/2"), String::new());- graph.insert(Node::from("A.com/1"), Node::from("A.com/3"), String::new());- graph.insert(Node::from("A.com/1"), Node::from("A.com/4"), String::new());- graph.insert(Node::from("A.com/2"), Node::from("A.com/1"), String::new());- graph.insert(Node::from("A.com/2"), Node::from("A.com/3"), String::new());- graph.insert(Node::from("A.com/2"), Node::from("A.com/4"), String::new());- graph.insert(Node::from("A.com/3"), Node::from("A.com/1"), String::new());- graph.insert(Node::from("A.com/3"), Node::from("A.com/2"), String::new());- graph.insert(Node::from("A.com/3"), Node::from("A.com/4"), String::new());- graph.insert(Node::from("A.com/4"), Node::from("A.com/1"), String::new());- graph.insert(Node::from("A.com/4"), Node::from("A.com/2"), String::new());- graph.insert(Node::from("A.com/4"), Node::from("A.com/3"), String::new());- graph.insert(Node::from("C.com"), Node::from("B.com"), String::new());- graph.insert(Node::from("D.com"), Node::from("B.com"), String::new());-- graph.commit();-- let centrality = HarmonicCentrality::calculate(&graph);-- assert!(- centrality.host.get(&Node::from("B.com")).unwrap()- > centrality.host.get(&Node::from("A.com")).unwrap_or(&0.0)+ graph.insert(+ Node::from("A.com/1").into_host(),+ Node::from("A.com/2").into_host(),+ String::new(),);- }-- #[test]- fn harmonic_centrality() {- let graph = test_graph();- let centrality = HarmonicCentrality::calculate(&graph);-- assert!(- centrality.host.get(&Node::from("C")).unwrap()- > centrality.host.get(&Node::from("A")).unwrap()+ graph.insert(+ Node::from("A.com/1").into_host(),+ Node::from("A.com/3").into_host(),+ String::new(),);- assert!(- centrality.host.get(&Node::from("A")).unwrap()- > centrality.host.get(&Node::from("B")).unwrap()+ graph.insert(+ Node::from("A.com/1").into_host(),+ Node::from("A.com/4").into_host(),+ String::new(),);- assert_eq!(centrality.host.get(&Node::from("D")), None);- }-- #[test]- fn www_subdomain_ignored() {- let mut graph = WebgraphBuilder::new_memory().open();-graph.insert(- Node::from("B.com").into_host(),- Node::from("A.com").into_host(),+ Node::from("A.com/2").into_host(),+ Node::from("A.com/1").into_host(),String::new(),);graph.insert(- Node::from("B.com").into_host(),- Node::from("www.A.com").into_host(),+ Node::from("A.com/2").into_host(),+ Node::from("A.com/3").into_host(),String::new(),);graph.insert(- Node::from("C.com").into_host(),- Node::from("A.com").into_host(),+ Node::from("A.com/2").into_host(),+ Node::from("A.com/4").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("A.com/3").into_host(),+ Node::from("A.com/1").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("A.com/3").into_host(),+ Node::from("A.com/2").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("A.com/3").into_host(),+ Node::from("A.com/4").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("A.com/4").into_host(),+ Node::from("A.com/1").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("A.com/4").into_host(),+ Node::from("A.com/2").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("A.com/4").into_host(),+ Node::from("A.com/3").into_host(),String::new(),);graph.insert(Node::from("C.com").into_host(),- Node::from("www.A.com").into_host(),+ Node::from("B.com").into_host(),+ String::new(),+ );+ graph.insert(+ Node::from("D.com").into_host(),+ Node::from("B.com").into_host(),String::new(),);graph.commit();+let centrality = HarmonicCentrality::calculate(&graph);- assert!(centrality.host.get(&Node::from("A.com")).is_some());- assert_eq!(centrality.host.get(&Node::from("www.A.com")), None);+ assert!(+ centrality+ .get(&graph.node2id(&Node::from("B.com")).unwrap())+ .unwrap()+ > centrality+ .get(&graph.node2id(&Node::from("A.com")).unwrap())+ .unwrap_or(0.0)+ );+ }++ #[test]+ fn harmonic_centrality() {+ let graph = test_graph();+ let centrality = HarmonicCentrality::calculate(&graph);++ assert!(+ centrality+ .get(&graph.node2id(&Node::from("C")).unwrap())+ .unwrap()+ > centrality+ .get(&graph.node2id(&Node::from("A")).unwrap())+ .unwrap()+ );+ assert!(+ centrality+ .get(&graph.node2id(&Node::from("A")).unwrap())+ .unwrap()+ > centrality+ .get(&graph.node2id(&Node::from("B")).unwrap())+ .unwrap()+ );+ assert_eq!(+ centrality.get(&graph.node2id(&Node::from("D")).unwrap()),+ None+ );}#[test]@@ -351,7 +385,7 @@ mod tests {let centrality_extra = HarmonicCentrality::calculate(&graph);- assert_eq!(centrality.host, centrality_extra.host);+ assert_eq!(centrality.0, centrality_extra.0);}#[test]@@ -375,6 +409,6 @@ mod tests {let orig_graph = test_graph();let orig_centrality = HarmonicCentrality::calculate(&orig_graph);- assert_eq!(centrality.host, orig_centrality.host);+ assert_eq!(centrality.0, orig_centrality.0);}}