web search done right
Huge refactor of webgraph to split graph into multiple segments. This allows for more parallelism
Mikkel Denkere4a27f27a0a0
diff --git a/Cargo.lock b/Cargo.lockindex d1cdee35..a4f69b8d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -142,7 +142,7 @@ version = "0.2.14"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"dependencies = [- "hermit-abi",+ "hermit-abi 0.1.19","libc","winapi",]@@ -602,9 +602,9 @@ dependencies = [[[package]]name = "crossbeam-channel"-version = "0.5.4"+version = "0.5.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"dependencies = ["cfg-if","crossbeam-utils",@@ -727,6 +727,7 @@ dependencies = ["chrono","clap 3.1.18","criterion",+ "crossbeam-channel","csv","encoding_rs","flate2",@@ -743,6 +744,7 @@ dependencies = ["md5","memmap","min-max-heap",+ "num_cpus","once_cell","optics","parse_wiki_text",@@ -1248,6 +1250,15 @@ dependencies = ["libc",]+[[package]]+name = "hermit-abi"+version = "0.2.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"+dependencies = [+ "libc",+]+[[package]]name = "hex"version = "0.4.3"@@ -1947,11 +1958,11 @@ dependencies = [[[package]]name = "num_cpus"-version = "1.13.1"+version = "1.15.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"dependencies = [- "hermit-abi",+ "hermit-abi 0.2.6","libc",]diff --git a/Cargo.toml b/Cargo.tomlindex 90b6d062..684df09b 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -63,3 +63,5 @@ bitvec = "1.0.1"criterion = "0.3.6"maplit = "1.0.2"rand = "0.8.5"+crossbeam-channel = "0.5.6"+num_cpus = "1.15.0"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex 3ed652fa..6586779f 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -67,6 +67,8 @@ bitvec = { workspace = true }tower-http = { workspace = true }once_cell = { workspace = true }optics = { path = "../optics" }+crossbeam-channel = { workspace = true }+num_cpus = { workspace = true }[dev-dependencies]criterion = { workspace = true }@@ -86,5 +88,5 @@ name = "harmonic-centrality"harness = false[[bench]]-name = "approximate-harmonic"-harness = false\ No newline at end of file+name = "online-harmonic"+harness = falsediff --git a/core/benches/approximate-harmonic.rs b/core/benches/approximate-harmonic.rsdeleted file mode 100644index b8a26353..00000000--- a/core/benches/approximate-harmonic.rs+++ /dev/null@@ -1,20 +0,0 @@-use criterion::{criterion_group, criterion_main, Criterion};-use cuely::webgraph::{- centrality::approximate_harmonic::ApproximatedHarmonicCentrality, WebgraphBuilder,-};--const WEBGRAPH_PATH: &str = "data/webgraph";--pub fn criterion_benchmark(c: &mut Criterion) {- let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).with_host_graph().open();- c.bench_function("Approximated harmonic centrality calculation", |b| {- b.iter(|| {- for _ in 0..10 {- ApproximatedHarmonicCentrality::new(&webgraph);- }- })- });-}--criterion_group!(benches, criterion_benchmark);-criterion_main!(benches);diff --git a/core/benches/harmonic-centrality.rs b/core/benches/harmonic-centrality.rsindex 5bd8d058..7540b0d3 100644--- a/core/benches/harmonic-centrality.rs+++ b/core/benches/harmonic-centrality.rs@@ -4,7 +4,7 @@ use cuely::webgraph::{centrality::harmonic::HarmonicCentrality, WebgraphBuilder}const WEBGRAPH_PATH: &str = "data/webgraph";pub fn criterion_benchmark(c: &mut Criterion) {- let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).with_host_graph().open();+ let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).open();c.bench_function("Harmonic centrality calculation", |b| {b.iter(|| {for _ in 0..10 {diff --git a/core/benches/online-harmonic.rs b/core/benches/online-harmonic.rsnew file mode 100644index 00000000..0dacbc7b--- /dev/null+++ b/core/benches/online-harmonic.rs@@ -0,0 +1,18 @@+use criterion::{criterion_group, criterion_main, Criterion};+use cuely::webgraph::{centrality::online_harmonic::OnlineHarmonicCentrality, WebgraphBuilder};++const WEBGRAPH_PATH: &str = "data/webgraph";++pub fn criterion_benchmark(c: &mut Criterion) {+ let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).open();+ c.bench_function("Online harmonic centrality calculation", |b| {+ b.iter(|| {+ for _ in 0..10 {+ OnlineHarmonicCentrality::new(&webgraph);+ }+ })+ });+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/core/examples/print_approx_harmonic_centrality.rs b/core/examples/print_approx_harmonic_centrality.rsindex 078e1a8f..b11d723e 100644--- a/core/examples/print_approx_harmonic_centrality.rs+++ b/core/examples/print_approx_harmonic_centrality.rs@@ -21,7 +21,7 @@ use std::{use cuely::{ranking::centrality_store::CentralityStore,- webgraph::{centrality::approximate_harmonic::Scorer, Node, WebgraphBuilder},+ webgraph::{centrality::online_harmonic::Scorer, Node, NodeID, WebgraphBuilder},webpage::Url,};@@ -50,13 +50,13 @@ impl Ord for ScoredNode {impl Eq for ScoredNode {}-fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[u64]) -> Vec<ScoredNode> {+fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[NodeID]) -> Vec<ScoredNode> {let mut top_nodes = BinaryHeap::with_capacity(top_n);for node in nodes {top_nodes.push(cmp::Reverse(ScoredNode {score: scorer.score(*node),- id: *node,+ id: node.0,}));while top_nodes.len() > top_n {@@ -72,12 +72,12 @@ fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[u64]) -> Vec<ScoredNode>nodes}-fn print_nodes(nodes: &[ScoredNode], id2node: &HashMap<u64, Node>) {+fn print_nodes(nodes: &[ScoredNode], id2node: &HashMap<NodeID, Node>) {for (i, node) in nodes.iter().rev().enumerate() {println!("{i} \t {:.3} \t {} ",node.score,- id2node.get(&node.id).unwrap().name+ id2node.get(&(node.id.into())).unwrap().name);}}@@ -86,15 +86,15 @@ fn print_top_nodes(liked_sites: &[&str],top_n: usize,store: &CentralityStore,- id2node: &HashMap<u64, Node>,- nodes: &[u64],+ id2node: &HashMap<NodeID, Node>,+ nodes: &[NodeID],) {let liked_nodes: Vec<_> = liked_sites.iter().map(|host| Node::from_url(&Url::from(host.to_string()))).collect();- let scorer = store.approx_harmonic.scorer(&liked_nodes, &[]);+ let scorer = store.online_harmonic.scorer(&liked_nodes, &[]);let top_nodes = get_top_nodes(scorer, top_n, nodes);println!("top {} sites for these liked sites {:?}",@@ -108,20 +108,17 @@ fn print_top_nodes(pub fn main() {const TOP_N: usize = 50;- let graph = WebgraphBuilder::new("data/webgraph")- .with_host_graph()- .read_only(true)- .open();- let nodes: Vec<_> = graph.host.as_ref().unwrap().nodes().collect();+ let graph = WebgraphBuilder::new("data/webgraph").read_only(true).open();+ let nodes: Vec<_> = graph.nodes().collect();let id2node: HashMap<_, _> = nodes.iter()- .map(|id| (*id, graph.host.as_ref().unwrap().id2node(id).unwrap()))+ .map(|id| (*id, graph.id2node(id).unwrap())).collect();let store = CentralityStore::open("data/centrality");let mut proxy_nodes = Vec::new();- for node in &store.approx_harmonic.proxy_nodes {+ for node in &store.online_harmonic.proxy_nodes {proxy_nodes.push(id2node.get(&node.id).unwrap().name.clone());}diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rsindex 0a1a4f6a..0b11f933 100644--- a/core/examples/print_inbound_links.rs+++ b/core/examples/print_inbound_links.rs@@ -17,10 +17,7 @@use cuely::webgraph::{Node, WebgraphBuilder};pub fn main() {- let graph = WebgraphBuilder::new("data/webgraph")- .with_host_graph()- .read_only(true)- .open();+ let graph = WebgraphBuilder::new("data/webgraph").read_only(true).open();for host in ["plapcatesq.ga",@@ -29,7 +26,7 @@ pub fn main() {"GroomersNew.ga",] {println!("{}:", host);- for edge in graph.ingoing_edges_host(Node::from(host)) {+ for edge in graph.ingoing_edges(Node::from(host)) {println!("{} -> {} ({})", edge.from.name, edge.to.name, edge.label);}println!();diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex c6aafa36..d306b1e3 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -22,10 +22,7 @@ pub struct Centrality {}impl Centrality {pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {- let graph = WebgraphBuilder::new(webgraph_path)- .read_only(true)- .with_host_graph()- .open();+ let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();CentralityStore::build(&graph, output_path);}diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex ad7468d6..2fab4029 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -91,7 +91,6 @@ fn create_webgraph() -> Result<()> {}),warc_paths: vec![warc_path.to_str().unwrap().to_string()],graph_base_path: out_path_tmp.to_str().unwrap().to_string(),- create_full_graph: true,};let graph = webgraph::process_job(&job);@@ -196,9 +195,9 @@ fn create_topic_centrality() {.to_str().unwrap().to_string();- let approximate_harmonic_path = Path::new(DATA_PATH)+ let online_harmonic_path = Path::new(DATA_PATH).join("centrality")- .join("approx_harmonic")+ .join("online_harmonic").to_str().unwrap().to_string();@@ -212,7 +211,7 @@ fn create_topic_centrality() {index_path,topics_path,webgraph_path,- approximate_harmonic_path,+ online_harmonic_path,output_path,);}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 848c5d6a..d8fd2b7e 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -72,13 +72,7 @@ impl IndexingWorker {) -> Self {Self {centrality_store: CentralityStore::open(centrality_store_path),- webgraph: webgraph_path.map(|path| {- WebgraphBuilder::new(path)- .with_full_graph()- .with_host_graph()- .read_only(true)- .open()- }),+ webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).read_only(true).open()),crawl_stabilty: crawl_stability_path.map(CrawlStability::open),topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),}@@ -176,10 +170,10 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {let node_id = worker.centrality_store- .approx_harmonic+ .online_harmonic.node2id.get(&Node::from_url(html.url()).into_host())- .copied();+ .cloned();let mut host_topic = None;let mut dmoz_description = None;diff --git a/core/src/entrypoint/topic_centrality.rs b/core/src/entrypoint/topic_centrality.rsindex b925dcb4..9c2a7cbc 100644--- a/core/src/entrypoint/topic_centrality.rs+++ b/core/src/entrypoint/topic_centrality.rs@@ -18,9 +18,7 @@ use crate::{human_website_annotations,index::Index,webgraph::{- centrality::{- approximate_harmonic::ApproximatedHarmonicCentrality, topic::TopicCentrality,- },+ centrality::{online_harmonic::OnlineHarmonicCentrality, topic::TopicCentrality},WebgraphBuilder,},};@@ -29,15 +27,15 @@ pub fn run(index_path: String,topics_path: String,webgraph_path: String,- approximate_harmonic_path: String,+ online_harmonic_path: String,output_path: String,) {let index = Index::open(index_path).unwrap();let topics = human_website_annotations::Mapper::open(topics_path).unwrap();- let webgraph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();- let approx = ApproximatedHarmonicCentrality::open(approximate_harmonic_path).unwrap();+ let webgraph = WebgraphBuilder::new(webgraph_path).open();+ let harmonic = OnlineHarmonicCentrality::open(online_harmonic_path).unwrap();- let centrality: TopicCentrality = TopicCentrality::build(&index, topics, webgraph, approx);+ let centrality: TopicCentrality = TopicCentrality::build(&index, topics, webgraph, harmonic);centrality.save(output_path).unwrap();}diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 8fcde02e..f81d4915 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -32,7 +32,6 @@ use tracing::{info, trace};#[derive(Debug, Serialize, Deserialize, Clone)]struct GraphPointer {path: String,- with_full_path: bool,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -46,17 +45,10 @@ pub struct Job {pub config: JobConfig,pub warc_paths: Vec<String>,pub graph_base_path: String,- pub create_full_graph: bool,}-fn open_graph<P: AsRef<Path>>(path: P, with_full_graph: bool) -> webgraph::Webgraph {- let mut builder = WebgraphBuilder::new(path).with_host_graph();-- if with_full_graph {- builder = builder.with_full_graph()- }-- builder.open()+fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {+ WebgraphBuilder::new(path).open()}async fn async_process_job(job: &Job) -> webgraph::Webgraph {@@ -64,10 +56,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {info!("processing {}", name);- let mut graph = open_graph(- Path::new(&job.graph_base_path).join(name),- job.create_full_graph,- );+ let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));let source = match job.config.clone() {JobConfig::Http(config) => WarcSource::HTTP(config),@@ -104,7 +93,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {}}- graph.flush();+ graph.commit();std::fs::remove_file(path).unwrap();}@@ -132,10 +121,7 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {impl Map<StatelessWorker, GraphPointer> for Job {fn map(&self, _worker: &StatelessWorker) -> GraphPointer {let graph = process_job(self);- GraphPointer {- path: graph.path,- with_full_path: self.create_full_graph,- }+ GraphPointer { path: graph.path }}}@@ -162,8 +148,8 @@ impl Reduce<GraphPointer> for GraphPointer {let self_clone = self.clone();{- let graph = open_graph(self.path, self.with_full_path);- let other_graph = open_graph(other.path, self.with_full_path);+ let graph = open_graph(self.path);+ let other_graph = open_graph(other.path);let _ = graph.reduce(other_graph);}@@ -174,7 +160,7 @@ impl Reduce<GraphPointer> for GraphPointer {impl Reduce<GraphPointer> for webgraph::Webgraph {fn reduce(self, other: GraphPointer) -> Self {- let other = open_graph(other.path, other.with_full_path);+ let other = open_graph(other.path);self.reduce(other)}}@@ -215,7 +201,6 @@ impl Webgraph {.graph_base_path.clone().unwrap_or_else(|| "data/webgraph".to_string()),- create_full_graph: config.create_full_graph.unwrap_or(true),}).collect::<Vec<_>>().into_iter(),@@ -276,7 +261,6 @@ impl Webgraph {.graph_base_path.clone().unwrap_or_else(|| "data/webgraph".to_string()),- create_full_graph: config.create_full_graph.unwrap_or(true),}).collect_vec().into_par_iter()@@ -285,7 +269,8 @@ impl Webgraph {if graphs.len() > 1 {let pointer = graphs.pop().unwrap();- let mut graph = open_graph(pointer.path, pointer.with_full_path);+ let mut graph = open_graph(pointer.path);+for other in graphs {graph = graph.reduce(other);}diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 5ddbe4c0..9016d33a 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -224,7 +224,7 @@ impl InvertedIndex {if id == u64::MAX {Ok(None)} else {- Ok(Some(id))+ Ok(Some(id.into()))}}diff --git a/core/src/kv/mod.rs b/core/src/kv/mod.rsindex 1311325e..cd7a32d4 100644--- a/core/src/kv/mod.rs+++ b/core/src/kv/mod.rs@@ -32,6 +32,14 @@ wherefn flush(&self);fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a>;+ fn delete_raw(&mut self, key: &[u8]);++ fn delete(&mut self, key: &K) {+ let key_bytes = bincode::serialize(key).expect("failed to serialize key");++ self.delete_raw(&key_bytes);+ }+fn get(&self, key: &K) -> Option<V> {let key_bytes = bincode::serialize(key).expect("failed to serialize key");@@ -75,6 +83,10 @@ whereunimplemented!()}+ fn delete_raw(&mut self, key: &[u8]) {+ unimplemented!()+ }+fn flush(&self) {}fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {@@ -88,4 +100,8 @@ wherefn insert(&self, key: K, val: V) {self.0.write().unwrap().insert(key, val);}++ fn delete(&mut self, key: &K) {+ self.0.write().unwrap().remove(key);+ }}diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rsindex 41c7cccf..d2efae6f 100644--- a/core/src/kv/rocksdb_store.rs+++ b/core/src/kv/rocksdb_store.rs@@ -36,7 +36,13 @@ impl RocksDbStore {fs::create_dir_all(path.as_ref()).expect("faild to create dir");}- Box::new(DB::open_default(path).expect("unable to open rocks db"))+ let mut options = Options::default();++ options.create_if_missing(true);+ options.set_max_open_files(1);+ options.set_max_file_opening_threads(1);++ Box::new(DB::open(&options, path).expect("unable to open rocks db"))}pub fn open_read_only<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>@@ -50,7 +56,10 @@ impl RocksDbStore {}let mut options = Options::default();+options.create_if_missing(true);+ options.set_max_open_files(1);+ options.set_max_file_opening_threads(1);Box::new(DB::open_for_read_only(&options, path, false).expect("unable to open rocks db"))}@@ -87,6 +96,10 @@ wherevalue: PhantomData::default(),})}++ fn delete_raw(&mut self, key: &[u8]) {+ rocksdb::DB::delete(self, key).unwrap();+ }}pub struct IntoIter<'a, K, V>diff --git a/core/src/lib.rs b/core/src/lib.rsindex 51c7d4a0..389190c2 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -99,7 +99,6 @@ pub struct WebgraphMasterConfig {workers: Vec<String>,graph_base_path: Option<String>,batch_size: Option<usize>,- create_full_graph: Option<bool>,}#[derive(Debug, Deserialize, Clone)]@@ -108,7 +107,6 @@ pub struct WebgraphLocalConfig {warc_source: WarcSource,graph_base_path: Option<String>,batch_size: Option<usize>,- create_full_graph: Option<bool>,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -238,6 +236,12 @@ pub enum Error {#[error("The stackoverflow schema was not structured as expected")]InvalidStackoverflowSchema,++ #[error("Rayong thread pool builder")]+ RayonThreadPool(#[from] rayon::ThreadPoolBuildError),++ #[error("Internal error")]+ InternalError(String),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/core/src/main.rs b/core/src/main.rsindex 445fbb41..f09e84c9 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -73,7 +73,7 @@ enum Commands {index_path: String,topics_path: String,webgraph_path: String,- approximate_harmonic_path: String,+ online_harmonic_path: String,output_path: String,},#[cfg(feature = "dev")]@@ -227,13 +227,13 @@ fn main() -> Result<()> {index_path,topics_path,webgraph_path,- approximate_harmonic_path,+ online_harmonic_path,output_path,} => entrypoint::topic_centrality::run(index_path,topics_path,webgraph_path,- approximate_harmonic_path,+ online_harmonic_path,output_path,),}diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 67b512fb..eee53d67 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -594,10 +594,7 @@ mod tests {fn liked_sites() {let mut index = Index::temporary().expect("Unable to open index");- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph = WebgraphBuilder::new_memory().open();graph.insert(Node::from("https://www.a.com").into_host(),@@ -611,7 +608,7 @@ mod tests {String::new(),);- graph.flush();+ graph.commit();let centrality_store = CentralityStore::build(&graph, gen_temp_path());@@ -645,7 +642,7 @@ mod tests {dmoz_description: None,node_id: Some(*centrality_store- .approx_harmonic+ .online_harmonic.node2id.get(&Node::from("www.a.com").into_host()).unwrap(),@@ -681,7 +678,7 @@ mod tests {host_topic: None,node_id: Some(*centrality_store- .approx_harmonic+ .online_harmonic.node2id.get(&Node::from("www.b.com").into_host()).unwrap(),@@ -717,7 +714,7 @@ mod tests {dmoz_description: None,node_id: Some(*centrality_store- .approx_harmonic+ .online_harmonic.node2id.get(&Node::from("www.c.com").into_host()).unwrap(),diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 81ccbaff..e91ff2fd 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -19,9 +19,7 @@ use std::{fs::File, path::Path};use crate::{kv::{rocksdb_store::RocksDbStore, Kv},webgraph::{- centrality::{- approximate_harmonic::ApproximatedHarmonicCentrality, harmonic::HarmonicCentrality,- },+ centrality::{harmonic::HarmonicCentrality, online_harmonic::OnlineHarmonicCentrality},Webgraph,},};@@ -47,7 +45,7 @@ impl HarmonicCentralityStore {pub struct CentralityStore {pub harmonic: HarmonicCentralityStore,- pub approx_harmonic: ApproximatedHarmonicCentrality,+ pub online_harmonic: OnlineHarmonicCentrality,pub base_path: String,}@@ -55,39 +53,13 @@ impl CentralityStore {pub fn open<P: AsRef<Path>>(path: P) -> Self {Self {harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),- approx_harmonic: ApproximatedHarmonicCentrality::open(- path.as_ref().join("approx_harmonic"),- )- .ok()- .unwrap_or_default(),+ online_harmonic: OnlineHarmonicCentrality::open(path.as_ref().join("online_harmonic"))+ .ok()+ .unwrap_or_default(),base_path: path.as_ref().to_str().unwrap().to_string(),}}- fn store_full<P: AsRef<Path>>(- output_path: P,- store: &mut CentralityStore,- harmonic_centrality: &HarmonicCentrality,- ) {- let csv_file = File::options()- .write(true)- .create(true)- .truncate(true)- .open(output_path.as_ref().join("harmonic_full.csv"))- .unwrap();- let mut wtr = csv::Writer::from_writer(csv_file);-- let mut full: Vec<_> = harmonic_centrality.full.clone().into_iter().collect();- full.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));- for (node, centrality) in full {- store.harmonic.full.insert(node.name.clone(), centrality);- wtr.write_record(&[node.name, centrality.to_string()])- .unwrap();- }-- wtr.flush().unwrap();- }-fn store_host<P: AsRef<Path>>(output_path: P,store: &mut CentralityStore,@@ -116,15 +88,9 @@ impl CentralityStore {let harmonic_centrality = HarmonicCentrality::calculate(graph);- if graph.full.is_some() {- Self::store_full(&output_path, &mut store, &harmonic_centrality)- }-- if graph.host.is_some() {- Self::store_host(&output_path, &mut store, &harmonic_centrality)- }+ Self::store_host(&output_path, &mut store, &harmonic_centrality);- store.approx_harmonic = ApproximatedHarmonicCentrality::new(graph);+ store.online_harmonic = OnlineHarmonicCentrality::new(graph);store.flush();@@ -134,8 +100,8 @@ impl CentralityStore {pub fn flush(&self) {self.harmonic.flush();- self.approx_harmonic- .save(Path::new(&self.base_path).join("approx_harmonic"))+ self.online_harmonic+ .save(Path::new(&self.base_path).join("online_harmonic")).unwrap();}}diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 2de09e87..2ccea2c0 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -34,7 +34,7 @@ use crate::{searcher::NUM_RESULTS_PER_PAGE,webgraph::{centrality::{- approximate_harmonic::{self, ApproximatedHarmonicCentrality, Scorer},+ online_harmonic::{self, OnlineHarmonicCentrality, Scorer},topic,},Node,@@ -141,14 +141,14 @@ impl Ranker {self.aggregator.set_topic_scorer(topic_scorer);}- pub fn set_query_centrality(&mut self, approx: approximate_harmonic::Scorer) {- self.aggregator.set_query_centrality(approx);+ pub fn set_query_centrality(&mut self, harmonic: online_harmonic::Scorer) {+ self.aggregator.set_query_centrality(harmonic);}}pub fn online_centrality_scorer(site_rankings: &SiteRankings,- approx_harmonic: &ApproximatedHarmonicCentrality,+ harmonic: &OnlineHarmonicCentrality,) -> Scorer {let mut liked_nodes = Vec::new();let mut disliked_nodes = Vec::new();@@ -161,7 +161,7 @@ pub fn online_centrality_scorer(disliked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());}- approx_harmonic.scorer(&liked_nodes, &disliked_nodes)+ harmonic.scorer(&liked_nodes, &disliked_nodes)}#[cfg(test)]@@ -173,9 +173,7 @@ mod tests {index::Index,searcher::{LocalSearcher, SearchQuery},webgraph::{- centrality::{- approximate_harmonic::ApproximatedHarmonicCentrality, topic::TopicCentrality,- },+ centrality::{online_harmonic::OnlineHarmonicCentrality, topic::TopicCentrality},Node, WebgraphBuilder,},webpage::{Html, Link, Webpage},@@ -1152,7 +1150,7 @@ mod tests {page_centrality: 0.0,primary_image: None,host_topic: Some(topic_a.clone()),- node_id: Some(1),+ node_id: Some(1.into()),dmoz_description: None,}).expect("failed to insert webpage");@@ -1187,7 +1185,7 @@ mod tests {crawl_stability: 0.0,primary_image: None,host_topic: Some(topic_b.clone()),- node_id: Some(2),+ node_id: Some(2.into()),dmoz_description: None,}).expect("failed to insert webpage");@@ -1222,7 +1220,7 @@ mod tests {crawl_stability: 0.0,primary_image: None,host_topic: None,- node_id: Some(0),+ node_id: Some(0.into()),dmoz_description: None,}).expect("failed to insert webpage");@@ -1245,9 +1243,7 @@ mod tests {);let topics = human_website_annotations::Mapper::from(mapper);- let mut webgraph = WebgraphBuilder::new(crate::gen_temp_path())- .with_host_graph()- .open();+ let mut webgraph = WebgraphBuilder::new(crate::gen_temp_path()).open();webgraph.insert(Node::from("www.wrong.com"),@@ -1260,11 +1256,11 @@ mod tests {String::new(),);- webgraph.flush();+ webgraph.commit();- let approx = ApproximatedHarmonicCentrality::new(&webgraph);+ let harmonic = OnlineHarmonicCentrality::new(&webgraph);- let topic_centrality = TopicCentrality::build(&index, topics, webgraph, approx);+ let topic_centrality = TopicCentrality::build(&index, topics, webgraph, harmonic);let mut searcher = LocalSearcher::new(index);searcher.set_topic_centrality(topic_centrality);diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex f677abec..66cd855c 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -69,10 +69,7 @@ mod tests {fn site_rankings() {let mut index = Index::temporary().expect("Unable to open index");- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph = WebgraphBuilder::new_memory().open();graph.insert(Node::from("https://www.first.com").into_host(),@@ -85,7 +82,7 @@ mod tests {String::new(),);- graph.flush();+ graph.commit();index.insert(Webpage {@@ -116,9 +113,6 @@ mod tests {host_topic: None,node_id: Some(graph- .host- .as_ref()- .unwrap().node2id(&Node::from("https://www.first.com").into_host()).unwrap(),),@@ -153,9 +147,6 @@ mod tests {dmoz_description: None,node_id: Some(graph- .host- .as_ref()- .unwrap().node2id(&Node::from("https://www.second.com").into_host()).unwrap(),),@@ -190,9 +181,6 @@ mod tests {host_topic: None,node_id: Some(graph- .host- .as_ref()- .unwrap().node2id(&Node::from("https://www.third.com").into_host()).unwrap(),),diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex d001b5dc..f35e9751 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -83,7 +83,7 @@ impl RankingWebsite {value.value.as_u64().unwrap() as f64 / FLOAT_SCALING as f64}Field::Fast(FastField::HostNodeID) => {- let node = value.value.as_u64().unwrap();+ let node = value.value.as_u64().unwrap().into();res.personal_centrality = aggregator.personal_centrality(node);res.topic_centrality = aggregator.topic_centrality(node).unwrap_or_default();diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 83937be1..3ef45e9c 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -17,9 +17,12 @@use crate::{fastfield_cache,schema::{FastField, TextField},- webgraph::centrality::{- approximate_harmonic::{self, SHIFT},- topic,+ webgraph::{+ centrality::{+ online_harmonic::{self, SHIFT},+ topic,+ },+ NodeID,},webpage::Webpage,};@@ -257,12 +260,12 @@ impl FieldBoost {pub struct SignalAggregator {fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,signal_coefficients: SignalCoefficient,- personal_centrality: Vec<Arc<approximate_harmonic::Scorer>>,+ personal_centrality: Vec<Arc<online_harmonic::Scorer>>,field_boost: FieldBoost,fetch_time_ms_cache: [f64; 1000],update_time_cache: Vec<f64>,topic_scorer: Option<topic::Scorer>,- query_centrality: Option<Arc<approximate_harmonic::Scorer>>,+ query_centrality: Option<Arc<online_harmonic::Scorer>>,}impl std::fmt::Debug for SignalAggregator {@@ -314,27 +317,27 @@ impl SignalAggregator {self.topic_scorer = Some(topic_scorer);}- pub fn set_query_centrality(&mut self, query_centrality: approximate_harmonic::Scorer) {+ pub fn set_query_centrality(&mut self, query_centrality: online_harmonic::Scorer) {self.query_centrality = Some(Arc::new(query_centrality));}- pub fn add_personal_harmonic(&mut self, personal_centrality: approximate_harmonic::Scorer) {+ pub fn add_personal_harmonic(&mut self, personal_centrality: online_harmonic::Scorer) {self.personal_centrality.push(Arc::new(personal_centrality))}- pub fn topic_centrality(&self, host_id: u64) -> Option<f64> {+ pub fn topic_centrality(&self, host_id: NodeID) -> Option<f64> {self.topic_scorer.as_ref().map(|scorer| scorer.score(host_id))}- pub fn query_centrality(&self, host_id: u64) -> Option<f64> {+ pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {self.query_centrality.as_ref().map(|scorer| scorer.score(host_id) - SHIFT)}- pub fn personal_centrality(&self, host_id: u64) -> f64 {+ pub fn personal_centrality(&self, host_id: NodeID) -> f64 {self.personal_centrality.iter().map(|scorer| scorer.score(host_id))@@ -349,10 +352,12 @@ impl SignalAggregator {current_timestamp: usize,selected_region: Option<Region>,) -> Score {- let host_id = self- .fastfield_cache- .as_ref()- .and_then(|cache| cache.get_doc_cache(&FastField::HostNodeID).get_u64(&doc));+ let host_id = self.fastfield_cache.as_ref().and_then(|cache| {+ cache+ .get_doc_cache(&FastField::HostNodeID)+ .get_u64(&doc)+ .map(NodeID::from)+ });let topic_score = host_id.and_then(|host_id| self.topic_centrality(host_id));diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 94a88888..2b842c14 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -93,16 +93,14 @@ impl LocalSearcher {}).unwrap_or_default();- if let (Some(optic), Some(approx_harmonic)) = (+ if let (Some(optic), Some(harmonic)) = (optic,self.centrality_store.as_ref()- .map(|store| &store.approx_harmonic),+ .map(|store| &store.online_harmonic),) {- query_aggregator.add_personal_harmonic(online_centrality_scorer(- &optic.site_rankings,- approx_harmonic,- ));+ query_aggregator+ .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));}let mut parsed_query = Query::parse(@@ -156,15 +154,13 @@ impl LocalSearcher {.and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator())).unwrap_or_default();- if let Some(approx_harmonic) = self+ if let Some(harmonic) = self.centrality_store.as_ref()- .map(|store| &store.approx_harmonic)+ .map(|store| &store.online_harmonic){- aggregator.add_personal_harmonic(online_centrality_scorer(- &optic.site_rankings,- approx_harmonic,- ));+ aggregator+ .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));}let mut ranker = Ranker::new(@@ -193,10 +189,10 @@ impl LocalSearcher {let top_host_nodes = self.index.top_nodes(&parsed_query, ranker.collector())?;if !top_host_nodes.is_empty() {- let approx = centrality_store- .approx_harmonic+ let harmonic = centrality_store+ .online_harmonic.scorer_from_ids(&top_host_nodes, &[]);- ranker.set_query_centrality(approx);+ ranker.set_query_centrality(harmonic);}}@@ -456,8 +452,6 @@ mod tests {assert!(!urls.is_empty());- dbg!(&urls);-for (i, url) in urls.into_iter().enumerate() {assert_eq!(url,diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rsindex 77bb88e2..924972cd 100644--- a/core/src/webgraph/centrality/betweenness.rs+++ b/core/src/webgraph/centrality/betweenness.rs@@ -24,15 +24,15 @@ use serde::{Deserialize, Serialize};use crate::{intmap::IntMap,- webgraph::{graph_store::GraphStore, Node, NodeID, Store, Webgraph},+ webgraph::{Node, NodeID, Webgraph},};-fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<Node, f64>, i32) {+fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32) {let mut centrality: HashMap<NodeID, f64> = HashMap::new();let mut n = 0;let mut max_dist = 0;- let nodes: Vec<_> = store.nodes().take(100_000).collect();+ let nodes: Vec<_> = graph.nodes().take(100_000).collect();let pb =if with_progress {@@ -60,38 +60,38 @@ fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<Nlet mut sigma = IntMap::new();- sigma.insert(s, 1);+ sigma.insert(s.0, 1);let mut distances = IntMap::new();- distances.insert(s, 0);+ distances.insert(s.0, 0);let mut q = VecDeque::new();q.push_back(s);while let Some(v) = q.pop_front() {stack.push(v);- for edge in store.outgoing_edges(v) {+ for edge in graph.raw_outgoing_edges(&v) {let w = edge.to;- if !distances.contains(&w) {- let dist_v = distances.get(&v).unwrap();+ if !distances.contains(&w.0) {+ let dist_v = distances.get(&v.0).unwrap();q.push_back(w);- distances.insert(w, dist_v + 1);+ distances.insert(w.0, dist_v + 1);}- if *distances.get(&w).unwrap() == distances.get(&v).unwrap() + 1 {- let sigma_v = *sigma.get(&v).unwrap_or(&0);+ if *distances.get(&w.0).unwrap() == distances.get(&v.0).unwrap() + 1 {+ let sigma_v = *sigma.get(&v.0).unwrap_or(&0);- if !sigma.contains(&w) {- sigma.insert(w, 0);+ if !sigma.contains(&w.0) {+ sigma.insert(w.0, 0);}- *sigma.get_mut(&w).unwrap() += sigma_v;+ *sigma.get_mut(&w.0).unwrap() += sigma_v;- if !predecessors.contains(&w) {- predecessors.insert(w, Vec::new());+ if !predecessors.contains(&w.0) {+ predecessors.insert(w.0, Vec::new());}- predecessors.get_mut(&w).unwrap().push(v);+ predecessors.get_mut(&w.0).unwrap().push(v.0);}}}@@ -100,20 +100,20 @@ fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<Nlet mut delta = IntMap::new();while let Some(w) = stack.pop() {- if let Some(pred) = predecessors.get(&w) {+ if let Some(pred) = predecessors.get(&w.0) {for v in pred {let dv = delta.get(v).copied().unwrap_or(0.0);delta.insert(*v,- dv + (*sigma.get(v).unwrap() as f64 / *sigma.get(&w).unwrap() as f64)- * (1.0 + delta.get(&w).unwrap_or(&0.0)),+ dv + (*sigma.get(v).unwrap() as f64 / *sigma.get(&w.0).unwrap() as f64)+ * (1.0 + delta.get(&w.0).unwrap_or(&0.0)),);}}if w != s {- *centrality.entry(w).or_insert(0.0) += *delta.get(&w).unwrap_or(&0.0);+ *centrality.entry(w).or_insert(0.0) += *delta.get(&w.0).unwrap_or(&0.0);}}}@@ -128,7 +128,7 @@ fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<N(centrality.into_iter()- .map(|(id, centrality)| (store.id2node(&id).unwrap(), centrality / norm))+ .map(|(id, centrality)| (graph.id2node(&id).unwrap(), centrality / norm)).collect(),max_dist,)@@ -143,34 +143,18 @@ pub struct Betweenness {impl Betweenness {#[allow(unused)]pub fn calculate(graph: &Webgraph) -> Self {- match &graph.host {- Some(store) => {- let (host, max_dist) = calculate(store, false);- Self {- centrality: host,- max_dist: max_dist.max(0) as usize,- }- }- None => Self {- centrality: HashMap::new(),- max_dist: 0,- },+ let (host, max_dist) = calculate(graph, false);+ Self {+ centrality: host,+ max_dist: max_dist.max(0) as usize,}}pub fn calculate_with_progress(graph: &Webgraph) -> Self {- match &graph.host {- Some(store) => {- let (host, max_dist) = calculate(store, true);- Self {- centrality: host,- max_dist: max_dist.max(0) as usize,- }- }- None => Self {- centrality: HashMap::new(),- max_dist: 0,- },+ let (host, max_dist) = calculate(graph, true);+ Self {+ centrality: host,+ max_dist: max_dist.max(0) as usize,}}}@@ -195,10 +179,7 @@ mod tests {// │// D- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph = WebgraphBuilder::new_memory().open();for i in 0..n - 1 {graph.insert(@@ -208,7 +189,7 @@ mod tests {);}- graph.flush();+ graph.commit();graph}diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex 44c58ec0..f8b7525b 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -23,7 +23,7 @@ use crate::{hyperloglog::HyperLogLog,intmap::IntMap,kahan_sum::KahanSum,- webgraph::{graph_store::GraphStore, Node, Store, Webgraph},+ webgraph::{Node, NodeID, Webgraph},};const HYPERLOGLOG_COUNTERS: usize = 64;@@ -63,14 +63,10 @@ impl JankyBloomFilter {}pub struct HarmonicCentrality {- pub full: HashMap<Node, f64>,pub host: HashMap<Node, f64>,}-fn calculate_centrality<S>(graph: &GraphStore<S>) -> HashMap<Node, f64>-where- S: Store + Sync,-{+fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {let nodes: Vec<_> = graph.nodes().collect();info!("Found {} nodes in the graph", nodes.len());let norm_factor = (nodes.len() - 1) as f64;@@ -79,9 +75,9 @@ where.iter().map(|node| {let mut counter = HyperLogLog::default();- counter.add(*node);+ counter.add(node.0);- (*node, counter)+ (node.0, counter)}).collect();@@ -89,12 +85,12 @@ wherelet mut t = 0;let mut centralities: IntMap<KahanSum> = nodes.iter()- .map(|node| (*node, KahanSum::default()))+ .map(|node| (node.0, KahanSum::default())).collect();let mut changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);for node in &nodes {- changed_nodes.insert(*node);+ changed_nodes.insert(node.0);}loop {@@ -108,12 +104,12 @@ wherelet mut new_changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);for edge in graph.edges() {- if !changed_nodes.contains(&edge.from) {+ if !changed_nodes.contains(&edge.from.0) {continue;}if let (Some(counter_to), Some(counter_from)) =- (new_counters.get_mut(&edge.to), counters.get(&edge.from))+ (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0)){if counter_to.registers()@@ -122,7 +118,7 @@ where.any(|(to, from)| *from > *to){counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to);+ new_changed_nodes.insert(edge.to.0);counter_changes += 1;}}@@ -152,31 +148,23 @@ where.into_iter().map(|(node_id, sum)| (node_id, f64::from(sum))).filter(|(_, centrality)| *centrality > 0.0)- .map(|(node_id, centrality)| (graph.id2node(&node_id).unwrap(), centrality / norm_factor))+ .map(|(node_id, centrality)| {+ (+ graph.id2node(&NodeID::from(node_id)).unwrap(),+ centrality / norm_factor,+ )+ }).collect()}-fn calculate_full(graph: &Webgraph) -> HashMap<Node, f64> {- graph- .full- .as_ref()- .map(calculate_centrality)- .unwrap_or_default()-}-fn calculate_host(graph: &Webgraph) -> HashMap<Node, f64> {- graph- .host- .as_ref()- .map(calculate_centrality)- .unwrap_or_default()+ calculate_centrality(&graph)}impl HarmonicCentrality {pub fn calculate(graph: &Webgraph) -> Self {Self {host: calculate_host(graph),- full: calculate_full(graph),}}}@@ -198,7 +186,7 @@ mod tests {// │// D- let mut graph = WebgraphBuilder::new_memory().with_host_graph().open();+ let mut graph = WebgraphBuilder::new_memory().open();graph.insert(Node::from("A"), Node::from("B"), String::new());graph.insert(Node::from("B"), Node::from("C"), String::new());@@ -206,17 +194,14 @@ mod tests {graph.insert(Node::from("C"), Node::from("A"), String::new());graph.insert(Node::from("D"), Node::from("C"), String::new());- graph.flush();+ graph.commit();graph}#[test]fn host_harmonic_centrality() {- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph = WebgraphBuilder::new_memory().open();graph.insert(Node::from("A.com/1"), Node::from("A.com/2"), String::new());graph.insert(Node::from("A.com/1"), Node::from("A.com/3"), String::new());@@ -233,13 +218,9 @@ mod tests {graph.insert(Node::from("C.com"), Node::from("B.com"), String::new());graph.insert(Node::from("D.com"), Node::from("B.com"), String::new());- graph.flush();+ graph.commit();let centrality = HarmonicCentrality::calculate(&graph);- assert!(- centrality.full.get(&Node::from("A.com/1")).unwrap()- > centrality.full.get(&Node::from("B.com")).unwrap()- );assert!(centrality.host.get(&Node::from("B.com")).unwrap()@@ -266,17 +247,14 @@ mod tests {#[test]fn www_subdomain_ignored() {- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph = WebgraphBuilder::new_memory().open();graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());graph.insert(Node::from("C.com"), Node::from("A.com"), String::new());graph.insert(Node::from("C.com"), Node::from("www.A.com"), String::new());- graph.flush();+ graph.commit();let centrality = HarmonicCentrality::calculate(&graph);assert!(centrality.host.get(&Node::from("A.com")).is_some());@@ -297,10 +275,10 @@ mod tests {graph.insert(Node::from("A"), Node::from("B"), "6".to_string());graph.insert(Node::from("A"), Node::from("B"), "7".to_string());- graph.flush();+ graph.commit();let centrality_extra = HarmonicCentrality::calculate(&graph);- assert_eq!(centrality.full, centrality_extra.full);+ assert_eq!(centrality.host, centrality_extra.host);}}diff --git a/core/src/webgraph/centrality/mod.rs b/core/src/webgraph/centrality/mod.rsindex 15c13d7a..2343dec6 100644--- a/core/src/webgraph/centrality/mod.rs+++ b/core/src/webgraph/centrality/mod.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-pub mod approximate_harmonic;pub mod betweenness;pub mod harmonic;+pub mod online_harmonic;pub mod topic;diff --git a/core/src/webgraph/centrality/approximate_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rssimilarity index 92%rename from core/src/webgraph/centrality/approximate_harmonic.rsrename to core/src/webgraph/centrality/online_harmonic.rsindex 6893c290..173c7422 100644--- a/core/src/webgraph/centrality/approximate_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -34,7 +34,7 @@ use rayon::prelude::*;use serde::{Deserialize, Serialize};use crate::intmap::IntMap;-use crate::webgraph::Node;+use crate::webgraph::{Node, ShortestPaths};use crate::webgraph::{NodeID, Webgraph};use crate::Result;@@ -52,8 +52,8 @@ pub struct ProxyNode {impl ProxyNode {fn dist(&self, from: &NodeID, to: &NodeID) -> Option<usize> {- if let Some(from_node_to_proxy) = self.dist_from_node.get(from) {- if let Some(from_proxy_to_node) = self.dist_to_node.get(to) {+ if let Some(from_node_to_proxy) = self.dist_from_node.get(&from.0) {+ if let Some(from_proxy_to_node) = self.dist_to_node.get(&to.0) {return Some(from_node_to_proxy + from_proxy_to_node);}}@@ -183,7 +183,7 @@ impl UserNode {fn new(id: NodeID, proxy_nodes: &[Arc<ProxyNode>]) -> Self {let mut heap = BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);for proxy in proxy_nodes {- if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id) {+ if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id.0) {let weighted_node = WeightedProxyNode {node: Arc::clone(proxy),dist: *dist_to_proxy,@@ -229,12 +229,12 @@ impl UserNode {}#[derive(Serialize, Deserialize, Default)]-pub struct ApproximatedHarmonicCentrality {+pub struct OnlineHarmonicCentrality {pub node2id: HashMap<Node, NodeID>,pub proxy_nodes: Vec<Arc<ProxyNode>>,}-impl ApproximatedHarmonicCentrality {+impl OnlineHarmonicCentrality {pub fn new(graph: &Webgraph) -> Self {Self::new_with_num_proxy(graph, NUM_PROXY_NODES)}@@ -246,15 +246,13 @@ impl ApproximatedHarmonicCentrality {// betweenness centrality, but I don't know how we can approximate betweenness// on a graph that cannot be in memorylet mut nodes = Vec::new();- if let Some(store) = graph.host.as_ref() {- for id in store.nodes() {- if let Some(node) = store.id2node(&id) {- if nodes.len() < num_proxy_nodes {- nodes.push(node.clone());- }-- node2id.insert(node, id);+ for id in graph.nodes() {+ if let Some(node) = graph.id2node(&id) {+ if nodes.len() < num_proxy_nodes {+ nodes.push(node.clone());}++ node2id.insert(node, id);}}@@ -263,10 +261,15 @@ impl ApproximatedHarmonicCentrality {.take(num_proxy_nodes).filter(|node| node2id.contains_key(node)).map(|node| {- let dist_to_node = graph.raw_host_distances(node.clone()).into_iter().collect();+ let dist_to_node = graph+ .raw_distances(node.clone())+ .into_iter()+ .map(|(n, v)| (n.0, v))+ .collect();let dist_from_node = graph- .raw_host_reversed_distances(node.clone())+ .raw_reversed_distances(node.clone()).into_iter()+ .map(|(n, v)| (n.0, v)).collect();Arc::new(ProxyNode {@@ -287,13 +290,13 @@ impl ApproximatedHarmonicCentrality {let liked_nodes = liked_nodes.iter().map(|node| node.clone().into_host())- .filter_map(|node| self.node2id.get(&node).copied())+ .filter_map(|node| self.node2id.get(&node).cloned()).collect_vec();let disliked_nodes = disliked_nodes.iter().map(|node| node.clone().into_host())- .filter_map(|node| self.node2id.get(&node).copied())+ .filter_map(|node| self.node2id.get(&node).cloned()).collect_vec();self.scorer_from_ids(&liked_nodes, &disliked_nodes)@@ -349,10 +352,7 @@ mod tests {B ----► D ◄------- C*/- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph = WebgraphBuilder::new_memory().open();graph.insert(Node::from("A"), Node::from("D"), String::new());graph.insert(Node::from("B"), Node::from("D"), String::new());@@ -364,7 +364,7 @@ mod tests {graph.insert(Node::from("F"), Node::from("G"), String::new());graph.insert(Node::from("G"), Node::from("E"), String::new());- graph.flush();+ graph.commit();graph}@@ -372,7 +372,7 @@ mod tests {#[test]fn ordering() {let graph = test_graph();- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);+ let centrality = OnlineHarmonicCentrality::new_with_num_proxy(&graph, 5);let liked_nodes = vec![Node::from("B".to_string()), Node::from("E".to_string())];@@ -423,7 +423,7 @@ mod tests {#[test]fn disliked_nodes_centrality() {let graph = test_graph();- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);+ let centrality = OnlineHarmonicCentrality::new_with_num_proxy(&graph, 5);let disliked_nodes = vec![Node::from("D".to_string()), Node::from("E".to_string())];@@ -437,7 +437,7 @@ mod tests {#[test]fn ordering_with_dislikes() {let graph = test_graph();- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);+ let centrality = OnlineHarmonicCentrality::new_with_num_proxy(&graph, 5);let liked_nodes = vec![Node::from("B".to_string()), Node::from("E".to_string())];let disliked_nodes = vec![Node::from("F".to_string())];diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rsindex 44639ead..5ca803e6 100644--- a/core/src/webgraph/centrality/topic.rs+++ b/core/src/webgraph/centrality/topic.rs@@ -37,11 +37,11 @@ use crate::{index::Index,ranking::{initial::InitialScoreTweaker, SignalAggregator},schema::{FastField, Field, TextField},- webgraph::{centrality::approximate_harmonic::SHIFT, Webgraph},+ webgraph::{centrality::online_harmonic::SHIFT, NodeID, Webgraph},};use crate::{query::Query, Result};-use super::approximate_harmonic::ApproximatedHarmonicCentrality;+use super::online_harmonic::OnlineHarmonicCentrality;const TOP_TERMS: usize = 1_000_000;pub const NUM_TOPICS: usize = 50;@@ -53,8 +53,8 @@ pub struct Scorer {}impl Scorer {- pub fn score(&self, host: u64) -> f64 {- match self.host_centrality.get(host as usize) {+ pub fn score(&self, host: NodeID) -> f64 {+ match self.host_centrality.get(host.0 as usize) {Some(host_score) => host_score.iter().zip_eq(self.term_weights.iter())@@ -76,7 +76,7 @@ impl TopicCentrality {index: &Index,topics: Mapper,webgraph: Webgraph,- approx: ApproximatedHarmonicCentrality,+ harmonic: OnlineHarmonicCentrality,) -> Self {let body_field = index.schema()@@ -148,7 +148,7 @@ impl TopicCentrality {let collector = TopDocs::with_limit(1000, index.inverted_index.fastfield_cache()).tweak_score(score_tweaker);- let mut nodes: Vec<_> = webgraph.host.as_ref().unwrap().nodes().collect();+ let mut nodes: Vec<_> = webgraph.nodes().collect();nodes.sort();let mut node_scores = vec![vec![0.0; NUM_TOPICS]; nodes.len()];@@ -168,7 +168,8 @@ impl TopicCentrality {if id == u64::MAX {None} else {- Some(webgraph.host.as_ref().unwrap().id2node(&id).unwrap())+ let id = id.into();+ Some(webgraph.id2node(&id).unwrap())}}).collect();@@ -177,10 +178,10 @@ impl TopicCentrality {assert!(!top_sites.is_empty());- let scorer = approx.scorer(&top_sites, &[]);+ let scorer = harmonic.scorer(&top_sites, &[]);for node in &nodes {- node_scores[*node as usize][i] = scorer.score(*node) - SHIFT;+ node_scores[node.0 as usize][i] = scorer.score(*node) - SHIFT;}}diff --git a/core/src/webgraph/executor.rs b/core/src/webgraph/executor.rsnew file mode 100644index 00000000..a5041424--- /dev/null+++ b/core/src/webgraph/executor.rs@@ -0,0 +1,153 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{Error, Result};++use crossbeam_channel::unbounded;+use rayon::{ThreadPool, ThreadPoolBuilder};++pub enum Executor {+ #[allow(unused)]+ SingleThread,+ ThreadPool(ThreadPool),+}++impl Executor {+ #[allow(unused)]+ pub fn single_thread() -> Executor {+ Executor::SingleThread+ }++ pub fn multi_thread(prefix: &'static str) -> Result<Executor> {+ Self::with_threads(num_cpus::get(), prefix)+ }++ pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {+ let pool = ThreadPoolBuilder::new()+ .num_threads(num_threads)+ .thread_name(move |num| format!("{}{}", prefix, num))+ .build()?;+ Ok(Executor::ThreadPool(pool))+ }++ pub fn map<J: Send, R: Send, JIterator: Iterator<Item = J>, F: Sized + Sync + Fn(J) -> R>(+ &self,+ f: F,+ jobs: JIterator,+ ) -> Result<Vec<R>> {+ match self {+ Executor::SingleThread => Ok(jobs.map(f).collect::<_>()),+ Executor::ThreadPool(pool) => {+ let jobs: Vec<J> = jobs.collect();+ let num_jobs = jobs.len();+ let rx = {+ let (tx, rx) = unbounded();+ pool.scope(|scope| {+ for (idx, arg) in jobs.into_iter().enumerate() {+ let tx_ref = &tx;+ let f_ref = &f;+ scope.spawn(move |_| {+ let res = f_ref(arg);+ if let Err(err) = tx_ref.send((idx, res)) {+ tracing::error!(+ "Failed to execute job. It probably means all executor \+ threads have panicked. {:?}",+ err+ );+ panic!();+ }+ });+ }+ });+ rx+ // This ends the scope of tx.+ // This is important as it makes it possible for the rx iteration to+ // terminate.+ };+ let mut result_placeholders: Vec<Option<R>> =+ std::iter::repeat_with(|| None).take(num_jobs).collect();++ for (pos, res) in rx {+ result_placeholders[pos] = Some(res);+ }+ let results: Vec<R> = result_placeholders.into_iter().flatten().collect();++ if results.len() != num_jobs {+ return Err(Error::InternalError(+ "At least one of the scheduled jobs failed.".to_string(),+ ));+ }++ Ok(results)+ }+ }+ }+}++#[cfg(test)]+mod tests {++ use super::Executor;++ #[test]+ #[should_panic(expected = "panic should propagate")]+ fn test_panic_propagates_single_thread() {+ let _result: Vec<usize> = Executor::single_thread()+ .map(+ |_| {+ panic!("panic should propagate");+ },+ vec![0].into_iter(),+ )+ .unwrap();+ }++ #[test]+ #[should_panic] //< unfortunately the panic message is not propagated+ fn test_panic_propagates_multi_thread() {+ let _result: Vec<usize> = Executor::with_threads(1, "search-test")+ .unwrap()+ .map(+ |_| {+ panic!("panic should propagate");+ },+ vec![0].into_iter(),+ )+ .unwrap();+ }++ #[test]+ fn test_map_singlethread() {+ let result: Vec<usize> = Executor::single_thread().map(|i| i * 2, 0..1_000).unwrap();+ assert_eq!(result.len(), 1_000);++ for (i, r) in result.iter().enumerate() {+ assert_eq!(*r, i * 2);+ }+ }++ #[test]+ fn test_map_multithread() {+ let result: Vec<usize> = Executor::with_threads(3, "search-test")+ .unwrap()+ .map(|i| i * 2, 0..10)+ .unwrap();+ assert_eq!(result.len(), 10);+ for (i, r) in result.iter().enumerate() {+ assert_eq!(*r, i * 2);+ }+ }+}diff --git a/core/src/webgraph/graph_store.rs b/core/src/webgraph/graph_store.rsdeleted file mode 100644index 08292caa..00000000--- a/core/src/webgraph/graph_store.rs+++ /dev/null@@ -1,420 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{- collections::{BTreeMap, HashSet},- fmt::Debug,- marker::PhantomData,- path::Path,- sync::{Arc, RwLock},-};--use serde::{de::DeserializeOwned, Serialize};--use super::{Edge, Node, NodeID, Store, StoredEdge};-use crate::kv::{rocksdb_store::RocksDbStore, Kv};-pub(crate) struct Adjacency {- pub(crate) tree: CachedTree<NodeID, HashSet<StoredEdge>>,-}--impl Adjacency {- fn insert(&mut self, from: NodeID, to: NodeID, label: String) {- let edge = StoredEdge { other: to, label };-- if let Some(existing) = self.tree.get(&from) {- existing.write().unwrap().insert(edge);- } else {- let mut set = HashSet::new();- set.insert(edge);- self.tree.insert(from, set);- }- }-- fn edges(&self, node: NodeID) -> Vec<StoredEdge> {- self.tree- .get(&node)- .map(|r| r.read().unwrap().clone())- .unwrap_or_default()- .into_iter()- .collect()- }-}--pub(crate) struct CachedTree<K, V>-where- K: Ord + Eq + Serialize + DeserializeOwned + Clone,- V: Serialize + DeserializeOwned + Clone,-{- store: Box<dyn Kv<K, V> + Send + Sync>,- cache: RwLock<BTreeMap<K, Arc<RwLock<V>>>>,- cache_size: usize,-}--impl<K, V> CachedTree<K, V>-where- K: Ord + Eq + Serialize + DeserializeOwned + Clone,- V: Serialize + DeserializeOwned + Clone + Debug,-{- pub(crate) fn new(store: Box<dyn Kv<K, V> + Send + Sync>, cache_size: usize) -> Self {- Self {- store,- cache_size,- cache: RwLock::new(BTreeMap::new()),- }- }-- fn update_cache(&self, key: &K) {- if !self.cache.read().unwrap().contains_key(key) {- let val = self.store.get(key);-- if let Some(val) = val {- self.cache- .write()- .unwrap()- .insert(key.clone(), Arc::new(RwLock::new(val)));- }- }- }-- pub(crate) fn get(&self, key: &K) -> Option<Arc<RwLock<V>>> {- self.update_cache(key);- let guard = self.cache.read().unwrap();- guard.get(key).cloned()- }-- fn insert(&mut self, key: K, value: V) {- let cache = self.cache.get_mut().unwrap();-- cache.insert(key, Arc::new(RwLock::new(value)));-- if cache.len() >= self.cache_size {- while cache.len() > self.cache_size / 2 {- if let Some((key, value)) = cache.pop_first() {- self.store.insert(key, value.write().unwrap().clone());- }- }-- self.store.flush();- }- }-- pub(crate) fn flush(&mut self) {- let cache = self.cache.get_mut().unwrap();-- while let Some((key, value)) = cache.pop_first() {- self.store.insert(key, value.write().unwrap().clone());- }-- self.store.flush();- }-- pub(crate) fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {- self.store.iter()- }-}--pub struct GraphStore<S> {- pub(crate) adjacency: Adjacency,- pub(crate) reversed_adjacency: Adjacency,- pub(crate) node2id: CachedTree<Node, NodeID>,- pub(crate) id2node: CachedTree<NodeID, Node>,- pub(crate) meta: CachedTree<String, u64>,- pub(crate) store: PhantomData<S>,-}--impl<S: Store> GraphStore<S> {- #[cfg(test)]- pub(crate) fn temporary() -> GraphStore<S> {- S::temporary()- }-- pub fn open<P: AsRef<Path>>(path: P) -> Self {- S::open(path)- }-- pub fn open_read_only<P: AsRef<Path>>(path: P) -> Self {- S::open_read_only(path)- }-- fn next_id(&self) -> NodeID {- let key = &"next_id".to_string();- self.meta- .get(key)- .map(|arc| *arc.read().unwrap())- .unwrap_or(0)- }-- fn increment_next_id(&mut self) {- let current_id = self.next_id();- let next_id = current_id + 1;- self.meta.insert("next_id".to_string(), next_id);- }-- fn id_and_increment(&mut self) -> NodeID {- let id = self.next_id();- self.increment_next_id();- id- }-- fn assign_id(&mut self, node: Node, id: NodeID) {- self.node2id.insert(node.clone(), id);- self.id2node.insert(id, node);- }-- fn id_or_assign(&mut self, node: Node) -> NodeID {- if let Some(id) = self.node2id.get(&node) {- return *id.read().unwrap();- }- let id = self.id_and_increment();- self.assign_id(node, id);- id- }-- #[allow(unused)]- pub fn outgoing_edges(&self, node: NodeID) -> Vec<Edge> {- self.adjacency- .edges(node)- .into_iter()- .map(|edge| Edge {- from: node,- to: edge.other,- label: edge.label,- })- .collect()- }-- pub fn ingoing_edges(&self, node: NodeID) -> Vec<Edge> {- self.reversed_adjacency- .edges(node)- .into_iter()- .map(|edge| Edge {- from: edge.other,- to: node,- label: edge.label,- })- .collect()- }-- pub fn nodes(&self) -> impl Iterator<Item = NodeID> {- self.node2id- .iter()- .map(|(_, id)| id)- .collect::<Vec<u64>>()- .into_iter()- }-- pub fn insert(&mut self, from: Node, to: Node, label: String) {- let from_id = self.id_or_assign(from);- let to_id = self.id_or_assign(to);-- self.adjacency.insert(from_id, to_id, label.clone());- self.reversed_adjacency.insert(to_id, from_id, label);- }-- pub fn node2id(&self, node: &Node) -> Option<NodeID> {- self.node2id.get(node).map(|arc| *arc.read().unwrap())- }-- pub fn id2node(&self, id: &NodeID) -> Option<Node> {- self.id2node.get(id).map(|r| r.read().unwrap().clone())- }-- pub fn flush(&mut self) {- self.adjacency.tree.flush();- self.reversed_adjacency.tree.flush();- self.node2id.flush();- self.id2node.flush();- self.meta.flush();- }-- pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {- self.adjacency.tree.iter().flat_map(|(node_id, edges)| {- edges.into_iter().map(move |stored_edge| Edge {- from: node_id,- to: stored_edge.other,- label: stored_edge.label,- })- })- }-- pub fn append(&mut self, mut other: GraphStore<S>) {- other.flush();-- for edge in other.edges() {- let from = other.id2node(&edge.from).expect("node not found");- let to = other.id2node(&edge.to).expect("node not found");-- self.insert(from, to, edge.label);- }-- self.flush();- }-}--impl Store for RocksDbStore {- fn open<P: AsRef<std::path::Path>>(path: P) -> GraphStore<Self> {- let adjacency = RocksDbStore::open(path.as_ref().join("adjacency"));- let reversed_adjacency = RocksDbStore::open(path.as_ref().join("reversed_adjacency"));- let node2id = RocksDbStore::open(path.as_ref().join("node2id"));- let id2node = RocksDbStore::open(path.as_ref().join("id2node"));- let meta = RocksDbStore::open(path.as_ref().join("meta"));-- GraphStore {- adjacency: Adjacency {- tree: CachedTree::new(adjacency, 50_000),- },- reversed_adjacency: Adjacency {- tree: CachedTree::new(reversed_adjacency, 50_000),- },- node2id: CachedTree::new(node2id, 100_000),- id2node: CachedTree::new(id2node, 100_000),- meta: CachedTree::new(meta, 1_000),- store: Default::default(),- }- }-- fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self> {- let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));- let reversed_adjacency =- RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));- let node2id = RocksDbStore::open_read_only(path.as_ref().join("node2id"));- let id2node = RocksDbStore::open_read_only(path.as_ref().join("id2node"));- let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));-- GraphStore {- adjacency: Adjacency {- tree: CachedTree::new(adjacency, 50_000),- },- reversed_adjacency: Adjacency {- tree: CachedTree::new(reversed_adjacency, 50_000),- },- node2id: CachedTree::new(node2id, 100_000),- id2node: CachedTree::new(id2node, 100_000),- meta: CachedTree::new(meta, 1_000),- store: Default::default(),- }- }-}--#[cfg(test)]-mod test {- use super::*;-- #[test]- fn simple_triangle_graph() {- // ┌────┐- // │ │- // ┌───A◄─┐ │- // │ │ │- // ▼ │ │- // B─────►C◄┘-- let mut store: GraphStore<RocksDbStore> = GraphStore::temporary();-- let a = Node {- name: "A".to_string(),- };- let b = Node {- name: "B".to_string(),- };- let c = Node {- name: "C".to_string(),- };-- store.insert(a.clone(), b.clone(), String::new());- store.insert(b.clone(), c.clone(), String::new());- store.insert(c.clone(), a.clone(), String::new());- store.insert(a.clone(), c.clone(), String::new());-- store.flush();-- let nodes: Vec<Node> = store- .nodes()- .map(|id| store.id2node(&id).unwrap())- .collect();- assert_eq!(nodes, vec![a.clone(), b.clone(), c.clone()]);-- let a_id = store.node2id(&a).unwrap();- let b_id = store.node2id(&b).unwrap();- let c_id = store.node2id(&c).unwrap();-- let mut out = store.outgoing_edges(a_id);- out.sort();-- assert_eq!(- out,- vec![- Edge {- from: a_id,- to: b_id,- label: String::new()- },- Edge {- from: a_id,- to: c_id,- label: String::new()- },- ]- );-- let mut out = store.outgoing_edges(b_id);- out.sort();- assert_eq!(- out,- vec![Edge {- from: b_id,- to: c_id,- label: String::new()- },]- );-- let mut out = store.ingoing_edges(c_id);- out.sort();- assert_eq!(- out,- vec![- Edge {- from: a_id,- to: c_id,- label: String::new()- },- Edge {- from: b_id,- to: c_id,- label: String::new()- },- ]- );-- assert_eq!(- store.ingoing_edges(a_id),- vec![Edge {- from: c_id,- to: a_id,- label: String::new()- },]- );-- assert_eq!(- store.ingoing_edges(b_id),- vec![Edge {- from: a_id,- to: b_id,- label: String::new()- },]- );- }-}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex a127eedb..1075cd31 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -13,14 +13,18 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-mod graph_store;+mod segment;use serde::{Deserialize, Serialize};-use std::collections::{BinaryHeap, HashMap};+use std::collections::{BinaryHeap, HashMap, HashSet};+use std::fs::File;+use std::io::{BufReader, BufWriter, Read, Write};use std::path::Path;+use std::sync::Arc;use std::{cmp, fs};+use uuid::Uuid;-use graph_store::GraphStore;+use segment::Segment;use crate::directory::{self, DirEntry};use crate::webpage::Url;@@ -28,12 +32,22 @@ use crate::webpage::Url;use crate::kv::rocksdb_store::RocksDbStore;pub mod centrality;+mod executor;+use self::executor::Executor;+use self::segment::{CachedStore, SegmentNodeID};-pub type NodeID = u64;+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]+pub struct NodeID(pub u64);++impl From<u64> for NodeID {+ fn from(val: u64) -> Self {+ Self(val)+ }+}#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]pub(crate) struct StoredEdge {- other: NodeID,+ other: SegmentNodeID,label: String,}@@ -107,8 +121,6 @@ pub struct FullEdge {pub struct WebgraphBuilder {path: Box<Path>,- full_graph_path: Option<Box<Path>>,- host_graph_path: Option<Box<Path>>,read_only: bool,}@@ -124,22 +136,10 @@ impl WebgraphBuilder {pub fn new<P: AsRef<Path>>(path: P) -> Self {Self {path: path.as_ref().into(),- full_graph_path: None,- host_graph_path: None,read_only: false,}}- pub fn with_full_graph(mut self) -> Self {- self.full_graph_path = Some(self.path.join("full").into());- self- }-- pub fn with_host_graph(mut self) -> Self {- self.host_graph_path = Some(self.path.join("host").into());- self- }-pub fn read_only(mut self, read_only: bool) -> Self {self.read_only = read_only;@@ -148,273 +148,443 @@ impl WebgraphBuilder {pub fn open(self) -> Webgraph {if self.read_only {- Webgraph {- full: self.full_graph_path.map(GraphStore::open_read_only),- host: self.host_graph_path.map(GraphStore::open_read_only),- path: self.path.to_str().unwrap().to_string(),- }+ Webgraph::open_read_only(self.path)} else {- Webgraph {- full: self.full_graph_path.map(GraphStore::open),- host: self.host_graph_path.map(GraphStore::open),- path: self.path.to_str().unwrap().to_string(),- }+ Webgraph::open(self.path)}}}pub trait Storewhere- Self: Sized,+ Self: Sized + Send + Sync,{- fn open<P: AsRef<Path>>(path: P) -> GraphStore<Self>;- fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self>;+ fn open<P: AsRef<Path>>(path: P, id: String) -> Segment<Self>;+ fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Segment<Self>;++ fn temporary() -> Segment<Self> {+ Self::open(crate::gen_temp_path(), "id123".to_string())+ }+}++pub trait ShortestPaths {+ fn distances(&self, source: Node) -> HashMap<Node, usize>;+ fn raw_distances(&self, source: Node) -> HashMap<NodeID, usize>;+ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize>;+ fn reversed_distances(&self, source: Node) -> HashMap<Node, usize>;+}++fn dijkstra<F1, F2>(+ source: Node,+ node_edges: F1,+ edge_node: F2,+ graph: &Webgraph,+) -> HashMap<NodeID, usize>+where+ F1: Fn(NodeID) -> Vec<Edge>,+ F2: Fn(&Edge) -> NodeID,+{+ const MAX_DIST: usize = 5;+ let source_id = graph.node2id(&source);+ if source_id.is_none() {+ return HashMap::new();+ }++ let source_id = source_id.unwrap();+ let mut distances: HashMap<NodeID, usize> = HashMap::default();++ let mut queue = BinaryHeap::new();++ queue.push(cmp::Reverse((0_usize, source_id)));+ distances.insert(source_id, 0);++ while let Some(state) = queue.pop() {+ let (cost, v) = state.0;++ if cost >= MAX_DIST {+ continue;+ }++ let current_dist = distances.get(&v).unwrap_or(&usize::MAX);++ if cost > *current_dist {+ continue;+ }++ for edge in node_edges(v) {+ if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&usize::MAX) {+ let d = cost + 1;++ if d > MAX_DIST {+ continue;+ }++ let next = cmp::Reverse((d, edge_node(&edge)));+ queue.push(next);+ distances.insert(edge_node(&edge), d);+ }+ }+ }++ distances+}++impl ShortestPaths for Webgraph {+ fn distances(&self, source: Node) -> HashMap<Node, usize> {+ self.raw_distances(source)+ .into_iter()+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))+ .collect()+ }++ fn raw_distances(&self, source: Node) -> HashMap<NodeID, usize> {+ dijkstra(+ source,+ |node| self.raw_outgoing_edges(&node),+ |edge| edge.to,+ self,+ )+ }++ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {+ dijkstra(+ source,+ |node| self.raw_ingoing_edges(&node),+ |edge| edge.from,+ self,+ )+ }- fn temporary() -> GraphStore<Self> {- Self::open(crate::gen_temp_path())+ fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {+ self.raw_reversed_distances(source)+ .into_iter()+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))+ .collect()}}+type SegmentID = String;++#[derive(Serialize, Deserialize, Default)]+struct Meta {+ comitted_segments: Vec<SegmentID>,+ next_node_id: u64,+}++struct SegmentMergeCandidate<S: Store> {+ segment: Segment<S>,+ merges: Vec<Segment<S>>,+}+pub struct Webgraph<S: Store = RocksDbStore> {pub path: String,- pub full: Option<GraphStore<S>>,- pub host: Option<GraphStore<S>>,+ live_segment: Option<Segment<S>>,+ segments: Vec<Segment<S>>,+ executor: Arc<Executor>,+ node2id: CachedStore<Node, NodeID>,+ id2node: CachedStore<NodeID, Node>,+ meta: Meta,}impl<S: Store> Webgraph<S> {- pub fn insert(&mut self, mut from: Node, mut to: Node, label: String) {- from.remove_protocol();- to.remove_protocol();+ fn meta<P: AsRef<Path>>(path: P) -> Meta {+ let meta_path = path.as_ref().join("metadata.json");+ let mut reader = BufReader::new(+ File::options()+ .create(true)+ .write(true)+ .read(true)+ .open(meta_path)+ .unwrap(),+ );+ let mut buf = String::new();+ reader.read_to_string(&mut buf).unwrap();+ serde_json::from_str(&buf).unwrap_or_default()+ }++ fn save_metadata(&mut self) {+ let path = Path::new(&self.path).join("metadata.json");+ let mut writer = BufWriter::new(+ File::options()+ .create(true)+ .write(true)+ .read(true)+ .open(path)+ .unwrap(),+ );++ let json = serde_json::to_string_pretty(&self.meta).unwrap();+ writer.write_all(json.as_bytes()).unwrap();+ }++ fn open<P: AsRef<Path>>(path: P) -> Self {+ fs::create_dir_all(&path).unwrap();+ let meta = Self::meta(&path);- if let Some(full_graph) = &mut self.full {- full_graph.insert(from.clone(), to.clone(), label.clone());+ let mut segments = Vec::new();+ for segment in &meta.comitted_segments {+ segments.push(Segment::open(+ path.as_ref().join("segments").join(segment),+ segment.clone(),+ ));}- if let Some(host_graph) = &mut self.host {- host_graph.insert(from.into_host(), to.into_host(), label);+ Self {+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),+ live_segment: None,+ segments,+ executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),+ node2id: CachedStore::new(RocksDbStore::open(path.as_ref().join("node2id")), 100_000),+ id2node: CachedStore::new(RocksDbStore::open(path.as_ref().join("id2node")), 100_000),+ meta,}}- pub fn merge(&mut self, other: Webgraph<S>) {- match (&mut self.full, other.full) {- (Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),- (None, Some(other_graph)) => self.full = Some(other_graph),- (Some(_), None) | (None, None) => {}- }+ fn open_read_only<P: AsRef<Path>>(path: P) -> Self {+ let meta = Self::meta(&path);- match (&mut self.host, other.host) {- (Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),- (None, Some(other_graph)) => self.host = Some(other_graph),- (Some(_), None) | (None, None) => {}- }+ let mut segments = Vec::new();- self.flush();- }-- fn dijkstra<F1, F2>(- source: Node,- node_edges: F1,- edge_node: F2,- store: &GraphStore<S>,- ) -> HashMap<NodeID, usize>- where- F1: Fn(NodeID) -> Vec<Edge>,- F2: Fn(&Edge) -> NodeID,- {- const MAX_DIST: usize = 5;- let source_id = store.node2id(&source);- if source_id.is_none() {- return HashMap::new();+ for segment in &meta.comitted_segments {+ segments.push(Segment::open(+ path.as_ref().join("segments").join(segment),+ segment.clone(),+ ));}- let source_id = source_id.unwrap();- let mut distances: HashMap<NodeID, usize> = HashMap::default();+ Self {+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),+ live_segment: None,+ segments,+ executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),+ node2id: CachedStore::new(+ RocksDbStore::open_read_only(path.as_ref().join("node2id")),+ 100_000,+ ),+ id2node: CachedStore::new(+ RocksDbStore::open_read_only(path.as_ref().join("id2node")),+ 100_000,+ ),+ meta,+ }+ }- let mut queue = BinaryHeap::new();+ fn id_and_increment(&mut self) -> NodeID {+ let id = self.meta.next_node_id.into();+ self.meta.next_node_id += 1;+ id+ }- queue.push(cmp::Reverse((0_usize, source_id)));- distances.insert(source_id, 0);+ fn id_or_assign(&mut self, node: &Node) -> NodeID {+ match self.node2id(node) {+ Some(id) => id,+ None => {+ let id = self.id_and_increment();- while let Some(state) = queue.pop() {- let (cost, v) = state.0;+ self.node2id.insert(node.clone(), id);+ self.id2node.insert(id, node.clone());- if cost >= MAX_DIST {- continue;+ id}+ }+ }- let current_dist = distances.get(&v).unwrap_or(&usize::MAX);+ pub fn insert(&mut self, mut from: Node, mut to: Node, label: String) {+ from.remove_protocol();+ to.remove_protocol();- if cost > *current_dist {- continue;- }+ let (from, to) = (from.into_host(), to.into_host());+ let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));- for edge in node_edges(v) {- if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&usize::MAX) {- let d = cost + 1;+ match &mut self.live_segment {+ Some(segment) => segment.insert(from_id, to_id, label),+ None => {+ let segment_id = Uuid::new_v4().to_string();+ let path = Path::new(&self.path).join("segments").join(&segment_id);+ let mut segment: Segment<S> = Segment::open(path, segment_id);- if d > MAX_DIST {- continue;- }+ segment.insert(from_id, to_id, label);- let next = cmp::Reverse((d, edge_node(&edge)));- queue.push(next);- distances.insert(edge_node(&edge), d);- }+ self.live_segment = Some(segment);}}-- distances}- pub fn distances(&self, source: Node) -> HashMap<Node, usize> {- self.full- .as_ref()- .map(|full_graph| {- let distances = Webgraph::dijkstra(- source,- |node_id| full_graph.outgoing_edges(node_id),- |edge| edge.to,- full_graph,- );+ pub fn merge(&mut self, mut other: Webgraph<S>) {+ other.commit();+ let mut mapping = Vec::new();- distances- .into_iter()- .map(|(id, dist)| (full_graph.id2node(&id).expect("unknown node"), dist))- .collect()- })- .unwrap_or_default()- }+ for (node, other_id) in other.node2id.iter() {+ match self.node2id(&node) {+ Some(this_id) => mapping.push((other_id, this_id)),+ None => {+ let new_id = self.id_or_assign(&node);+ mapping.push((other_id, new_id));+ }+ }+ }- #[allow(unused)]- fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {- self.full- .as_ref()- .map(|full_graph| {- Webgraph::dijkstra(- source,- |node| full_graph.ingoing_edges(node),- |edge| edge.from,- full_graph,- )- })- .unwrap_or_default()- }-- #[allow(unused)]- pub fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {- self.full- .as_ref()- .map(|full_graph| {- self.raw_reversed_distances(source)- .into_iter()- .map(|(id, dist)| (full_graph.id2node(&id).expect("unknown node"), dist))- .collect()- })- .unwrap_or_default()- }-- #[allow(unused)]- pub fn host_distances(&self, source: Node) -> HashMap<Node, usize> {- self.host- .as_ref()- .map(|host_graph| {- self.raw_host_distances(source)- .into_iter()- .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))- .collect()- })- .unwrap_or_default()- }-- pub fn raw_host_distances(&self, source: Node) -> HashMap<NodeID, usize> {- self.host- .as_ref()- .map(|host_graph| {- Webgraph::dijkstra(- source,- |node| host_graph.outgoing_edges(node),- |edge| edge.to,- host_graph,- )- })- .unwrap_or_default()- }-- pub fn raw_host_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {- self.host- .as_ref()- .map(|host_graph| {- Webgraph::dijkstra(- source,- |node| host_graph.ingoing_edges(node),- |edge| edge.from,- host_graph,- )- })- .unwrap_or_default()- }-- #[allow(unused)]- pub fn host_reversed_distances(&self, source: Node) -> HashMap<Node, usize> {- self.host- .as_ref()- .map(|host_graph| {- self.raw_host_reversed_distances(source)- .into_iter()- .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))- .collect()- })- .unwrap_or_default()- }-- pub fn flush(&mut self) {- if let Some(full_graph) = &mut self.full {- full_graph.flush();+ self.executor+ .map(+ |segment| segment.update_id_mapping(mapping.clone()),+ other.segments.iter_mut(),+ )+ .expect("failed to merge webgraphs");++ for mut segment in other.segments {+ segment.flush();+ let id = segment.id();+ let new_path = Path::new(&self.path).join("segments").join(segment.id());+ std::fs::rename(segment.path(), &new_path).unwrap();++ self.meta.comitted_segments.push(segment.id());+ drop(segment);+ self.segments.push(Segment::open(new_path, id));}- if let Some(host_graph) = &mut self.host {- host_graph.flush();++ self.commit();+ }++ pub fn commit(&mut self) {+ if let Some(mut segment) = self.live_segment.take() {+ segment.flush();+ self.meta.comitted_segments.push(segment.id());+ self.segments.push(segment);+ self.save_metadata();}++ self.node2id.flush();+ self.id2node.flush();}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {- if let Some(graph) = &self.full {- if let Some(node_id) = graph.node2id(&node) {- graph- .ingoing_edges(node_id)- .into_iter()- .map(|edge| FullEdge {- from: graph.id2node(&edge.from).unwrap(),- to: graph.id2node(&edge.to).unwrap(),- label: edge.label,- })- .collect()- } else {- Vec::new()- }+ if let Some(node_id) = self.node2id(&node) {+ self.raw_ingoing_edges(&node_id)+ .into_iter()+ .map(|edge| FullEdge {+ from: self.id2node(&edge.from).unwrap(),+ to: self.id2node(&edge.to).unwrap(),+ label: edge.label,+ })+ .collect()} else {Vec::new()}}- pub fn ingoing_edges_host(&self, node: Node) -> Vec<FullEdge> {- if let Some(graph) = &self.host {- if let Some(node_id) = graph.node2id(&node) {- graph- .ingoing_edges(node_id)- .into_iter()- .map(|edge| FullEdge {- from: graph.id2node(&edge.from).unwrap(),- to: graph.id2node(&edge.to).unwrap(),- label: edge.label,- })- .collect()- } else {- Vec::new()- }+ pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {+ self.executor+ .map(|segment| segment.ingoing_edges(node), self.segments.iter())+ .unwrap()+ .into_iter()+ .flatten()+ .collect()+ }++ pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {+ self.executor+ .map(|segment| segment.outgoing_edges(node), self.segments.iter())+ .unwrap()+ .into_iter()+ .flatten()+ .collect()+ }++ pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {+ if let Some(node_id) = self.node2id(&node) {+ self.raw_outgoing_edges(&node_id)+ .into_iter()+ .map(|edge| FullEdge {+ from: self.id2node(&edge.from).unwrap(),+ to: self.id2node(&edge.to).unwrap(),+ label: edge.label,+ })+ .collect()} else {Vec::new()}}++ pub fn node2id(&self, node: &Node) -> Option<NodeID> {+ self.node2id+ .get(node)+ .map(|lock| lock.read().unwrap().0.into())+ }++ pub fn id2node(&self, id: &NodeID) -> Option<Node> {+ self.id2node+ .get(id)+ .map(|lock| lock.read().unwrap().clone())+ }++ pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {+ self.node2id.iter().map(|(_, id)| id)+ }++ pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {+ self.segments.iter().flat_map(|segment| segment.edges())+ }++ pub fn merge_segments(&mut self, num_segments: usize) {+ if num_segments >= self.segments.len() {+ return;+ }++ self.segments+ .sort_by_key(|b| std::cmp::Reverse(b.num_nodes()));++ let mut candidates = Vec::with_capacity(num_segments);++ for segment in self.segments.drain(0..num_segments) {+ candidates.push(SegmentMergeCandidate {+ segment,+ merges: Vec::new(),+ });+ }++ let num_candidates = candidates.len();++ for (next_candidate, segment) in self.segments.drain(0..).enumerate() {+ candidates[next_candidate % num_candidates]+ .merges+ .push(segment);+ }++ let segments = self+ .executor+ .map(+ |mut candidate| {+ for other in candidate.merges {+ candidate.segment.merge(other);+ }++ candidate.segment+ },+ candidates.into_iter(),+ )+ .unwrap();++ self.segments = segments;++ self.meta.comitted_segments = self.segments.iter().map(|segment| segment.id()).collect();++ self.garbage_collect();+ }++ fn garbage_collect(&self) {+ let path = Path::new(&self.path).join("segments");+ let segments: HashSet<_> = self.meta.comitted_segments.iter().cloned().collect();++ for path in fs::read_dir(path).unwrap() {+ let path = path.unwrap();+ let file_name = path.file_name();+ let name = file_name.as_os_str().to_str().unwrap();++ if !segments.contains(name) {+ fs::remove_dir_all(path.path()).unwrap();+ }+ }+ }}impl From<FrozenWebgraph> for Webgraph {@@ -435,48 +605,40 @@ impl From<FrozenWebgraph> for Webgraph {directory::recreate_folder(&frozen.root).unwrap();- let mut builder = WebgraphBuilder::new(path);-- if frozen.has_full {- builder = builder.with_full_graph();- }-- if frozen.has_host {- builder = builder.with_host_graph();- }-- builder.open()+ WebgraphBuilder::new(path).open()}}impl From<Webgraph> for FrozenWebgraph {fn from(mut graph: Webgraph) -> Self {- graph.flush();+ graph.commit();let path = graph.path.clone();- let has_full = graph.full.is_some();- let has_host = graph.host.is_some();drop(graph);let root = directory::scan_folder(path).unwrap();- Self {- root,- has_full,- has_host,- }+ Self { root }}}#[derive(Debug, Serialize, Deserialize)]pub struct FrozenWebgraph {pub root: DirEntry,- has_full: bool,- has_host: bool,}#[cfg(test)]mod test {use super::*;+ fn test_edges() -> Vec<(Node, Node, String)> {+ vec![+ (Node::from("A"), Node::from("B"), String::new()),+ (Node::from("B"), Node::from("C"), String::new()),+ (Node::from("A"), Node::from("C"), String::new()),+ (Node::from("C"), Node::from("A"), String::new()),+ (Node::from("D"), Node::from("C"), String::new()),+ ]+ }+fn test_graph() -> Webgraph {// ┌────┐// │ │@@ -489,18 +651,13 @@ mod test {// │// D- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph = WebgraphBuilder::new_memory().open();- graph.insert(Node::from("A"), Node::from("B"), String::new());- graph.insert(Node::from("B"), Node::from("C"), String::new());- graph.insert(Node::from("A"), Node::from("C"), String::new());- graph.insert(Node::from("C"), Node::from("A"), String::new());- graph.insert(Node::from("D"), Node::from("C"), String::new());+ for (from, to, label) in test_edges() {+ graph.insert(from, to, label);+ }- graph.flush();+ graph.commit();graph}@@ -542,18 +699,14 @@ mod test {#[test]fn merge() {- let mut graph1 = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph1 = WebgraphBuilder::new_memory().open();graph1.insert(Node::from("A"), Node::from("B"), String::new());+ graph1.commit();- let mut graph2 = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph2 = WebgraphBuilder::new_memory().open();graph2.insert(Node::from("B"), Node::from("C"), String::new());+ graph2.commit();graph1.merge(graph2);@@ -566,6 +719,7 @@ mod test {#[test]fn serialize_deserialize_bincode() {let graph = test_graph();+let path = graph.path.clone();let frozen: FrozenWebgraph = graph.into();let bytes = bincode::serialize(&frozen).unwrap();@@ -601,10 +755,7 @@ mod test {assert_eq!(&n.name, "www.example.com?test");- let mut graph = WebgraphBuilder::new_memory()- .with_full_graph()- .with_host_graph()- .open();+ let mut graph = WebgraphBuilder::new_memory().open();graph.insert(Node::from("http://A"),@@ -612,7 +763,7 @@ mod test {String::new(),);- graph.flush();+ graph.commit();let distances = graph.distances(Node::from("A"));assert_eq!(distances.get(&Node::from("B")), Some(&1));@@ -620,4 +771,122 @@ mod test {let distances = graph.distances(Node::from("http://A"));assert!(distances.is_empty());}++ #[test]+ fn merge_segments() {+ let mut graph = WebgraphBuilder::new_memory().open();++ let edges = test_edges();+ let num_edges = edges.len();++ for (from, to, label) in test_edges() {+ graph.insert(from, to, label);+ graph.commit();+ }++ graph.commit();++ assert_eq!(num_edges, graph.segments.len());++ let distances = graph.distances(Node::from("D"));++ assert_eq!(distances.get(&Node::from("C")), Some(&1));+ assert_eq!(distances.get(&Node::from("A")), Some(&2));+ assert_eq!(distances.get(&Node::from("B")), Some(&3));++ let mut res = graph.outgoing_edges(Node::from("A"));+ res.sort_by(|a, b| a.to.name.cmp(&b.to.name));++ assert_eq!(+ res,+ vec![+ FullEdge {+ from: Node::from("A"),+ to: Node::from("B"),+ label: String::new()+ },+ FullEdge {+ from: Node::from("A"),+ to: Node::from("C"),+ label: String::new()+ }+ ]+ );++ let mut res = graph.ingoing_edges(Node::from("C"));+ res.sort_by(|a, b| a.from.name.cmp(&b.from.name));++ assert_eq!(+ res,+ vec![+ FullEdge {+ from: Node::from("A"),+ to: Node::from("C"),+ label: String::new()+ },+ FullEdge {+ from: Node::from("B"),+ to: Node::from("C"),+ label: String::new()+ },+ FullEdge {+ from: Node::from("D"),+ to: Node::from("C"),+ label: String::new()+ },+ ]+ );++ graph.merge_segments(2);+ assert_eq!(graph.segments.len(), 2);++ let distances = graph.distances(Node::from("D"));++ assert_eq!(distances.get(&Node::from("C")), Some(&1));+ assert_eq!(distances.get(&Node::from("A")), Some(&2));+ assert_eq!(distances.get(&Node::from("B")), Some(&3));++ let mut res = graph.outgoing_edges(Node::from("A"));+ res.sort_by(|a, b| a.to.name.cmp(&b.to.name));++ assert_eq!(+ res,+ vec![+ FullEdge {+ from: Node::from("A"),+ to: Node::from("B"),+ label: String::new()+ },+ FullEdge {+ from: Node::from("A"),+ to: Node::from("C"),+ label: String::new()+ }+ ]+ );++ let mut res = graph.ingoing_edges(Node::from("C"));+ res.sort_by(|a, b| a.from.name.cmp(&b.from.name));++ assert_eq!(+ res,+ vec![+ FullEdge {+ from: Node::from("A"),+ to: Node::from("C"),+ label: String::new()+ },+ FullEdge {+ from: Node::from("B"),+ to: Node::from("C"),+ label: String::new()+ },+ FullEdge {+ from: Node::from("D"),+ to: Node::from("C"),+ label: String::new()+ },+ ]+ );+ }}diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsnew file mode 100644index 00000000..aa20d7a0--- /dev/null+++ b/core/src/webgraph/segment.rs@@ -0,0 +1,460 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use serde::{de::DeserializeOwned, Deserialize, Serialize};+use std::{+ collections::{BTreeMap, HashSet},+ fmt::Debug,+ marker::PhantomData,+ path::Path,+ sync::{Arc, RwLock},+};++use super::{Edge, NodeID, Store, StoredEdge};+use crate::kv::{rocksdb_store::RocksDbStore, Kv};++#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]+pub struct SegmentNodeID(u64);++impl From<u64> for SegmentNodeID {+ fn from(val: u64) -> Self {+ Self(val)+ }+}++pub(crate) struct Adjacency {+ pub(crate) tree: CachedStore<SegmentNodeID, HashSet<StoredEdge>>,+}++impl Adjacency {+ fn insert(&mut self, from: SegmentNodeID, to: SegmentNodeID, label: String) {+ let edge = StoredEdge { other: to, label };++ if let Some(existing) = self.tree.get(&from) {+ existing.write().unwrap().insert(edge);+ } else {+ let mut set = HashSet::new();+ set.insert(edge);+ self.tree.insert(from, set);+ }+ }++ fn edges(&self, node: &SegmentNodeID) -> Vec<StoredEdge> {+ self.tree+ .get(node)+ .map(|r| r.read().unwrap().clone())+ .unwrap_or_default()+ .into_iter()+ .collect()+ }+}++pub struct CachedStore<K, V>+where+ K: Ord + Eq + Serialize + DeserializeOwned + Clone,+ V: Serialize + DeserializeOwned + Clone,+{+ store: Box<dyn Kv<K, V> + Send + Sync>,+ cache: RwLock<BTreeMap<K, Arc<RwLock<V>>>>,+ cache_size: usize,+}++impl<K, V> CachedStore<K, V>+where+ K: Ord + Eq + Serialize + DeserializeOwned + Clone,+ V: Serialize + DeserializeOwned + Clone + Debug,+{+ pub fn new(store: Box<dyn Kv<K, V> + Send + Sync>, cache_size: usize) -> Self {+ Self {+ store,+ cache_size,+ cache: RwLock::new(BTreeMap::new()),+ }+ }++ fn update_cache(&self, key: &K) {+ if !self.cache.read().unwrap().contains_key(key) {+ let val = self.store.get(key);++ if let Some(val) = val {+ self.cache+ .write()+ .unwrap()+ .insert(key.clone(), Arc::new(RwLock::new(val)));+ }+ }+ }++ pub fn get(&self, key: &K) -> Option<Arc<RwLock<V>>> {+ self.update_cache(key);+ let guard = self.cache.read().unwrap();+ guard.get(key).cloned()+ }++ pub fn insert(&mut self, key: K, value: V) {+ let cache = self.cache.get_mut().unwrap();++ cache.insert(key, Arc::new(RwLock::new(value)));++ if cache.len() >= self.cache_size {+ while cache.len() > self.cache_size / 2 {+ if let Some((key, value)) = cache.pop_first() {+ self.store.insert(key, value.write().unwrap().clone());+ }+ }++ self.store.flush();+ }+ }++ pub fn flush(&mut self) {+ let cache = self.cache.get_mut().unwrap();++ while let Some((key, value)) = cache.pop_first() {+ self.store.insert(key, value.write().unwrap().clone());+ }++ self.store.flush();+ }++ pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {+ self.store.iter()+ }++ pub fn delete(&mut self, key: &K) {+ self.store.delete(key);+ self.cache.write().unwrap().remove(key);+ }+}++pub struct Segment<S: Store> {+ adjacency: Adjacency,+ reversed_adjacency: Adjacency,+ id_mapping: CachedStore<NodeID, SegmentNodeID>,+ rev_id_mapping: CachedStore<SegmentNodeID, NodeID>,+ meta: CachedStore<String, u64>,+ store: PhantomData<S>,+ id: String,+ path: String,+}++impl<S: Store> Segment<S> {+ #[cfg(test)]+ pub(crate) fn temporary() -> Segment<S> {+ S::temporary()+ }++ pub fn open<P: AsRef<Path>>(path: P, id: String) -> Self {+ S::open(path, id)+ }++ pub fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Self {+ S::open_read_only(path, id)+ }++ fn next_id(&self) -> SegmentNodeID {+ let key = &"next_id".to_string();+ self.meta+ .get(key)+ .map(|arc| *arc.read().unwrap())+ .unwrap_or(0)+ .into()+ }++ pub fn num_nodes(&self) -> usize {+ self.next_id().0 as usize+ }++ fn increment_next_id(&mut self) {+ let current_id = self.next_id().0;+ let next_id = current_id + 1;+ self.meta.insert("next_id".to_string(), next_id);+ }++ fn id_and_increment(&mut self) -> SegmentNodeID {+ let id = self.next_id();+ self.increment_next_id();+ id+ }++ fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {+ self.id_mapping.get(node).map(|lock| *lock.read().unwrap())+ }++ fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {+ self.rev_id_mapping+ .get(node)+ .map(|lock| *lock.read().unwrap())+ }++ pub fn outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {+ match self.id_mapping(node) {+ Some(segment_id) => self+ .adjacency+ .edges(&segment_id)+ .into_iter()+ .map(|edge| Edge {+ from: *node,+ to: self.rev_id_mapping(&edge.other).unwrap(),+ label: edge.label,+ })+ .collect(),+ None => Vec::new(),+ }+ }++ pub fn ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {+ match self.id_mapping(node) {+ Some(segment_id) => self+ .reversed_adjacency+ .edges(&segment_id)+ .into_iter()+ .map(|edge| Edge {+ from: self.rev_id_mapping(&edge.other).unwrap(),+ to: *node,+ label: edge.label,+ })+ .collect(),+ None => Vec::new(),+ }+ }++ fn id_or_assign(&mut self, node: &NodeID) -> SegmentNodeID {+ match self.id_mapping(node) {+ Some(id) => id,+ None => {+ let id = self.id_and_increment();++ self.id_mapping.insert(*node, id);+ self.rev_id_mapping.insert(id, *node);++ id+ }+ }+ }++ pub fn insert(&mut self, from: NodeID, to: NodeID, label: String) {+ let from_id = self.id_or_assign(&from);+ let to_id = self.id_or_assign(&to);++ self.adjacency.insert(from_id, to_id, label.clone());+ self.reversed_adjacency.insert(to_id, from_id, label);+ }++ pub fn flush(&mut self) {+ self.adjacency.tree.flush();+ self.reversed_adjacency.tree.flush();+ self.id_mapping.flush();+ self.rev_id_mapping.flush();+ self.meta.flush();+ }++ pub fn id(&self) -> String {+ self.id.clone()+ }++ pub fn path(&self) -> &Path {+ Path::new(&self.path)+ }++ pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {+ self.adjacency+ .tree+ .iter()+ .flat_map(move |(node_id, edges)| {+ let from = self.rev_id_mapping(&node_id).unwrap();++ edges.into_iter().map(move |stored_edge| Edge {+ from,+ to: self.rev_id_mapping(&stored_edge.other).unwrap(),+ label: stored_edge.label,+ })+ })+ }++ pub fn merge(&mut self, mut other: Segment<S>) {+ other.flush();++ for edge in other.edges() {+ self.insert(edge.from, edge.to, edge.label);+ }++ self.flush();+ }++ pub fn update_id_mapping(&mut self, mapping: Vec<(NodeID, NodeID)>) {+ let mut new_mappings = Vec::with_capacity(mapping.len());++ for (old_id, new_id) in mapping {+ if let Some(segment_id) = self.id_mapping(&old_id) {+ self.id_mapping.delete(&old_id);+ self.rev_id_mapping.delete(&segment_id);++ new_mappings.push((segment_id, new_id));+ }+ }++ for (segment_id, new_id) in new_mappings {+ self.id_mapping.insert(new_id, segment_id);+ self.rev_id_mapping.insert(segment_id, new_id);+ }++ self.flush();+ }+}++impl Store for RocksDbStore {+ fn open<P: AsRef<std::path::Path>>(path: P, id: String) -> Segment<Self> {+ let adjacency = RocksDbStore::open(path.as_ref().join("adjacency"));+ let reversed_adjacency = RocksDbStore::open(path.as_ref().join("reversed_adjacency"));+ let id_mapping = RocksDbStore::open(path.as_ref().join("id_mapping"));+ let rev_id_mapping = RocksDbStore::open(path.as_ref().join("rev_id_mapping"));+ let meta = RocksDbStore::open(path.as_ref().join("meta"));++ Segment {+ adjacency: Adjacency {+ tree: CachedStore::new(adjacency, 50_000),+ },+ reversed_adjacency: Adjacency {+ tree: CachedStore::new(reversed_adjacency, 50_000),+ },+ id_mapping: CachedStore::new(id_mapping, 100_000),+ rev_id_mapping: CachedStore::new(rev_id_mapping, 100_000),+ meta: CachedStore::new(meta, 1_000),+ store: Default::default(),+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),+ id,+ }+ }++ fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Segment<Self> {+ let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));+ let reversed_adjacency =+ RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));+ let id_mapping = RocksDbStore::open_read_only(path.as_ref().join("id_mapping"));+ let rev_id_mapping = RocksDbStore::open_read_only(path.as_ref().join("rev_id_mapping"));+ let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));++ Segment {+ adjacency: Adjacency {+ tree: CachedStore::new(adjacency, 50_000),+ },+ reversed_adjacency: Adjacency {+ tree: CachedStore::new(reversed_adjacency, 50_000),+ },+ id_mapping: CachedStore::new(id_mapping, 100_000),+ rev_id_mapping: CachedStore::new(rev_id_mapping, 100_000),+ meta: CachedStore::new(meta, 1_000),+ store: Default::default(),+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),+ id,+ }+ }+}++#[cfg(test)]+mod test {+ use super::*;++ #[test]+ fn simple_triangle_graph() {+ // ┌────┐+ // │ │+ // ┌───0◄─┐ │+ // │ │ │+ // ▼ │ │+ // 1─────►2◄┘++ let mut store: Segment<RocksDbStore> = Segment::temporary();++ let a = NodeID(0);+ let b = NodeID(1);+ let c = NodeID(2);++ store.insert(a, b, String::new());+ store.insert(b, c, String::new());+ store.insert(c, a, String::new());+ store.insert(a, c, String::new());++ store.flush();++ let mut out = store.outgoing_edges(&a);+ out.sort();++ assert_eq!(+ out,+ vec![+ Edge {+ from: a,+ to: b,+ label: String::new()+ },+ Edge {+ from: a,+ to: c,+ label: String::new()+ },+ ]+ );++ let mut out = store.outgoing_edges(&b);+ out.sort();+ assert_eq!(+ out,+ vec![Edge {+ from: b,+ to: c,+ label: String::new()+ },]+ );++ let mut out = store.ingoing_edges(&c);+ out.sort();+ assert_eq!(+ out,+ vec![+ Edge {+ from: a,+ to: c,+ label: String::new()+ },+ Edge {+ from: b,+ to: c,+ label: String::new()+ },+ ]+ );++ assert_eq!(+ store.ingoing_edges(&a),+ vec![Edge {+ from: c,+ to: a,+ label: String::new()+ },]+ );++ assert_eq!(+ store.ingoing_edges(&b),+ vec![Edge {+ from: a,+ to: b,+ label: String::new()+ },]+ );+ }+}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex d428cbab..032768c2 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -19,6 +19,7 @@ use crate::{schema::{FastField, TextField},simhash,tokenizer::{self, FlattenedJson},+ webgraph::NodeID,Error, Result,};use chrono::{DateTime, FixedOffset, Utc};@@ -131,7 +132,7 @@ pub struct Webpage {pub fetch_time_ms: u64,pub pre_computed_score: f64,pub primary_image: Option<StoredPrimaryImage>,- pub node_id: Option<u64>,+ pub node_id: Option<NodeID>,pub crawl_stability: f64,pub host_topic: Option<Topic>,pub dmoz_description: Option<String>,@@ -249,7 +250,7 @@ impl Webpage {schema.get_field(Field::Fast(FastField::HostNodeID).name()).expect("Failed to get node_id field"),- self.node_id.unwrap_or(u64::MAX),+ self.node_id.map(|n| n.0).unwrap_or(u64::MAX),);doc.add_u64(diff --git a/scripts/export_crossencoder b/scripts/export_crossencodernew file mode 100755index 00000000..fba0becb--- /dev/null+++ b/scripts/export_crossencoder@@ -0,0 +1,4 @@+#!/bin/bash+pip3 install transformers[onnx] torch+python3 -m transformers.onnx --model=cross-encoder/ms-marco-MiniLM-L-6-v2 data/+mv data/model.onnx data/crossencoder.onnxdiff --git a/scripts/run_frontend.sh b/scripts/run_frontend.shindex f7c69417..0a8564b1 100644--- a/scripts/run_frontend.sh+++ b/scripts/run_frontend.sh@@ -1,4 +1,4 @@-#!/usr/bin/bash+#!/bin/bashtrap 'kill $(jobs -p)' EXITcargo run -- search-server configs/search_server.toml &cargo run -- frontend configs/frontend.toml