web search done right
memory mapped graph store
Mikkel Denker0cc4cac100d6
diff --git a/Cargo.lock b/Cargo.lockindex 94a74300..70132f79 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -391,6 +391,28 @@ version = "3.12.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"+[[package]]+name = "bytecheck"+version = "0.6.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "13fe11640a23eb24562225322cd3e452b93a3d4091d62fab69c70542fcd17d1f"+dependencies = [+ "bytecheck_derive",+ "ptr_meta",+ "simdutf8",+]++[[package]]+name = "bytecheck_derive"+version = "0.6.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e31225543cb46f81a7e224762764f4a6a0f097b1db0b175f69e8065efaa42de5"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "bytemuck"version = "1.13.0"@@ -2048,9 +2070,9 @@ dependencies = [[[package]]name = "memmap2"-version = "0.5.8"+version = "0.5.10"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc"+checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327"dependencies = ["libc",]@@ -2706,6 +2728,26 @@ dependencies = ["unicode-ident",]+[[package]]+name = "ptr_meta"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"+dependencies = [+ "ptr_meta_derive",+]++[[package]]+name = "ptr_meta_derive"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "quick-xml"version = "0.23.1"@@ -2905,6 +2947,15 @@ dependencies = ["winapi",]+[[package]]+name = "rend"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab"+dependencies = [+ "bytecheck",+]+[[package]]name = "reqwest"version = "0.11.14"@@ -2959,6 +3010,31 @@ dependencies = ["winapi",]+[[package]]+name = "rkyv"+version = "0.7.40"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c30f1d45d9aa61cbc8cd1eb87705470892289bb2d01943e7803b873a57404dc3"+dependencies = [+ "bytecheck",+ "hashbrown",+ "ptr_meta",+ "rend",+ "rkyv_derive",+ "seahash",+]++[[package]]+name = "rkyv_derive"+version = "0.7.40"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff26ed6c7c4dfc2aa9480b86a60e3c7233543a270a680e10758a507c5a4ce476"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "rocksdb"version = "0.19.0"@@ -3133,6 +3209,12 @@ dependencies = ["syn",]+[[package]]+name = "seahash"+version = "4.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"+[[package]]name = "security-framework"version = "2.8.2"@@ -3295,6 +3377,12 @@ version = "0.3.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "14a5df39617d7c8558154693a1bb8157a4aab8179209540cc0b10e5dc24e0b18"+[[package]]+name = "simdutf8"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"+[[package]]name = "siphasher"version = "0.3.10"@@ -3409,6 +3497,7 @@ dependencies = ["maplit","md5","memmap",+ "memmap2","min-max-heap","num_cpus","once_cell",@@ -3420,6 +3509,7 @@ dependencies = ["rayon","regex","reqwest",+ "rkyv","rocksdb","rust-stemmers","scylla",diff --git a/Cargo.toml b/Cargo.tomlindex 19c017a8..a30df524 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -70,4 +70,6 @@ byteorder = "1.4.3"async-stream = "0.3.3"tokio-stream = "0.1.11"scylla = "0.7.0"-chitchat = "0.5.0"\ No newline at end of file+chitchat = "0.5.0"+memmap2 = "0.5.10"+rkyv = "0.7.40"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex b5eef1e2..fad948c6 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -75,6 +75,8 @@ async-stream = { workspace = true }tokio-stream = { workspace = true }scylla = { workspace = true }chitchat = { workspace = true }+memmap2 = { workspace = true }+rkyv = { workspace = true }[build-dependencies]lalrpop = { workspace = true }diff --git a/core/benches/online-harmonic.rs b/core/benches/online-harmonic.rsindex aa402093..d812ce6f 100644--- a/core/benches/online-harmonic.rs+++ b/core/benches/online-harmonic.rs@@ -17,7 +17,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {for (node, centrality) in centrality.host {harmonic_centrality_store.host- .insert(webgraph.node2id(&node).unwrap(), centrality);+ .insert(*webgraph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();diff --git a/core/examples/print_approx_harmonic_centrality.rs b/core/examples/print_approx_harmonic_centrality.rsindex 1be026d7..0278282c 100644--- a/core/examples/print_approx_harmonic_centrality.rs+++ b/core/examples/print_approx_harmonic_centrality.rs@@ -110,11 +110,11 @@ fn print_top_nodes(pub fn main() {const TOP_N: usize = 50;- let graph = WebgraphBuilder::new("data/webgraph").read_only(true).open();- let nodes: Vec<_> = graph.nodes().collect();+ let graph = WebgraphBuilder::new("data/webgraph").open();+ let nodes: Vec<_> = graph.nodes().cloned().collect();let id2node: HashMap<_, _> = nodes.iter()- .map(|id| (*id, graph.id2node(id).unwrap()))+ .map(|id| (*id, graph.id2node(id).unwrap().clone())).collect();let store = CentralityStore::open("data/centrality");diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rsindex 711fb14f..e919f8a0 100644--- a/core/examples/print_inbound_links.rs+++ b/core/examples/print_inbound_links.rs@@ -20,7 +20,7 @@ use stract::{};pub fn main() {- let graph = WebgraphBuilder::new("data/webgraph").read_only(true).open();+ let graph = WebgraphBuilder::new("data/webgraph").open();let inbound = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();for host in ["archwaypublishing.com", "wordpressfoundation.org"] {@@ -28,7 +28,7 @@ pub fn main() {let node = Node::from(host);let node_id = graph.node2id(&node).unwrap();- let inbound_vec = inbound.get(&node_id).unwrap();+ let inbound_vec = inbound.get(node_id).unwrap();println!("{:?}", inbound_vec);for edge in graph.ingoing_edges(node) {diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex 0f2d501a..7d5fd945 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -22,17 +22,17 @@ pub struct Centrality {}impl Centrality {pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, output_path: P) {- let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();+ let graph = WebgraphBuilder::new(webgraph_path).open();CentralityStore::build_harmonic(&graph, output_path);}pub fn build_online<P: AsRef<Path>>(webgraph_path: P, output_path: P) {- let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();+ let graph = WebgraphBuilder::new(webgraph_path).open();CentralityStore::build_online(&graph, output_path);}pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {- let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();+ let graph = WebgraphBuilder::new(webgraph_path).open();CentralityStore::build_similarity(&graph, output_path);}}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 2cbf5999..792b553a 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -74,7 +74,7 @@ impl IndexingWorker {) -> Self {Self {centrality_store: IndexerCentralityStore::open(centrality_store_path),- webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).read_only(true).open()),+ webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).open()),crawl_stabilty: crawl_stability_path.map(CrawlStability::open),topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),}diff --git a/core/src/executor.rs b/core/src/executor.rsindex 53a088f7..4244bd95 100644--- a/core/src/executor.rs+++ b/core/src/executor.rs@@ -78,6 +78,7 @@ impl Executor {// This is important as it makes it possible for the rx iteration to// terminate.};+let mut result_placeholders: Vec<Option<R>> =std::iter::repeat_with(|| None).take(num_jobs).collect();diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rsindex 028ca5ed..6e1a9cc6 100644--- a/core/src/kv/rocksdb_store.rs+++ b/core/src/kv/rocksdb_store.rs@@ -44,25 +44,6 @@ impl RocksDbStore {Box::new(DB::open(&options, path).expect("unable to open rocks db"))}-- pub fn open_read_only<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>- where- P: AsRef<std::path::Path>,- K: Serialize + DeserializeOwned + 'static,- V: Serialize + DeserializeOwned + 'static,- {- if !path.as_ref().exists() {- fs::create_dir_all(path.as_ref()).expect("faild to create dir");- }-- let mut options = Options::default();-- options.create_if_missing(true);- options.set_max_open_files(1);- options.set_max_file_opening_threads(1);-- Box::new(DB::open_for_read_only(&options, path, false).expect("unable to open rocks db"))- }}impl<K, V> Kv<K, V> for rocksdb::DBdiff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 7fbf474f..f937b96b 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -626,7 +626,7 @@ mod tests {primary_image: None,host_topic: None,dmoz_description: None,- node_id: Some(graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),+ node_id: Some(*graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),}).expect("failed to insert webpage");index@@ -656,7 +656,7 @@ mod tests {fetch_time_ms: 500,dmoz_description: None,host_topic: None,- node_id: Some(graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),+ node_id: Some(*graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),}).expect("failed to insert webpage");index@@ -686,7 +686,7 @@ mod tests {fetch_time_ms: 500,host_topic: None,dmoz_description: None,- node_id: Some(graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),+ node_id: Some(*graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),}).expect("failed to insert webpage");diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex c5d00dbb..6070d093 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -170,7 +170,10 @@ impl CentralityStore {pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {let mut store = CentralityStore::open(output_path.as_ref());- store.node2id = graph.node_ids().collect();+ store.node2id = graph+ .node_ids()+ .map(|(node, node_id)| (node.clone(), *node_id))+ .collect();let harmonic_centrality = HarmonicCentrality::calculate(graph);Self::store_host(&output_path, &mut store, harmonic_centrality);diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex b12d11ce..d4a1e3b7 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -255,18 +255,18 @@ mod tests {for (node, centrality) in harmonic.host {harmonic_centrality_store.host- .insert(graph.node2id(&node).unwrap(), centrality);+ .insert(*graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();let inbound =InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);- let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);+ let scorer = inbound.scorer(&[*graph.node2id(&Node::from("b.com")).unwrap()], &[]);let e = graph.node2id(&Node::from("e.com")).unwrap();let d = graph.node2id(&Node::from("d.com")).unwrap();- assert!(scorer.score(&e) > scorer.score(&d));+ assert!(scorer.score(e) > scorer.score(d));}#[test]@@ -285,7 +285,7 @@ mod tests {for (node, centrality) in harmonic.host {harmonic_centrality_store.host- .insert(graph.node2id(&node).unwrap(), centrality);+ .insert(*graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();@@ -323,7 +323,7 @@ mod tests {crawl_stability: 0.0,primary_image: None,host_topic: None,- node_id: Some(graph.node2id(&Node::from("e.com")).unwrap()),+ node_id: Some(*graph.node2id(&Node::from("e.com")).unwrap()),dmoz_description: None,}).expect("failed to insert webpage");@@ -353,7 +353,7 @@ mod tests {crawl_stability: 0.0,primary_image: None,host_topic: None,- node_id: Some(graph.node2id(&Node::from("d.com")).unwrap()),+ node_id: Some(*graph.node2id(&Node::from("d.com")).unwrap()),dmoz_description: None,}).expect("failed to insert webpage");diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex e2d0b847..6cffce1d 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -1233,7 +1233,7 @@ mod tests {for (node, centrality) in harmonic.host {harmonic_centrality_store.host- .insert(webgraph.node2id(&node).unwrap(), centrality);+ .insert(*webgraph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 6f4d55cf..63fc8ad4 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -77,6 +77,7 @@ mod tests {node_id: Some(graph.node2id(&Node::from("https://www.first.com").into_host())+ .copied().unwrap(),),})@@ -111,6 +112,7 @@ mod tests {node_id: Some(graph.node2id(&Node::from("https://www.second.com").into_host())+ .copied().unwrap(),),})@@ -143,7 +145,7 @@ mod tests {dmoz_description: None,host_topic: None,node_id: Some(- graph+ *graph.node2id(&Node::from("https://www.third.com").into_host()).unwrap(),),diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex caebf050..d45cc4ee 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -100,6 +100,7 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {for (website, score) in websites.iter_mut().zip(scores.into_iter()) {let website = website.as_mut_ranking();+ website.signals.insert(Signal::CrossEncoder, score);website.score += score;}}diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex fb4273ff..e2487be7 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -552,9 +552,13 @@ struct RuleBoost {boost: f64,}+struct OpticBoosts {+ rules: Vec<RuleBoost>,+}+struct SegmentReader {text_fields: EnumMap<TextField, TextFieldData>,- optic_rule_boosts: Vec<RuleBoost>,+ optic_boosts: OpticBoosts,fastfield_reader: Arc<fastfield_reader::SegmentReader>,}@@ -702,7 +706,9 @@ impl SignalAggregator {self.segment_reader = Some(SegmentReader {text_fields,fastfield_reader: fastfield_segment_reader,- optic_rule_boosts,+ optic_boosts: OpticBoosts {+ rules: optic_rule_boosts,+ },});Ok(())@@ -772,22 +778,29 @@ impl SignalAggregator {}pub fn boosts(&mut self, doc: DocId) -> Option<f64> {- self.segment_reader.as_mut().and_then(|segment_reader| {- let mut res = 0.0;- for rule in &mut segment_reader.optic_rule_boosts {+ self.segment_reader.as_mut().map(|segment_reader| {+ let mut downrank = 0.0;+ let mut boost = 0.0;++ for rule in &mut segment_reader.optic_boosts.rules {if rule.docset.doc() > doc {continue;}if rule.docset.doc() == doc || rule.docset.seek(doc) == doc {- res += rule.boost;+ if rule.boost < 0.0 {+ downrank += rule.boost.abs();+ } else {+ boost += rule.boost;+ }}}- if res == 0.0 {- None+ if downrank > boost {+ let diff = downrank - boost;+ 1.0 / (1.0 + diff)} else {- Some(res)+ boost - downrank + 1.0}})}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 8c5f71c5..e3a4eb24 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -515,7 +515,7 @@ impl DistributedSearcher {.collect();let median = if scores.len() % 2 == 0 {- (scores[(scores.len() / 2) - 1] + scores[(scores.len() / 2)]) / 2.0+ (scores[(scores.len() / 2) - 1] + scores[scores.len() / 2]) / 2.0} else {scores[scores.len() / 2]};diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rsindex b2601ddf..10d7a466 100644--- a/core/src/webgraph/centrality/betweenness.rs+++ b/core/src/webgraph/centrality/betweenness.rs@@ -32,7 +32,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)let mut n = 0;let mut max_dist = 0;- let nodes: Vec<_> = graph.nodes().take(100_000).collect();+ let nodes: Vec<_> = graph.nodes().cloned().take(100_000).collect();let pb =if with_progress {@@ -128,7 +128,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)(centrality.into_iter()- .map(|(id, centrality)| (graph.id2node(&id).unwrap(), centrality / norm))+ .map(|(id, centrality)| (graph.id2node(&id).unwrap().clone(), centrality / norm)).collect(),max_dist,)diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex 2ddb1237..4c5e54ea 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -150,7 +150,7 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {.filter(|(_, centrality)| *centrality > 0.0).map(|(node_id, centrality)| {(- graph.id2node(&NodeID::from(node_id)).unwrap(),+ graph.id2node(&NodeID::from(node_id)).unwrap().clone(),centrality / norm_factor,)})diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex bfd03cb1..4febd1c5 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -267,7 +267,7 @@ whereF1: Fn(NodeID) -> Vec<Edge>,F2: Fn(&Edge) -> NodeID,{- let source_id = graph.node2id(&source);+ let source_id = graph.node2id(&source).copied();if source_id.is_none() {return BTreeMap::new();@@ -358,7 +358,7 @@ impl OnlineHarmonicCentrality {// as an estimatelet mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();- for (node, node_id) in graph.node_ids() {+ for (node, node_id) in graph.node_ids().map(|(node, id)| (node.clone(), *id)) {let score = centrality.host.get(&node_id).unwrap_or(0.0);let candidate = ProxyNodeCandidate { node, score };@@ -389,7 +389,7 @@ impl OnlineHarmonicCentrality {.collect();Arc::new(ProxyNode {- id: graph.node2id(&node).unwrap(),+ id: *graph.node2id(&node).unwrap(),dist_to_node,dist_from_node,})@@ -476,7 +476,7 @@ mod tests {for (node, centrality) in harmonic.host {harmonic_centrality_store.host- .insert(graph.node2id(&node).unwrap(), centrality);+ .insert(*graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();@@ -485,23 +485,23 @@ mod tests {let liked_nodes: Vec<_> = vec![Node::from("B".to_string()), Node::from("E".to_string())].into_iter()- .filter_map(|node| graph.node2id(&node))+ .filter_map(|node| graph.node2id(&node).copied()).collect();let scorer = centrality.scorer(&liked_nodes, &[]);assert!(- scorer.score(graph.node2id(&Node::from("E".to_string())).unwrap())- > scorer.score(graph.node2id(&Node::from("H".to_string())).unwrap())+ scorer.score(*graph.node2id(&Node::from("E".to_string())).unwrap())+ > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap()));assert!(- scorer.score(graph.node2id(&Node::from("H".to_string())).unwrap())- > scorer.score(graph.node2id(&Node::from("C".to_string())).unwrap())+ scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())+ > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap()));assert!(- scorer.score(graph.node2id(&Node::from("C".to_string())).unwrap())- > scorer.score(graph.node2id(&Node::from("A".to_string())).unwrap())+ scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())+ > scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap()));}@@ -514,7 +514,7 @@ mod tests {for (node, centrality) in harmonic.host {harmonic_centrality_store.host- .insert(graph.node2id(&node).unwrap(), centrality);+ .insert(*graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();@@ -523,7 +523,7 @@ mod tests {let disliked_nodes: Vec<_> = vec![Node::from("D".to_string()), Node::from("E".to_string())].into_iter()- .map(|node| graph.node2id(&node).unwrap())+ .map(|node| *graph.node2id(&node).unwrap()).collect();let scorer = centrality.scorer(&[], &disliked_nodes);@@ -542,7 +542,7 @@ mod tests {for (node, centrality) in harmonic.host {harmonic_centrality_store.host- .insert(graph.node2id(&node).unwrap(), centrality);+ .insert(*graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();@@ -551,29 +551,29 @@ mod tests {let liked_nodes: Vec<_> = vec![Node::from("B".to_string()), Node::from("E".to_string())].into_iter()- .filter_map(|node| graph.node2id(&node))+ .filter_map(|node| graph.node2id(&node).copied()).collect();let disliked_nodes: Vec<_> = vec![Node::from("F".to_string())].into_iter()- .filter_map(|node| graph.node2id(&node))+ .filter_map(|node| graph.node2id(&node).copied()).collect();let scorer = centrality.scorer(&liked_nodes, &disliked_nodes);assert!(- scorer.score(graph.node2id(&Node::from("E".to_string())).unwrap())- > scorer.score(graph.node2id(&Node::from("H".to_string())).unwrap())+ scorer.score(*graph.node2id(&Node::from("E".to_string())).unwrap())+ > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap()));assert!(- scorer.score(graph.node2id(&Node::from("H".to_string())).unwrap())- > scorer.score(graph.node2id(&Node::from("C".to_string())).unwrap())+ scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())+ > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap()));assert!(- scorer.score(graph.node2id(&Node::from("C".to_string())).unwrap())- > scorer.score(graph.node2id(&Node::from("A".to_string())).unwrap())+ scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())+ > scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap()));}}diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rsindex 660256a6..a4d8b407 100644--- a/core/src/webgraph/centrality/topic.rs+++ b/core/src/webgraph/centrality/topic.rs@@ -151,7 +151,7 @@ impl TopicCentrality {TopDocs::with_limit(1000, index.inverted_index.fastfield_reader(&tv_searcher)).tweak_score(score_tweaker);- let mut nodes: Vec<_> = webgraph.nodes().collect();+ let mut nodes: Vec<_> = webgraph.nodes().copied().collect();nodes.sort();let mut node_scores = vec![vec![0.0; NUM_TOPICS]; nodes.len()];@@ -179,7 +179,7 @@ impl TopicCentrality {let top_sites: Vec<_> = top_sites.into_iter()- .filter_map(|node| webgraph.node2id(&node))+ .filter_map(|node| webgraph.node2id(node).copied()).collect();if top_sites.is_empty() {diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 1dd0a844..7e3b3fc6 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -15,25 +15,23 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.mod segment;+use rkyv::Archive;+use serde::de::DeserializeOwned;use serde::{Deserialize, Serialize};use std::collections::{BTreeMap, BinaryHeap, HashSet};-use std::fs::File;+use std::fs::{File, OpenOptions};use std::io::{BufReader, BufWriter, Read, Write};use std::path::Path;use std::sync::Arc;use std::{cmp, fs};-use uuid::Uuid;--use segment::Segment;use crate::directory::{self, DirEntry};use crate::executor::Executor;use crate::webpage::Url;-use crate::kv::rocksdb_store::RocksDbStore;-pub mod centrality;-use self::segment::{CachedStore, SegmentNodeID};+mod store;+use self::segment::{LiveSegment, SegmentNodeID, StoredSegment};const MAX_NUM_SEGMENTS: usize = 400;@@ -46,7 +44,8 @@ impl From<u64> for NodeID {}}-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]+#[derive(Debug, Clone, Archive, rkyv::Serialize, rkyv::Deserialize, PartialEq, Eq, Hash)]+#[archive_attr(derive(Eq, Hash, PartialEq, Debug))]pub(crate) struct StoredEdge {other: SegmentNodeID,label: String,@@ -106,11 +105,25 @@ impl From<Url> for Node {}}-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]+#[derive(Debug, PartialEq, Eq)]pub struct Edge {pub from: NodeID,pub to: NodeID,- pub label: String,+ pub label: Loaded<String>,+}++#[derive(Debug, PartialEq, Eq)]+pub enum Loaded<T> {+ Some(T),+ NotYet,+}+impl<T> Loaded<T> {+ fn loaded(self) -> Option<T> {+ match self {+ Loaded::Some(t) => Some(t),+ Loaded::NotYet => None,+ }+ }}#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]@@ -122,7 +135,6 @@ pub struct FullEdge {pub struct WebgraphBuilder {path: Box<Path>,- read_only: bool,}impl WebgraphBuilder {@@ -137,34 +149,11 @@ impl WebgraphBuilder {pub fn new<P: AsRef<Path>>(path: P) -> Self {Self {path: path.as_ref().into(),- read_only: false,}}- pub fn read_only(mut self, read_only: bool) -> Self {- self.read_only = read_only;-- self- }-pub fn open(self) -> Webgraph {- if self.read_only {- Webgraph::open_read_only(self.path)- } else {- Webgraph::open(self.path)- }- }-}--pub trait Store-where- Self: Sized + Send + Sync,-{- fn open<P: AsRef<Path>>(path: P, id: String) -> Segment<Self>;- fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Segment<Self>;-- fn temporary() -> Segment<Self> {- Self::open(crate::gen_temp_path(), "id123".to_string())+ Webgraph::open(self.path)}}@@ -195,8 +184,8 @@ wherelet mut queue = BinaryHeap::new();- queue.push(cmp::Reverse((0, source_id)));- distances.insert(source_id, 0);+ queue.push(cmp::Reverse((0, *source_id)));+ distances.insert(*source_id, 0);while let Some(state) = queue.pop() {let (cost, v) = state.0;@@ -225,7 +214,7 @@ impl ShortestPaths for Webgraph {fn distances(&self, source: Node) -> BTreeMap<Node, u8> {self.raw_distances(source).into_iter()- .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node").clone(), dist)).collect()}@@ -250,7 +239,7 @@ impl ShortestPaths for Webgraph {fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {self.raw_reversed_distances(source).into_iter()- .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node").clone(), dist)).collect()}}@@ -263,22 +252,53 @@ struct Meta {next_node_id: u64,}-struct SegmentMergeCandidate<S: Store> {- segment: Segment<S>,- merges: Vec<Segment<S>>,+struct SegmentMergeCandidate {+ segment: StoredSegment,+ merges: Vec<StoredSegment>,}-pub struct Webgraph<S: Store = RocksDbStore> {+fn open_bin<T, P>(path: P) -> T+where+ P: AsRef<Path>,+ T: DeserializeOwned + Default,+{+ let f = OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .open(path)+ .unwrap();++ let reader = BufReader::new(f);++ bincode::deserialize_from(reader).unwrap_or_default()+}++fn save_bin<T, P>(val: &T, path: P)+where+ P: AsRef<Path>,+ T: Serialize,+{+ let f = OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .open(path)+ .unwrap();++ bincode::serialize_into(f, val).unwrap();+}+pub struct Webgraph {pub path: String,- live_segment: Option<Segment<S>>,- segments: Vec<Segment<S>>,+ live_segment: LiveSegment,+ segments: Vec<StoredSegment>,executor: Arc<Executor>,- node2id: CachedStore<Node, NodeID>,- id2node: CachedStore<NodeID, Node>,+ node2id: BTreeMap<Node, NodeID>,+ id2node: BTreeMap<NodeID, Node>,meta: Meta,}-impl<S: Store> Webgraph<S> {+impl Webgraph {fn meta<P: AsRef<Path>>(path: P) -> Meta {let meta_path = path.as_ref().join("metadata.json");let mut reader = BufReader::new(@@ -316,7 +336,7 @@ impl<S: Store> Webgraph<S> {fs::create_dir_all(path.as_ref().join("segments")).unwrap();let mut segments = Vec::new();for segment in &meta.comitted_segments {- segments.push(Segment::open(+ segments.push(StoredSegment::open(path.as_ref().join("segments").join(segment),segment.clone(),));@@ -324,41 +344,11 @@ impl<S: Store> Webgraph<S> {Self {path: path.as_ref().as_os_str().to_str().unwrap().to_string(),- live_segment: None,+ live_segment: LiveSegment::default(),segments,executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),- node2id: CachedStore::new(RocksDbStore::open(path.as_ref().join("node2id")), 100_000),- id2node: CachedStore::new(RocksDbStore::open(path.as_ref().join("id2node")), 100_000),- meta,- }- }-- fn open_read_only<P: AsRef<Path>>(path: P) -> Self {- fs::create_dir_all(&path).unwrap();- let meta = Self::meta(&path);-- fs::create_dir_all(path.as_ref().join("segments")).unwrap();- let mut segments = Vec::new();- for segment in &meta.comitted_segments {- segments.push(Segment::open(- path.as_ref().join("segments").join(segment),- segment.clone(),- ));- }-- Self {- path: path.as_ref().as_os_str().to_str().unwrap().to_string(),- live_segment: None,- segments,- executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),- node2id: CachedStore::new(- RocksDbStore::open_read_only(path.as_ref().join("node2id")),- 100_000,- ),- id2node: CachedStore::new(- RocksDbStore::open_read_only(path.as_ref().join("id2node")),- 100_000,- ),+ node2id: open_bin(path.as_ref().join("node2id.bin")),+ id2node: open_bin(path.as_ref().join("id2node.bin")),meta,}}@@ -371,7 +361,7 @@ impl<S: Store> Webgraph<S> {fn id_or_assign(&mut self, node: &Node) -> NodeID {match self.node2id(node) {- Some(id) => id,+ Some(id) => *id,None => {let id = self.id_and_increment();@@ -390,30 +380,19 @@ impl<S: Store> Webgraph<S> {let (from, to) = (from.into_host(), to.into_host());let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));- match &mut self.live_segment {- Some(segment) => segment.insert(from_id, to_id, label),- None => {- let segment_id = Uuid::new_v4().to_string();- let path = Path::new(&self.path).join("segments").join(&segment_id);- let mut segment: Segment<S> = Segment::open(path, segment_id);-- segment.insert(from_id, to_id, label);-- self.live_segment = Some(segment);- }- }+ self.live_segment.insert(from_id, to_id, label);}- pub fn merge(&mut self, mut other: Webgraph<S>) {+ pub fn merge(&mut self, mut other: Webgraph) {other.commit();let mut mapping = Vec::new();for (node, other_id) in other.node2id.iter() {- match self.node2id(&node) {- Some(this_id) => mapping.push((other_id, this_id)),+ match self.node2id(node) {+ Some(this_id) => mapping.push((*other_id, *this_id)),None => {- let new_id = self.id_or_assign(&node);- mapping.push((other_id, new_id));+ let new_id = self.id_or_assign(node);+ mapping.push((*other_id, new_id));}}}@@ -425,30 +404,31 @@ impl<S: Store> Webgraph<S> {).expect("failed to merge webgraphs");- for mut segment in other.segments {- segment.flush();+ for segment in other.segments {let id = segment.id();let new_path = Path::new(&self.path).join("segments").join(segment.id());std::fs::rename(segment.path(), &new_path).unwrap();self.meta.comitted_segments.push(segment.id());drop(segment);- self.segments.push(Segment::open(new_path, id));+ self.segments.push(StoredSegment::open(new_path, id));}self.commit();}pub fn commit(&mut self) {- if let Some(mut segment) = self.live_segment.take() {- segment.flush();+ if !self.live_segment.is_empty() {+ let live_segment = std::mem::take(&mut self.live_segment);+ let segment = live_segment.commit(Path::new(&self.path).join("segments"));+self.meta.comitted_segments.push(segment.id());self.segments.push(segment);}self.save_metadata();- self.node2id.flush();- self.id2node.flush();+ save_bin(&self.node2id, Path::new(&self.path).join("node2id.bin"));+ save_bin(&self.id2node, Path::new(&self.path).join("id2node.bin"));if self.segments.len() > MAX_NUM_SEGMENTS {self.merge_segments((MAX_NUM_SEGMENTS / 2).max(1));@@ -457,12 +437,12 @@ impl<S: Store> Webgraph<S> {pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {if let Some(node_id) = self.node2id(&node) {- self.raw_ingoing_edges(&node_id)+ self.inner_ingoing_edges(node_id, true).into_iter().map(|edge| FullEdge {- from: self.id2node(&edge.from).unwrap(),- to: self.id2node(&edge.to).unwrap(),- label: edge.label,+ from: self.id2node(&edge.from).unwrap().clone(),+ to: self.id2node(&edge.to).unwrap().clone(),+ label: edge.label.loaded().unwrap(),}).collect()} else {@@ -471,8 +451,15 @@ impl<S: Store> Webgraph<S> {}pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {+ self.inner_ingoing_edges(node, false)+ }++ fn inner_ingoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {self.executor- .map(|segment| segment.ingoing_edges(node), self.segments.iter())+ .map(+ |segment| segment.ingoing_edges(node, load_labels),+ self.segments.iter(),+ ).unwrap().into_iter().flatten()@@ -480,8 +467,15 @@ impl<S: Store> Webgraph<S> {}pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {+ self.inner_outgoing_edges(node, false)+ }++ fn inner_outgoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {self.executor- .map(|segment| segment.outgoing_edges(node), self.segments.iter())+ .map(+ |segment| segment.outgoing_edges(node, load_labels),+ self.segments.iter(),+ ).unwrap().into_iter().flatten()@@ -490,12 +484,12 @@ impl<S: Store> Webgraph<S> {pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {if let Some(node_id) = self.node2id(&node) {- self.raw_outgoing_edges(&node_id)+ self.inner_outgoing_edges(node_id, true).into_iter().map(|edge| FullEdge {- from: self.id2node(&edge.from).unwrap(),- to: self.id2node(&edge.to).unwrap(),- label: edge.label,+ from: self.id2node(&edge.from).unwrap().clone(),+ to: self.id2node(&edge.to).unwrap().clone(),+ label: edge.label.loaded().unwrap(),}).collect()} else {@@ -503,23 +497,19 @@ impl<S: Store> Webgraph<S> {}}- pub fn node2id(&self, node: &Node) -> Option<NodeID> {- self.node2id- .get(node)- .map(|lock| lock.read().unwrap().0.into())+ pub fn node2id(&self, node: &Node) -> Option<&NodeID> {+ self.node2id.get(node)}- pub fn id2node(&self, id: &NodeID) -> Option<Node> {- self.id2node- .get(id)- .map(|lock| lock.read().unwrap().clone())+ pub fn id2node(&self, id: &NodeID) -> Option<&Node> {+ self.id2node.get(id)}- pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {- self.node2id.iter().map(|(_, id)| id)+ pub fn nodes(&self) -> impl Iterator<Item = &NodeID> + '_ {+ self.node2id.values()}- pub fn node_ids(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {+ pub fn node_ids(&self) -> impl Iterator<Item = (&Node, &NodeID)> + '_ {self.node2id.iter()}@@ -557,7 +547,7 @@ impl<S: Store> Webgraph<S> {.map(|mut candidate| {for other in candidate.merges {- candidate.segment.merge(other);+ candidate.segment = candidate.segment.merge(other);}candidate.segment@@ -699,19 +689,68 @@ mod test {#[test]fn merge() {- let mut graph1 = WebgraphBuilder::new_memory().open();+ let mut graphs = Vec::new();+ for (from, to, label) in &[+ (Node::from("A"), Node::from("B"), String::new()),+ (Node::from("B"), Node::from("C"), String::new()),+ (Node::from("C"), Node::from("D"), String::new()),+ (Node::from("D"), Node::from("E"), String::new()),+ (Node::from("E"), Node::from("F"), String::new()),+ (Node::from("F"), Node::from("G"), String::new()),+ (Node::from("G"), Node::from("H"), String::new()),+ ] {+ let mut graph = WebgraphBuilder::new_memory().open();+ graph.insert(from.clone(), to.clone(), label.clone());+ graph.commit();+ graphs.push(graph);+ }- graph1.insert(Node::from("A"), Node::from("B"), String::new());- graph1.commit();+ let mut graph = graphs.pop().unwrap();- let mut graph2 = WebgraphBuilder::new_memory().open();- graph2.insert(Node::from("B"), Node::from("C"), String::new());- graph2.commit();+ for other in graphs {+ graph.merge(other);+ }- graph1.merge(graph2);+ assert_eq!(+ graph.distances(Node::from("A")).get(&Node::from("H")),+ Some(&7)+ );+ graph.merge_segments(1);assert_eq!(- graph1.distances(Node::from("A")).get(&Node::from("C")),+ graph.distances(Node::from("A")).get(&Node::from("H")),+ Some(&7)+ )+ }+ #[test]+ fn merge_cycle() {+ let mut graphs = Vec::new();+ for (from, to, label) in &[+ (Node::from("A"), Node::from("B"), String::new()),+ (Node::from("B"), Node::from("B"), String::new()),+ (Node::from("B"), Node::from("C"), String::new()),+ (Node::from("C"), Node::from("A"), String::new()),+ ] {+ let mut graph = WebgraphBuilder::new_memory().open();+ graph.insert(from.clone(), to.clone(), label.clone());+ graph.commit();+ graphs.push(graph);+ }++ let mut graph = graphs.pop().unwrap();++ for other in graphs {+ graph.merge(other);+ }++ assert_eq!(+ graph.distances(Node::from("A")).get(&Node::from("C")),+ Some(&2)+ );++ graph.merge_segments(1);+ assert_eq!(+ graph.distances(Node::from("A")).get(&Node::from("C")),Some(&2))}@@ -721,6 +760,7 @@ mod test {let graph = test_graph();let path = graph.path.clone();+ dbg!(&path);let frozen: FrozenWebgraph = graph.into();let bytes = bincode::serialize(&frozen).unwrap();@@ -728,6 +768,7 @@ mod test {let deserialized_frozen: FrozenWebgraph = bincode::deserialize(&bytes).unwrap();let graph: Webgraph = deserialized_frozen.into();+ dbg!(&graph.path);let distances = graph.distances(Node::from("D"));@@ -840,12 +881,6 @@ mod test {graph.merge_segments(2);assert_eq!(graph.segments.len(), 2);- let distances = graph.distances(Node::from("D"));-- assert_eq!(distances.get(&Node::from("C")), Some(&1));- assert_eq!(distances.get(&Node::from("A")), Some(&2));- assert_eq!(distances.get(&Node::from("B")), Some(&3));-let mut res = graph.outgoing_edges(Node::from("A"));res.sort_by(|a, b| a.to.name.cmp(&b.to.name));@@ -888,5 +923,22 @@ mod test {},]);++ let res = graph.outgoing_edges(Node::from("D"));++ assert_eq!(+ res,+ vec![FullEdge {+ from: Node::from("D"),+ to: Node::from("C"),+ label: String::new()+ },]+ );++ let distances = graph.distances(Node::from("D"));++ assert_eq!(distances.get(&Node::from("C")), Some(&1));+ assert_eq!(distances.get(&Node::from("A")), Some(&2));+ assert_eq!(distances.get(&Node::from("B")), Some(&3));}}diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 92c49930..240d3aab 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -14,19 +14,31 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{de::DeserializeOwned, Deserialize, Serialize};+use rkyv::{Archive, Deserialize};use std::{collections::{BTreeMap, HashSet},fmt::Debug,- marker::PhantomData,path::Path,- sync::{Arc, RwLock},};-use super::{Edge, NodeID, Store, StoredEdge};-use crate::kv::{rocksdb_store::RocksDbStore, Kv};--#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]+use super::{open_bin, save_bin, store::Store, Edge, Loaded, NodeID, StoredEdge};++#[derive(+ Debug,+ Clone,+ Copy,+ serde::Serialize,+ serde::Deserialize,+ Archive,+ rkyv::Serialize,+ rkyv::Deserialize,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Hash,+)]+#[archive_attr(derive(Eq, Hash, PartialEq, Debug))]pub struct SegmentNodeID(u64);impl From<u64> for SegmentNodeID {@@ -35,336 +47,422 @@ impl From<u64> for SegmentNodeID {}}-pub(crate) struct Adjacency {- pub(crate) tree: CachedStore<SegmentNodeID, HashSet<StoredEdge>>,+pub struct StoredSegment {+ adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,+ reversed_adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,+ id_mapping: BTreeMap<NodeID, SegmentNodeID>,+ rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,+ meta: Meta,+ id: String,+ folder_path: String,}-impl Adjacency {- fn insert(&mut self, from: SegmentNodeID, to: SegmentNodeID, label: String) {- let edge = StoredEdge { other: to, label };-- if let Some(existing) = self.tree.get(&from) {- existing.write().unwrap().insert(edge);- } else {- let mut set = HashSet::new();- set.insert(edge);- self.tree.insert(from, set);+impl StoredSegment {+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {+ StoredSegment {+ adjacency: Store::open(folder_path.as_ref().join("adjacency"), &id).unwrap(),+ reversed_adjacency: Store::open(folder_path.as_ref().join("reversed_adjacency"), &id)+ .unwrap(),+ id_mapping: open_bin(folder_path.as_ref().join("id_mapping.bin")),+ rev_id_mapping: open_bin(folder_path.as_ref().join("rev_id_mapping.bin")),+ meta: open_bin(folder_path.as_ref().join("meta.bin")),+ folder_path: folder_path+ .as_ref()+ .as_os_str()+ .to_str()+ .unwrap()+ .to_string(),+ id,}}- fn edges(&self, node: &SegmentNodeID) -> Vec<StoredEdge> {- self.tree- .get(node)- .map(|r| r.read().unwrap().clone())- .unwrap_or_default()- .into_iter()- .collect()+ pub fn num_nodes(&self) -> usize {+ self.meta.num_nodes as usize}-}--pub struct CachedStore<K, V>-where- K: Ord + Eq + Serialize + DeserializeOwned + Clone,- V: Serialize + DeserializeOwned + Clone,-{- store: Box<dyn Kv<K, V> + Send + Sync>,- cache: RwLock<BTreeMap<K, Arc<RwLock<V>>>>,- cache_size: usize,-}-impl<K, V> CachedStore<K, V>-where- K: Ord + Eq + Serialize + DeserializeOwned + Clone,- V: Serialize + DeserializeOwned + Clone + Debug,-{- pub fn new(store: Box<dyn Kv<K, V> + Send + Sync>, cache_size: usize) -> Self {- Self {- store,- cache_size,- cache: RwLock::new(BTreeMap::new()),- }+ fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {+ self.id_mapping.get(node).copied()}- fn update_cache(&self, key: &K) {- if !self.cache.read().unwrap().contains_key(key) {- let mut cache = self.cache.write().unwrap();-- while cache.len() >= self.cache_size {- cache.pop_first();- }-- let val = self.store.get(key);+ fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {+ self.rev_id_mapping.get(node).copied()+ }++ pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {+ self.id_mapping(node)+ .and_then(|segment_id| {+ self.adjacency.get(&segment_id).map(|edges| {+ edges+ .iter()+ .map(move |edge| {+ if load_label {+ Edge {+ from: *node,+ to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),+ label: Loaded::Some(+ edge.label.deserialize(&mut rkyv::Infallible).unwrap(),+ ),+ }+ } else {+ Edge {+ from: *node,+ to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),+ label: Loaded::NotYet,+ }+ }+ })+ .collect()+ })+ })+ .unwrap_or_default()+ }- if let Some(val) = val {- cache.insert(key.clone(), Arc::new(RwLock::new(val)));- }- }+ pub fn ingoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {+ self.id_mapping(node)+ .and_then(|segment_id| {+ self.reversed_adjacency.get(&segment_id).map(|edges| {+ edges+ .iter()+ .map(move |edge| {+ if load_label {+ Edge {+ from: self+ .rev_id_mapping(&SegmentNodeID(edge.other.0))+ .unwrap(),+ to: *node,+ label: Loaded::Some(+ edge.label.deserialize(&mut rkyv::Infallible).unwrap(),+ ),+ }+ } else {+ Edge {+ from: self+ .rev_id_mapping(&SegmentNodeID(edge.other.0))+ .unwrap(),+ to: *node,+ label: Loaded::NotYet,+ }+ }+ })+ .collect()+ })+ })+ .unwrap_or_default()}- pub fn get(&self, key: &K) -> Option<Arc<RwLock<V>>> {- self.update_cache(key);- let guard = self.cache.read().unwrap();- guard.get(key).cloned()+ fn flush(&mut self) {+ self.adjacency.flush().unwrap();+ self.reversed_adjacency.flush().unwrap();+ save_bin(&self.id_mapping, self.path().join("id_mapping.bin"));+ save_bin(&self.rev_id_mapping, self.path().join("rev_id_mapping.bin"));+ save_bin(&self.meta, self.path().join("meta.bin"));}- pub fn insert(&mut self, key: K, value: V) {- let cache = self.cache.get_mut().unwrap();+ pub fn id(&self) -> String {+ self.id.clone()+ }- cache.insert(key, Arc::new(RwLock::new(value)));+ pub fn path(&self) -> &Path {+ Path::new(&self.folder_path)+ }- if cache.len() >= self.cache_size {- while cache.len() > self.cache_size / 2 {- if let Some((key, value)) = cache.pop_first() {- self.store.insert(key, value.write().unwrap().clone());+ pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {+ self.adjacency.iter().flat_map(move |(node_id, edges)| {+ let from = self.rev_id_mapping(node_id).unwrap();++ edges.iter().map(move |stored_edge| Edge {+ from,+ to: self+ .rev_id_mapping(&SegmentNodeID(stored_edge.other.0))+ .unwrap(),+ label: Loaded::NotYet,+ })+ })+ }++ #[allow(clippy::too_many_arguments)]+ fn merge_adjacency<F1, F2>(+ &self,+ self_edges_fn: F1,+ other: &Self,+ other_edges_fn: F2,+ next_segment_id: &mut SegmentNodeID,+ new_id_mapping: &mut BTreeMap<NodeID, SegmentNodeID>,+ new_rev_id_mapping: &mut BTreeMap<SegmentNodeID, NodeID>,+ new_adjacency: &mut Store<SegmentNodeID, HashSet<StoredEdge>>,+ ) where+ F1: Fn(&Self, &SegmentNodeID) -> HashSet<StoredEdge>,+ F2: Fn(&Self, &SegmentNodeID) -> HashSet<StoredEdge>,+ {+ for (node_id, segment_id) in &self.id_mapping {+ let mut edges = self_edges_fn(self, segment_id);++ if let Some(other_segment_id) = other.id_mapping.get(node_id) {+ let other_edges = other_edges_fn(other, other_segment_id);++ for other_edge in other_edges {+ let node = other.rev_id_mapping(&other_edge.other).unwrap();++ if let Some(segment_id) = new_id_mapping.get(&node).copied() {+ edges.insert(StoredEdge {+ other: segment_id,+ label: other_edge.label,+ });+ } else {+ let r = new_id_mapping.insert(node, *next_segment_id);+ debug_assert!(r.is_none());+ let r = new_rev_id_mapping.insert(*next_segment_id, node);+ debug_assert!(r.is_none());++ edges.insert(StoredEdge {+ other: *next_segment_id,+ label: other_edge.label,+ });++ *next_segment_id = SegmentNodeID(next_segment_id.0 + 1);+ }}}- self.store.flush();+ if !edges.is_empty() {+ let segment_id = new_id_mapping.get(node_id).copied().unwrap();+ println!("{:?} -> {:?}", node_id, segment_id);+ new_adjacency.insert(segment_id, &edges).unwrap();+ }}- }- pub fn flush(&mut self) {- let cache = self.cache.get_mut().unwrap();-- while let Some((key, value)) = cache.pop_first() {- self.store.insert(key, value.write().unwrap().clone());+ for (node_id, segment_id) in &other.id_mapping {+ if self.id_mapping.get(node_id).is_none() {+ let this_segment_id = if let Some(segment_id) = new_id_mapping.get(node_id) {+ *segment_id+ } else {+ let r = new_id_mapping.insert(*node_id, *next_segment_id);+ debug_assert!(r.is_none());++ let r = new_rev_id_mapping.insert(*next_segment_id, *node_id);+ debug_assert!(r.is_none());+ let this = *next_segment_id;++ *next_segment_id = SegmentNodeID(next_segment_id.0 + 1);++ this+ };++ let edges = other_edges_fn(other, segment_id);++ let edges: HashSet<_> = edges+ .into_iter()+ .map(|edge| {+ let other_node = other.rev_id_mapping(&edge.other).unwrap();+ if let Some(segment_id) = new_id_mapping.get(&other_node) {+ StoredEdge {+ other: *segment_id,+ label: edge.label,+ }+ } else {+ let new_segment_id = *next_segment_id;+ *next_segment_id = SegmentNodeID(next_segment_id.0 + 1);++ new_id_mapping.insert(other_node, new_segment_id);+ new_rev_id_mapping.insert(new_segment_id, other_node);++ StoredEdge {+ other: new_segment_id,+ label: edge.label,+ }+ }+ })+ .collect();++ if !edges.is_empty() {+ new_adjacency.insert(this_segment_id, &edges).unwrap();+ }+ }}-- self.store.flush();}- pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {- self.store.iter()- }+ pub fn merge(self, other: StoredSegment) -> Self {+ let new_segment_id = uuid::Uuid::new_v4().to_string();- pub fn delete(&mut self, key: &K) {- self.store.delete(key);- self.cache.write().unwrap().remove(key);- }-}--pub struct Segment<S: Store> {- adjacency: Adjacency,- reversed_adjacency: Adjacency,- id_mapping: CachedStore<NodeID, SegmentNodeID>,- rev_id_mapping: CachedStore<SegmentNodeID, NodeID>,- meta: CachedStore<String, u64>,- store: PhantomData<S>,- id: String,- path: String,-}+ let mut new_adjacency = Store::open(+ self.path().join(&new_segment_id).join("adjacency"),+ &new_segment_id,+ )+ .unwrap();-impl<S: Store> Segment<S> {- #[cfg(test)]- pub(crate) fn temporary() -> Segment<S> {- S::temporary()- }+ let mut next_segment_id = SegmentNodeID(self.meta.num_nodes);- pub fn open<P: AsRef<Path>>(path: P, id: String) -> Self {- S::open(path, id)- }+ let mut new_id_mapping = self.id_mapping.clone();+ let mut new_rev_id_mapping = self.rev_id_mapping.clone();- pub fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Self {- S::open_read_only(path, id)- }+ self.merge_adjacency(+ |_self: &StoredSegment, segment_id: &SegmentNodeID| {+ _self+ .adjacency+ .get(segment_id)+ .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())+ .unwrap_or_default()+ },+ &other,+ |_other: &StoredSegment, segment_id: &SegmentNodeID| {+ _other+ .adjacency+ .get(segment_id)+ .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())+ .unwrap_or_default()+ },+ &mut next_segment_id,+ &mut new_id_mapping,+ &mut new_rev_id_mapping,+ &mut new_adjacency,+ );- fn next_id(&self) -> SegmentNodeID {- let key = &"next_id".to_string();- self.meta- .get(key)- .map(|arc| *arc.read().unwrap())- .unwrap_or(0)- .into()- }+ let mut new_reversed_adjacency = Store::open(+ self.path().join(&new_segment_id).join("reversed_adjacency"),+ &new_segment_id,+ )+ .unwrap();++ self.merge_adjacency(+ |_self: &StoredSegment, segment_id: &SegmentNodeID| {+ _self+ .reversed_adjacency+ .get(segment_id)+ .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())+ .unwrap_or_default()+ },+ &other,+ |_other: &StoredSegment, segment_id: &SegmentNodeID| {+ _other+ .reversed_adjacency+ .get(segment_id)+ .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())+ .unwrap_or_default()+ },+ &mut next_segment_id,+ &mut new_id_mapping,+ &mut new_rev_id_mapping,+ &mut new_reversed_adjacency,+ );- pub fn num_nodes(&self) -> usize {- self.next_id().0 as usize- }+ let mut res = Self {+ adjacency: new_adjacency,+ reversed_adjacency: new_reversed_adjacency,+ id_mapping: new_id_mapping,+ rev_id_mapping: new_rev_id_mapping,+ meta: Meta {+ num_nodes: next_segment_id.0,+ },+ id: new_segment_id,+ folder_path: self.folder_path,+ };- fn increment_next_id(&mut self) {- let current_id = self.next_id().0;- let next_id = current_id + 1;- self.meta.insert("next_id".to_string(), next_id);- }+ res.flush();- fn id_and_increment(&mut self) -> SegmentNodeID {- let id = self.next_id();- self.increment_next_id();- id+ res}- fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {- self.id_mapping.get(node).map(|lock| *lock.read().unwrap())- }+ pub fn update_id_mapping(&mut self, mapping: Vec<(NodeID, NodeID)>) {+ let mut new_mappings = Vec::with_capacity(mapping.len());- fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {- self.rev_id_mapping- .get(node)- .map(|lock| *lock.read().unwrap())- }+ for (old_id, new_id) in mapping {+ if let Some(segment_id) = self.id_mapping(&old_id) {+ self.id_mapping.remove(&old_id);+ self.rev_id_mapping.remove(&segment_id);- pub fn outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {- match self.id_mapping(node) {- Some(segment_id) => self- .adjacency- .edges(&segment_id)- .into_iter()- .map(|edge| Edge {- from: *node,- to: self.rev_id_mapping(&edge.other).unwrap(),- label: edge.label,- })- .collect(),- None => Vec::new(),+ new_mappings.push((segment_id, new_id));+ }}- }- pub fn ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {- match self.id_mapping(node) {- Some(segment_id) => self- .reversed_adjacency- .edges(&segment_id)- .into_iter()- .map(|edge| Edge {- from: self.rev_id_mapping(&edge.other).unwrap(),- to: *node,- label: edge.label,- })- .collect(),- None => Vec::new(),+ for (segment_id, new_id) in new_mappings {+ self.id_mapping.insert(new_id, segment_id);+ self.rev_id_mapping.insert(segment_id, new_id);}++ self.flush();}+}- fn id_or_assign(&mut self, node: &NodeID) -> SegmentNodeID {- match self.id_mapping(node) {- Some(id) => id,- None => {- let id = self.id_and_increment();+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]+struct Meta {+ num_nodes: u64,+}- self.id_mapping.insert(*node, id);- self.rev_id_mapping.insert(id, *node);+#[derive(Default)]+pub struct LiveSegment {+ adjacency: BTreeMap<SegmentNodeID, HashSet<StoredEdge>>,+ reversed_adjacency: BTreeMap<SegmentNodeID, HashSet<StoredEdge>>,+ id_mapping: BTreeMap<NodeID, SegmentNodeID>,+ rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,+ next_id: u64,+}- id- }- }- }+impl LiveSegment {+ fn get_or_create_id(&mut self, id: NodeID) -> SegmentNodeID {+ if let Some(segment_id) = self.id_mapping.get(&id) {+ *segment_id+ } else {+ let segment_id = SegmentNodeID(self.next_id);+ self.next_id += 1;- pub fn insert(&mut self, from: NodeID, to: NodeID, label: String) {- let from_id = self.id_or_assign(&from);- let to_id = self.id_or_assign(&to);+ self.id_mapping.insert(id, segment_id);+ self.rev_id_mapping.insert(segment_id, id);- self.adjacency.insert(from_id, to_id, label.clone());- self.reversed_adjacency.insert(to_id, from_id, label);+ segment_id+ }}- pub fn flush(&mut self) {- self.adjacency.tree.flush();- self.reversed_adjacency.tree.flush();- self.id_mapping.flush();- self.rev_id_mapping.flush();- self.meta.flush();- }+ pub fn insert(&mut self, from: NodeID, to: NodeID, label: String) {+ let from = self.get_or_create_id(from);+ let to = self.get_or_create_id(to);- pub fn id(&self) -> String {- self.id.clone()- }+ self.adjacency.entry(from).or_default().insert(StoredEdge {+ other: to,+ label: label.clone(),+ });- pub fn path(&self) -> &Path {- Path::new(&self.path)+ self.reversed_adjacency+ .entry(to)+ .or_default()+ .insert(StoredEdge { other: from, label });}- pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {- self.adjacency- .tree- .iter()- .flat_map(move |(node_id, edges)| {- let from = self.rev_id_mapping(&node_id).unwrap();-- edges.into_iter().map(move |stored_edge| Edge {- from,- to: self.rev_id_mapping(&stored_edge.other).unwrap(),- label: stored_edge.label,- })- })- }+ pub fn commit<P: AsRef<Path>>(self, folder_path: P) -> StoredSegment {+ let segment_id = uuid::Uuid::new_v4().to_string();+ let path = folder_path.as_ref().join(&segment_id);- pub fn merge(&mut self, mut other: Segment<S>) {- other.flush();+ let mut adjacency = Store::open(path.join("adjacency"), &segment_id).unwrap();- for edge in other.edges() {- self.insert(edge.from, edge.to, edge.label);+ for (from, edges) in self.adjacency {+ adjacency.insert(from, &edges).unwrap();}- self.flush();- }-- pub fn update_id_mapping(&mut self, mapping: Vec<(NodeID, NodeID)>) {- let mut new_mappings = Vec::with_capacity(mapping.len());+ let mut reversed_adjacency =+ Store::open(path.join("reversed_adjacency"), &segment_id).unwrap();- for (old_id, new_id) in mapping {- if let Some(segment_id) = self.id_mapping(&old_id) {- self.id_mapping.delete(&old_id);- self.rev_id_mapping.delete(&segment_id);-- new_mappings.push((segment_id, new_id));- }+ for (to, edges) in self.reversed_adjacency {+ reversed_adjacency.insert(to, &edges).unwrap();}- for (segment_id, new_id) in new_mappings {- self.id_mapping.insert(new_id, segment_id);- self.rev_id_mapping.insert(segment_id, new_id);- }+ let mut stored_segment = StoredSegment {+ adjacency,+ reversed_adjacency,+ id_mapping: self.id_mapping,+ rev_id_mapping: self.rev_id_mapping,+ meta: Meta {+ num_nodes: self.next_id,+ },+ folder_path: path.as_os_str().to_str().unwrap().to_string(),+ id: segment_id,+ };- self.flush();- }-}+ stored_segment.flush();-impl Store for RocksDbStore {- fn open<P: AsRef<std::path::Path>>(path: P, id: String) -> Segment<Self> {- let adjacency = RocksDbStore::open(path.as_ref().join("adjacency"));- let reversed_adjacency = RocksDbStore::open(path.as_ref().join("reversed_adjacency"));- let id_mapping = RocksDbStore::open(path.as_ref().join("id_mapping"));- let rev_id_mapping = RocksDbStore::open(path.as_ref().join("rev_id_mapping"));- let meta = RocksDbStore::open(path.as_ref().join("meta"));-- Segment {- adjacency: Adjacency {- tree: CachedStore::new(adjacency, 100),- },- reversed_adjacency: Adjacency {- tree: CachedStore::new(reversed_adjacency, 100),- },- id_mapping: CachedStore::new(id_mapping, 100),- rev_id_mapping: CachedStore::new(rev_id_mapping, 100),- meta: CachedStore::new(meta, 1_000),- store: Default::default(),- path: path.as_ref().as_os_str().to_str().unwrap().to_string(),- id,- }+ stored_segment}- fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Segment<Self> {- let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));- let reversed_adjacency =- RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));- let id_mapping = RocksDbStore::open_read_only(path.as_ref().join("id_mapping"));- let rev_id_mapping = RocksDbStore::open_read_only(path.as_ref().join("rev_id_mapping"));- let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));-- Segment {- adjacency: Adjacency {- tree: CachedStore::new(adjacency, 100),- },- reversed_adjacency: Adjacency {- tree: CachedStore::new(reversed_adjacency, 100),- },- id_mapping: CachedStore::new(id_mapping, 100),- rev_id_mapping: CachedStore::new(rev_id_mapping, 100),- meta: CachedStore::new(meta, 1_000),- store: Default::default(),- path: path.as_ref().as_os_str().to_str().unwrap().to_string(),- id,- }+ pub fn is_empty(&self) -> bool {+ self.id_mapping.is_empty()}}@@ -381,7 +479,7 @@ mod test {// ▼ │ │// 1─────►2◄┘- let mut store: Segment<RocksDbStore> = Segment::temporary();+ let mut store = LiveSegment::default();let a = NodeID(0);let b = NodeID(1);@@ -392,10 +490,11 @@ mod test {store.insert(c, a, String::new());store.insert(a, c, String::new());- store.flush();+ let store: StoredSegment = store.commit(crate::gen_temp_path());++ let mut out: Vec<Edge> = store.outgoing_edges(&a, false);- let mut out = store.outgoing_edges(&a);- out.sort();+ out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,@@ -403,61 +502,74 @@ mod test {Edge {from: a,to: b,- label: String::new()+ label: Loaded::NotYet},Edge {from: a,to: c,- label: String::new()+ label: Loaded::NotYet},]);- let mut out = store.outgoing_edges(&b);- out.sort();+ let mut out: Vec<Edge> = store.outgoing_edges(&b, false);+ out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,vec![Edge {from: b,to: c,- label: String::new()+ label: Loaded::NotYet},]);- let mut out = store.ingoing_edges(&c);- out.sort();+ let mut out: Vec<Edge> = store.outgoing_edges(&c, false);+ out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,- vec![- Edge {- from: a,- to: c,- label: String::new()- },- Edge {- from: b,- to: c,- label: String::new()- },- ]+ vec![Edge {+ from: c,+ to: a,+ label: Loaded::NotYet+ },]);+ let out: Vec<Edge> = store.ingoing_edges(&a, false);assert_eq!(- store.ingoing_edges(&a),+ out,vec![Edge {from: c,to: a,- label: String::new()+ label: Loaded::NotYet},]);+ let out: Vec<Edge> = store.ingoing_edges(&b, false);assert_eq!(- store.ingoing_edges(&b),+ out,vec![Edge {from: a,to: b,- label: String::new()+ label: Loaded::NotYet},]);++ let mut out: Vec<Edge> = store.ingoing_edges(&c, false);+ out.sort_by(|a, b| a.from.cmp(&b.from));+ assert_eq!(+ out,+ vec![+ Edge {+ from: a,+ to: c,+ label: Loaded::NotYet+ },+ Edge {+ from: b,+ to: c,+ label: Loaded::NotYet+ },+ ]+ );}}diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsnew file mode 100644index 00000000..88b8a158--- /dev/null+++ b/core/src/webgraph/store.rs@@ -0,0 +1,295 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ collections::BTreeMap,+ fs::OpenOptions,+ io::BufReader,+ ops::Range,+ path::{Path, PathBuf},+};++use rkyv::{ser::serializers::AllocSerializer, Archived};++type Result<T> = std::result::Result<T, Error>;++const SCRATCH_SIZE: usize = 4096;+const LOCATIONS_NAME: &str = "locations.bin";+const MMAP_NAME: &str = "data.bin";++#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("key already exists")]+ KeyExists,++ #[error("io error: {0}")]+ Io(std::io::Error),++ #[error("bincode error: {0}")]+ Bincode(bincode::Error),+}++pub struct Store<K, V> {+ locations: BTreeMap<K, Range<usize>>,+ folder_path: PathBuf,+ cur_end_range: usize,+ map: memmap2::MmapMut,+ has_new_inserts: bool,+ segment_name: String,+ _values: std::marker::PhantomData<V>,+}++impl<K, V> Store<K, V>+where+ K: serde::de::DeserializeOwned + serde::Serialize + Ord,+ V: rkyv::Archive,+{+ pub fn open<P: AsRef<Path>>(folder_path: P, segment_name: &str) -> Result<Self> {+ if folder_path.as_ref().is_file() {+ return Err(Error::Io(std::io::Error::new(+ std::io::ErrorKind::NotFound,+ format!("{} is not a directory", folder_path.as_ref().display()),+ )));+ }++ std::fs::create_dir_all(folder_path.as_ref()).map_err(Error::Io)?;++ let locations_path = folder_path+ .as_ref()+ .join(format!("{segment_name}-{LOCATIONS_NAME}"));+ let f = OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .open(locations_path)+ .map_err(Error::Io)?;++ let mut reader = BufReader::new(f);+ let locations: BTreeMap<K, Range<usize>> =+ bincode::deserialize_from(&mut reader).unwrap_or_default();++ let cur_end_range = locations.values().map(|range| range.end).max().unwrap_or(0);++ let mmap_path = folder_path+ .as_ref()+ .join(format!("{segment_name}-{MMAP_NAME}"));+ let map = unsafe {+ let file = OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .open(mmap_path)+ .map_err(Error::Io)?;++ memmap2::MmapOptions::new()+ .map_mut(&file)+ .map_err(Error::Io)?+ };++ Ok(Self {+ locations,+ cur_end_range,+ map,+ folder_path: folder_path.as_ref().to_path_buf(),+ has_new_inserts: false,+ _values: std::marker::PhantomData,+ segment_name: segment_name.to_string(),+ })+ }++ pub fn get(&self, key: &K) -> Option<&Archived<V>> {+ let range = self.locations.get(key)?;+ let bytes = &self.map[range.clone()];++ let archived = unsafe { rkyv::archived_root::<V>(bytes) };+ Some(archived)+ }++ pub fn iter(&self) -> impl Iterator<Item = (&K, &Archived<V>)> + '_ {+ self.locations.iter().map(move |(key, range)| {+ let bytes = &self.map[range.clone()];+ let archived = unsafe { rkyv::archived_root::<V>(bytes) };+ (key, archived)+ })+ }++ pub fn insert(&mut self, key: K, value: &V) -> Result<()>+ where+ V: rkyv::Serialize<AllocSerializer<SCRATCH_SIZE>>,+ {+ if self.locations.contains_key(&key) {+ return Err(Error::KeyExists);+ }++ let bytes = rkyv::to_bytes::<_, SCRATCH_SIZE>(value).unwrap();+ self.insert_raw(key, &bytes)+ }++ fn insert_raw(&mut self, key: K, bytes: &[u8]) -> Result<()> {+ let old_end = self.cur_end_range;+ let new_end = bytes.len() + self.cur_end_range;++ if new_end > self.map.len() {+ self.resize(new_end)?;+ }++ self.map[old_end..new_end].copy_from_slice(bytes);+ self.locations.insert(key, old_end..new_end);+ self.cur_end_range = new_end;+ self.has_new_inserts = true;++ Ok(())+ }++ fn resize(&mut self, size: usize) -> Result<()> {+ if size > self.map.len() {+ self.flush_map()?;++ let file = OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .open(+ self.folder_path+ .join(format!("{}-{MMAP_NAME}", self.segment_name)),+ )+ .map_err(Error::Io)?;++ file.set_len(size as u64).map_err(Error::Io)?;++ self.map = unsafe {+ memmap2::MmapOptions::new()+ .map_mut(&file)+ .map_err(Error::Io)?+ };+ }++ Ok(())+ }++ fn flush_map(&mut self) -> Result<()> {+ if self.has_new_inserts {+ self.map.flush().map_err(Error::Io)?;+ }++ Ok(())+ }++ pub fn flush(&mut self) -> Result<()> {+ if self.has_new_inserts {+ self.flush_map()?;++ let f = OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .open(+ self.folder_path+ .join(format!("{}-{}", self.segment_name, LOCATIONS_NAME)),+ )+ .map_err(Error::Io)?;++ bincode::serialize_into(f, &self.locations).map_err(Error::Bincode)?;+ }+ self.has_new_inserts = false;++ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use rkyv::Archive;++ use super::*;++ #[derive(Debug, PartialEq, Eq, Clone, Archive, rkyv::Serialize, rkyv::Deserialize)]+ #[archive(compare(PartialEq))]+ #[archive_attr(derive(Debug))]+ struct TestStruct {+ a: String,+ b: i32,+ }++ #[test]+ fn test_insert() {+ let mut kv =+ Store::<String, TestStruct>::open(crate::gen_temp_path(), "test-segment").unwrap();++ assert!(kv.get(&"test".to_string()).is_none());++ let test_struct = TestStruct {+ a: "test".to_string(),+ b: 5,+ };++ kv.insert("test".to_string(), &test_struct).unwrap();+ kv.flush().unwrap();++ let archived = kv.get(&"test".to_string()).unwrap();+ assert_eq!(&test_struct, archived);+ }++ #[test]+ fn test_re_open() {+ let path = crate::gen_temp_path();+ let segment_name = "test-segment";+ let test_struct = TestStruct {+ a: "test".to_string(),+ b: 5,+ };++ {+ let mut kv = Store::<String, TestStruct>::open(path.clone(), segment_name).unwrap();++ assert!(kv.get(&"test".to_string()).is_none());++ kv.insert("test".to_string(), &test_struct).unwrap();+ kv.flush().unwrap();+ }++ let kv = Store::<String, TestStruct>::open(path, segment_name).unwrap();++ let archived = kv.get(&"test".to_string()).unwrap();+ assert_eq!(&test_struct, archived);+ }++ #[test]+ fn test_insert_existing_key() {+ let mut kv =+ Store::<String, TestStruct>::open(crate::gen_temp_path(), "test-segment").unwrap();++ assert!(kv.get(&"test".to_string()).is_none());++ let test_struct = TestStruct {+ a: "test".to_string(),+ b: 5,+ };++ kv.insert("test".to_string(), &test_struct).unwrap();+ kv.flush().unwrap();++ let archived = kv.get(&"test".to_string()).unwrap();+ assert_eq!(&test_struct, archived);++ let test_struct = TestStruct {+ a: "test".to_string(),+ b: 6,+ };++ assert!(kv.insert("test".to_string(), &test_struct).is_err());+ }+}diff --git a/ltr/generate_data.py b/ltr/generate_data.pyindex df84b831..593ba072 100644--- a/ltr/generate_data.py+++ b/ltr/generate_data.py@@ -14,6 +14,38 @@ TOP_N_RESULTS = 10openai.api_key = os.getenv("OPENAI_API_KEY")+"""+NEW PROMPT:++You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a reputable source. The score should be between 0.0 and 1.0.++query: england politics++result: {domain: "https://abingdonrowcondos.com", title: "Gareth Southgate hails hat-trick hero Harry Kane’s return to form", snippet: "Plus, Nesbitt Realty will pay a down-payment assistance of $1,903 toward your funds for your down-payment. Please get in touch with Will Nesbitt for learn more about our first-time buyer's credit or 607 Bashford Ln #1 or any ."}++score: 0.7++query: q learning algorithm++result: {domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange", snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently"}++score: 1.0++query: learn python++result: {domain: "jqxxj.fulltime-abbigliamento.it", title: "Vim plugins python", snippet: "In fact, VimL can be learned fast, but using. omplete Python syntax highlighter for Vim. shows what I get with default Vim that illustrates … Upgrade to PyCharm, the leading Python IDE: best in class debugging, code navigation, refactoring."}++score: 0.1++query: savings interest formula++result: {domain: "readingrockets.org", title: "Choosing Words to Teach | Reading Rockets", snippet: "15. Oct. 2020 Only a sobering revelation (that essential detail) saves him, his marriage, and his wife. A couple of points should be emphasized here. The words were selected not so much because they are essential to comprehension of the story but because they seem..."}++score:++"""++def search(q):url = f"http://{STRACT_ENDPOINT}/beta/api/search"r = requests.post(diff --git a/optics/testcases b/optics/testcasesindex afa95e5d..d80320e3 160000--- a/optics/testcases+++ b/optics/testcases@@ -1 +1 @@-Subproject commit afa95e5d018e7c4b53c48bdc02cf7712de6d21ba+Subproject commit d80320e33b78ddb233cfec56af498f0703d34b12