Skip to content

web search done right

memory mapped graph store

Mikkel Denker0cc4cac100d6
diff --git a/Cargo.lock b/Cargo.lock
index 94a74300..70132f79 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -391,6 +391,28 @@ version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
+[[package]]
+name = "bytecheck"
+version = "0.6.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13fe11640a23eb24562225322cd3e452b93a3d4091d62fab69c70542fcd17d1f"
+dependencies = [
+ "bytecheck_derive",
+ "ptr_meta",
+ "simdutf8",
+]
+
+[[package]]
+name = "bytecheck_derive"
+version = "0.6.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e31225543cb46f81a7e224762764f4a6a0f097b1db0b175f69e8065efaa42de5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "bytemuck"
version = "1.13.0"
@@ -2048,9 +2070,9 @@ dependencies = [
[[package]]
name = "memmap2"
-version = "0.5.8"
+version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc"
+checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327"
dependencies = [
"libc",
]
@@ -2706,6 +2728,26 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "ptr_meta"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
+dependencies = [
+ "ptr_meta_derive",
+]
+
+[[package]]
+name = "ptr_meta_derive"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "quick-xml"
version = "0.23.1"
@@ -2905,6 +2947,15 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "rend"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "581008d2099240d37fb08d77ad713bcaec2c4d89d50b5b21a8bb1996bbab68ab"
+dependencies = [
+ "bytecheck",
+]
+
[[package]]
name = "reqwest"
version = "0.11.14"
@@ -2959,6 +3010,31 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "rkyv"
+version = "0.7.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c30f1d45d9aa61cbc8cd1eb87705470892289bb2d01943e7803b873a57404dc3"
+dependencies = [
+ "bytecheck",
+ "hashbrown",
+ "ptr_meta",
+ "rend",
+ "rkyv_derive",
+ "seahash",
+]
+
+[[package]]
+name = "rkyv_derive"
+version = "0.7.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff26ed6c7c4dfc2aa9480b86a60e3c7233543a270a680e10758a507c5a4ce476"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "rocksdb"
version = "0.19.0"
@@ -3133,6 +3209,12 @@ dependencies = [
"syn",
]
+[[package]]
+name = "seahash"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
+
[[package]]
name = "security-framework"
version = "2.8.2"
@@ -3295,6 +3377,12 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14a5df39617d7c8558154693a1bb8157a4aab8179209540cc0b10e5dc24e0b18"
+[[package]]
+name = "simdutf8"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
+
[[package]]
name = "siphasher"
version = "0.3.10"
@@ -3409,6 +3497,7 @@ dependencies = [
"maplit",
"md5",
"memmap",
+ "memmap2",
"min-max-heap",
"num_cpus",
"once_cell",
@@ -3420,6 +3509,7 @@ dependencies = [
"rayon",
"regex",
"reqwest",
+ "rkyv",
"rocksdb",
"rust-stemmers",
"scylla",
diff --git a/Cargo.toml b/Cargo.toml
index 19c017a8..a30df524 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -70,4 +70,6 @@ byteorder = "1.4.3"
async-stream = "0.3.3"
tokio-stream = "0.1.11"
scylla = "0.7.0"
-chitchat = "0.5.0"
\ No newline at end of file
+chitchat = "0.5.0"
+memmap2 = "0.5.10"
+rkyv = "0.7.40"
\ No newline at end of file
diff --git a/core/Cargo.toml b/core/Cargo.toml
index b5eef1e2..fad948c6 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -75,6 +75,8 @@ async-stream = { workspace = true }
tokio-stream = { workspace = true }
scylla = { workspace = true }
chitchat = { workspace = true }
+memmap2 = { workspace = true }
+rkyv = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/benches/online-harmonic.rs b/core/benches/online-harmonic.rs
index aa402093..d812ce6f 100644
--- a/core/benches/online-harmonic.rs
+++ b/core/benches/online-harmonic.rs
@@ -17,7 +17,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
for (node, centrality) in centrality.host {
harmonic_centrality_store
.host
- .insert(webgraph.node2id(&node).unwrap(), centrality);
+ .insert(*webgraph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
diff --git a/core/examples/print_approx_harmonic_centrality.rs b/core/examples/print_approx_harmonic_centrality.rs
index 1be026d7..0278282c 100644
--- a/core/examples/print_approx_harmonic_centrality.rs
+++ b/core/examples/print_approx_harmonic_centrality.rs
@@ -110,11 +110,11 @@ fn print_top_nodes(
pub fn main() {
const TOP_N: usize = 50;
- let graph = WebgraphBuilder::new("data/webgraph").read_only(true).open();
- let nodes: Vec<_> = graph.nodes().collect();
+ let graph = WebgraphBuilder::new("data/webgraph").open();
+ let nodes: Vec<_> = graph.nodes().cloned().collect();
let id2node: HashMap<_, _> = nodes
.iter()
- .map(|id| (*id, graph.id2node(id).unwrap()))
+ .map(|id| (*id, graph.id2node(id).unwrap().clone()))
.collect();
let store = CentralityStore::open("data/centrality");
diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rs
index 711fb14f..e919f8a0 100644
--- a/core/examples/print_inbound_links.rs
+++ b/core/examples/print_inbound_links.rs
@@ -20,7 +20,7 @@ use stract::{
};
pub fn main() {
- let graph = WebgraphBuilder::new("data/webgraph").read_only(true).open();
+ let graph = WebgraphBuilder::new("data/webgraph").open();
let inbound = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();
for host in ["archwaypublishing.com", "wordpressfoundation.org"] {
@@ -28,7 +28,7 @@ pub fn main() {
let node = Node::from(host);
let node_id = graph.node2id(&node).unwrap();
- let inbound_vec = inbound.get(&node_id).unwrap();
+ let inbound_vec = inbound.get(node_id).unwrap();
println!("{:?}", inbound_vec);
for edge in graph.ingoing_edges(node) {
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index 0f2d501a..7d5fd945 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -22,17 +22,17 @@ pub struct Centrality {}
impl Centrality {
pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
- let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();
+ let graph = WebgraphBuilder::new(webgraph_path).open();
CentralityStore::build_harmonic(&graph, output_path);
}
pub fn build_online<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
- let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();
+ let graph = WebgraphBuilder::new(webgraph_path).open();
CentralityStore::build_online(&graph, output_path);
}
pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
- let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();
+ let graph = WebgraphBuilder::new(webgraph_path).open();
CentralityStore::build_similarity(&graph, output_path);
}
}
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 2cbf5999..792b553a 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -74,7 +74,7 @@ impl IndexingWorker {
) -> Self {
Self {
centrality_store: IndexerCentralityStore::open(centrality_store_path),
- webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).read_only(true).open()),
+ webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).open()),
crawl_stabilty: crawl_stability_path.map(CrawlStability::open),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
}
diff --git a/core/src/executor.rs b/core/src/executor.rs
index 53a088f7..4244bd95 100644
--- a/core/src/executor.rs
+++ b/core/src/executor.rs
@@ -78,6 +78,7 @@ impl Executor {
// This is important as it makes it possible for the rx iteration to
// terminate.
};
+
let mut result_placeholders: Vec<Option<R>> =
std::iter::repeat_with(|| None).take(num_jobs).collect();
diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rs
index 028ca5ed..6e1a9cc6 100644
--- a/core/src/kv/rocksdb_store.rs
+++ b/core/src/kv/rocksdb_store.rs
@@ -44,25 +44,6 @@ impl RocksDbStore {
Box::new(DB::open(&options, path).expect("unable to open rocks db"))
}
-
- pub fn open_read_only<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>
- where
- P: AsRef<std::path::Path>,
- K: Serialize + DeserializeOwned + 'static,
- V: Serialize + DeserializeOwned + 'static,
- {
- if !path.as_ref().exists() {
- fs::create_dir_all(path.as_ref()).expect("faild to create dir");
- }
-
- let mut options = Options::default();
-
- options.create_if_missing(true);
- options.set_max_open_files(1);
- options.set_max_file_opening_threads(1);
-
- Box::new(DB::open_for_read_only(&options, path, false).expect("unable to open rocks db"))
- }
}
impl<K, V> Kv<K, V> for rocksdb::DB
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 7fbf474f..f937b96b 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -626,7 +626,7 @@ mod tests {
primary_image: None,
host_topic: None,
dmoz_description: None,
- node_id: Some(graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),
+ node_id: Some(*graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),
})
.expect("failed to insert webpage");
index
@@ -656,7 +656,7 @@ mod tests {
fetch_time_ms: 500,
dmoz_description: None,
host_topic: None,
- node_id: Some(graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),
+ node_id: Some(*graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),
})
.expect("failed to insert webpage");
index
@@ -686,7 +686,7 @@ mod tests {
fetch_time_ms: 500,
host_topic: None,
dmoz_description: None,
- node_id: Some(graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),
+ node_id: Some(*graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),
})
.expect("failed to insert webpage");
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index c5d00dbb..6070d093 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -170,7 +170,10 @@ impl CentralityStore {
pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
let mut store = CentralityStore::open(output_path.as_ref());
- store.node2id = graph.node_ids().collect();
+ store.node2id = graph
+ .node_ids()
+ .map(|(node, node_id)| (node.clone(), *node_id))
+ .collect();
let harmonic_centrality = HarmonicCentrality::calculate(graph);
Self::store_host(&output_path, &mut store, harmonic_centrality);
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index b12d11ce..d4a1e3b7 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -255,18 +255,18 @@ mod tests {
for (node, centrality) in harmonic.host {
harmonic_centrality_store
.host
- .insert(graph.node2id(&node).unwrap(), centrality);
+ .insert(*graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
let inbound =
InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);
- let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);
+ let scorer = inbound.scorer(&[*graph.node2id(&Node::from("b.com")).unwrap()], &[]);
let e = graph.node2id(&Node::from("e.com")).unwrap();
let d = graph.node2id(&Node::from("d.com")).unwrap();
- assert!(scorer.score(&e) > scorer.score(&d));
+ assert!(scorer.score(e) > scorer.score(d));
}
#[test]
@@ -285,7 +285,7 @@ mod tests {
for (node, centrality) in harmonic.host {
harmonic_centrality_store
.host
- .insert(graph.node2id(&node).unwrap(), centrality);
+ .insert(*graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
@@ -323,7 +323,7 @@ mod tests {
crawl_stability: 0.0,
primary_image: None,
host_topic: None,
- node_id: Some(graph.node2id(&Node::from("e.com")).unwrap()),
+ node_id: Some(*graph.node2id(&Node::from("e.com")).unwrap()),
dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -353,7 +353,7 @@ mod tests {
crawl_stability: 0.0,
primary_image: None,
host_topic: None,
- node_id: Some(graph.node2id(&Node::from("d.com")).unwrap()),
+ node_id: Some(*graph.node2id(&Node::from("d.com")).unwrap()),
dmoz_description: None,
})
.expect("failed to insert webpage");
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index e2d0b847..6cffce1d 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -1233,7 +1233,7 @@ mod tests {
for (node, centrality) in harmonic.host {
harmonic_centrality_store
.host
- .insert(webgraph.node2id(&node).unwrap(), centrality);
+ .insert(*webgraph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 6f4d55cf..63fc8ad4 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -77,6 +77,7 @@ mod tests {
node_id: Some(
graph
.node2id(&Node::from("https://www.first.com").into_host())
+ .copied()
.unwrap(),
),
})
@@ -111,6 +112,7 @@ mod tests {
node_id: Some(
graph
.node2id(&Node::from("https://www.second.com").into_host())
+ .copied()
.unwrap(),
),
})
@@ -143,7 +145,7 @@ mod tests {
dmoz_description: None,
host_topic: None,
node_id: Some(
- graph
+ *graph
.node2id(&Node::from("https://www.third.com").into_host())
.unwrap(),
),
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index caebf050..d45cc4ee 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -100,6 +100,7 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
for (website, score) in websites.iter_mut().zip(scores.into_iter()) {
let website = website.as_mut_ranking();
+ website.signals.insert(Signal::CrossEncoder, score);
website.score += score;
}
}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index fb4273ff..e2487be7 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -552,9 +552,13 @@ struct RuleBoost {
boost: f64,
}
+struct OpticBoosts {
+ rules: Vec<RuleBoost>,
+}
+
struct SegmentReader {
text_fields: EnumMap<TextField, TextFieldData>,
- optic_rule_boosts: Vec<RuleBoost>,
+ optic_boosts: OpticBoosts,
fastfield_reader: Arc<fastfield_reader::SegmentReader>,
}
@@ -702,7 +706,9 @@ impl SignalAggregator {
self.segment_reader = Some(SegmentReader {
text_fields,
fastfield_reader: fastfield_segment_reader,
- optic_rule_boosts,
+ optic_boosts: OpticBoosts {
+ rules: optic_rule_boosts,
+ },
});
Ok(())
@@ -772,22 +778,29 @@ impl SignalAggregator {
}
pub fn boosts(&mut self, doc: DocId) -> Option<f64> {
- self.segment_reader.as_mut().and_then(|segment_reader| {
- let mut res = 0.0;
- for rule in &mut segment_reader.optic_rule_boosts {
+ self.segment_reader.as_mut().map(|segment_reader| {
+ let mut downrank = 0.0;
+ let mut boost = 0.0;
+
+ for rule in &mut segment_reader.optic_boosts.rules {
if rule.docset.doc() > doc {
continue;
}
if rule.docset.doc() == doc || rule.docset.seek(doc) == doc {
- res += rule.boost;
+ if rule.boost < 0.0 {
+ downrank += rule.boost.abs();
+ } else {
+ boost += rule.boost;
+ }
}
}
- if res == 0.0 {
- None
+ if downrank > boost {
+ let diff = downrank - boost;
+ 1.0 / (1.0 + diff)
} else {
- Some(res)
+ boost - downrank + 1.0
}
})
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 8c5f71c5..e3a4eb24 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -515,7 +515,7 @@ impl DistributedSearcher {
.collect();
let median = if scores.len() % 2 == 0 {
- (scores[(scores.len() / 2) - 1] + scores[(scores.len() / 2)]) / 2.0
+ (scores[(scores.len() / 2) - 1] + scores[scores.len() / 2]) / 2.0
} else {
scores[scores.len() / 2]
};
diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rs
index b2601ddf..10d7a466 100644
--- a/core/src/webgraph/centrality/betweenness.rs
+++ b/core/src/webgraph/centrality/betweenness.rs
@@ -32,7 +32,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
let mut n = 0;
let mut max_dist = 0;
- let nodes: Vec<_> = graph.nodes().take(100_000).collect();
+ let nodes: Vec<_> = graph.nodes().cloned().take(100_000).collect();
let pb =
if with_progress {
@@ -128,7 +128,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
(
centrality
.into_iter()
- .map(|(id, centrality)| (graph.id2node(&id).unwrap(), centrality / norm))
+ .map(|(id, centrality)| (graph.id2node(&id).unwrap().clone(), centrality / norm))
.collect(),
max_dist,
)
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index 2ddb1237..4c5e54ea 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -150,7 +150,7 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {
.filter(|(_, centrality)| *centrality > 0.0)
.map(|(node_id, centrality)| {
(
- graph.id2node(&NodeID::from(node_id)).unwrap(),
+ graph.id2node(&NodeID::from(node_id)).unwrap().clone(),
centrality / norm_factor,
)
})
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index bfd03cb1..4febd1c5 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -267,7 +267,7 @@ where
F1: Fn(NodeID) -> Vec<Edge>,
F2: Fn(&Edge) -> NodeID,
{
- let source_id = graph.node2id(&source);
+ let source_id = graph.node2id(&source).copied();
if source_id.is_none() {
return BTreeMap::new();
@@ -358,7 +358,7 @@ impl OnlineHarmonicCentrality {
// as an estimate
let mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();
- for (node, node_id) in graph.node_ids() {
+ for (node, node_id) in graph.node_ids().map(|(node, id)| (node.clone(), *id)) {
let score = centrality.host.get(&node_id).unwrap_or(0.0);
let candidate = ProxyNodeCandidate { node, score };
@@ -389,7 +389,7 @@ impl OnlineHarmonicCentrality {
.collect();
Arc::new(ProxyNode {
- id: graph.node2id(&node).unwrap(),
+ id: *graph.node2id(&node).unwrap(),
dist_to_node,
dist_from_node,
})
@@ -476,7 +476,7 @@ mod tests {
for (node, centrality) in harmonic.host {
harmonic_centrality_store
.host
- .insert(graph.node2id(&node).unwrap(), centrality);
+ .insert(*graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
@@ -485,23 +485,23 @@ mod tests {
let liked_nodes: Vec<_> = vec![Node::from("B".to_string()), Node::from("E".to_string())]
.into_iter()
- .filter_map(|node| graph.node2id(&node))
+ .filter_map(|node| graph.node2id(&node).copied())
.collect();
let scorer = centrality.scorer(&liked_nodes, &[]);
assert!(
- scorer.score(graph.node2id(&Node::from("E".to_string())).unwrap())
- > scorer.score(graph.node2id(&Node::from("H".to_string())).unwrap())
+ scorer.score(*graph.node2id(&Node::from("E".to_string())).unwrap())
+ > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
);
assert!(
- scorer.score(graph.node2id(&Node::from("H".to_string())).unwrap())
- > scorer.score(graph.node2id(&Node::from("C".to_string())).unwrap())
+ scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
+ > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
);
assert!(
- scorer.score(graph.node2id(&Node::from("C".to_string())).unwrap())
- > scorer.score(graph.node2id(&Node::from("A".to_string())).unwrap())
+ scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
+ > scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())
);
}
@@ -514,7 +514,7 @@ mod tests {
for (node, centrality) in harmonic.host {
harmonic_centrality_store
.host
- .insert(graph.node2id(&node).unwrap(), centrality);
+ .insert(*graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
@@ -523,7 +523,7 @@ mod tests {
let disliked_nodes: Vec<_> = vec![Node::from("D".to_string()), Node::from("E".to_string())]
.into_iter()
- .map(|node| graph.node2id(&node).unwrap())
+ .map(|node| *graph.node2id(&node).unwrap())
.collect();
let scorer = centrality.scorer(&[], &disliked_nodes);
@@ -542,7 +542,7 @@ mod tests {
for (node, centrality) in harmonic.host {
harmonic_centrality_store
.host
- .insert(graph.node2id(&node).unwrap(), centrality);
+ .insert(*graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
@@ -551,29 +551,29 @@ mod tests {
let liked_nodes: Vec<_> = vec![Node::from("B".to_string()), Node::from("E".to_string())]
.into_iter()
- .filter_map(|node| graph.node2id(&node))
+ .filter_map(|node| graph.node2id(&node).copied())
.collect();
let disliked_nodes: Vec<_> = vec![Node::from("F".to_string())]
.into_iter()
- .filter_map(|node| graph.node2id(&node))
+ .filter_map(|node| graph.node2id(&node).copied())
.collect();
let scorer = centrality.scorer(&liked_nodes, &disliked_nodes);
assert!(
- scorer.score(graph.node2id(&Node::from("E".to_string())).unwrap())
- > scorer.score(graph.node2id(&Node::from("H".to_string())).unwrap())
+ scorer.score(*graph.node2id(&Node::from("E".to_string())).unwrap())
+ > scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
);
assert!(
- scorer.score(graph.node2id(&Node::from("H".to_string())).unwrap())
- > scorer.score(graph.node2id(&Node::from("C".to_string())).unwrap())
+ scorer.score(*graph.node2id(&Node::from("H".to_string())).unwrap())
+ > scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
);
assert!(
- scorer.score(graph.node2id(&Node::from("C".to_string())).unwrap())
- > scorer.score(graph.node2id(&Node::from("A".to_string())).unwrap())
+ scorer.score(*graph.node2id(&Node::from("C".to_string())).unwrap())
+ > scorer.score(*graph.node2id(&Node::from("A".to_string())).unwrap())
);
}
}
diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rs
index 660256a6..a4d8b407 100644
--- a/core/src/webgraph/centrality/topic.rs
+++ b/core/src/webgraph/centrality/topic.rs
@@ -151,7 +151,7 @@ impl TopicCentrality {
TopDocs::with_limit(1000, index.inverted_index.fastfield_reader(&tv_searcher))
.tweak_score(score_tweaker);
- let mut nodes: Vec<_> = webgraph.nodes().collect();
+ let mut nodes: Vec<_> = webgraph.nodes().copied().collect();
nodes.sort();
let mut node_scores = vec![vec![0.0; NUM_TOPICS]; nodes.len()];
@@ -179,7 +179,7 @@ impl TopicCentrality {
let top_sites: Vec<_> = top_sites
.into_iter()
- .filter_map(|node| webgraph.node2id(&node))
+ .filter_map(|node| webgraph.node2id(node).copied())
.collect();
if top_sites.is_empty() {
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 1dd0a844..7e3b3fc6 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -15,25 +15,23 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
mod segment;
+use rkyv::Archive;
+use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BinaryHeap, HashSet};
-use std::fs::File;
+use std::fs::{File, OpenOptions};
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::Path;
use std::sync::Arc;
use std::{cmp, fs};
-use uuid::Uuid;
-
-use segment::Segment;
use crate::directory::{self, DirEntry};
use crate::executor::Executor;
use crate::webpage::Url;
-use crate::kv::rocksdb_store::RocksDbStore;
-
pub mod centrality;
-use self::segment::{CachedStore, SegmentNodeID};
+mod store;
+use self::segment::{LiveSegment, SegmentNodeID, StoredSegment};
const MAX_NUM_SEGMENTS: usize = 400;
@@ -46,7 +44,8 @@ impl From<u64> for NodeID {
}
}
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, Archive, rkyv::Serialize, rkyv::Deserialize, PartialEq, Eq, Hash)]
+#[archive_attr(derive(Eq, Hash, PartialEq, Debug))]
pub(crate) struct StoredEdge {
other: SegmentNodeID,
label: String,
@@ -106,11 +105,25 @@ impl From<Url> for Node {
}
}
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq)]
pub struct Edge {
pub from: NodeID,
pub to: NodeID,
- pub label: String,
+ pub label: Loaded<String>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum Loaded<T> {
+ Some(T),
+ NotYet,
+}
+impl<T> Loaded<T> {
+ fn loaded(self) -> Option<T> {
+ match self {
+ Loaded::Some(t) => Some(t),
+ Loaded::NotYet => None,
+ }
+ }
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -122,7 +135,6 @@ pub struct FullEdge {
pub struct WebgraphBuilder {
path: Box<Path>,
- read_only: bool,
}
impl WebgraphBuilder {
@@ -137,34 +149,11 @@ impl WebgraphBuilder {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
Self {
path: path.as_ref().into(),
- read_only: false,
}
}
- pub fn read_only(mut self, read_only: bool) -> Self {
- self.read_only = read_only;
-
- self
- }
-
pub fn open(self) -> Webgraph {
- if self.read_only {
- Webgraph::open_read_only(self.path)
- } else {
- Webgraph::open(self.path)
- }
- }
-}
-
-pub trait Store
-where
- Self: Sized + Send + Sync,
-{
- fn open<P: AsRef<Path>>(path: P, id: String) -> Segment<Self>;
- fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Segment<Self>;
-
- fn temporary() -> Segment<Self> {
- Self::open(crate::gen_temp_path(), "id123".to_string())
+ Webgraph::open(self.path)
}
}
@@ -195,8 +184,8 @@ where
let mut queue = BinaryHeap::new();
- queue.push(cmp::Reverse((0, source_id)));
- distances.insert(source_id, 0);
+ queue.push(cmp::Reverse((0, *source_id)));
+ distances.insert(*source_id, 0);
while let Some(state) = queue.pop() {
let (cost, v) = state.0;
@@ -225,7 +214,7 @@ impl ShortestPaths for Webgraph {
fn distances(&self, source: Node) -> BTreeMap<Node, u8> {
self.raw_distances(source)
.into_iter()
- .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))
+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node").clone(), dist))
.collect()
}
@@ -250,7 +239,7 @@ impl ShortestPaths for Webgraph {
fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {
self.raw_reversed_distances(source)
.into_iter()
- .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))
+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node").clone(), dist))
.collect()
}
}
@@ -263,22 +252,53 @@ struct Meta {
next_node_id: u64,
}
-struct SegmentMergeCandidate<S: Store> {
- segment: Segment<S>,
- merges: Vec<Segment<S>>,
+struct SegmentMergeCandidate {
+ segment: StoredSegment,
+ merges: Vec<StoredSegment>,
}
-pub struct Webgraph<S: Store = RocksDbStore> {
+fn open_bin<T, P>(path: P) -> T
+where
+ P: AsRef<Path>,
+ T: DeserializeOwned + Default,
+{
+ let f = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(path)
+ .unwrap();
+
+ let reader = BufReader::new(f);
+
+ bincode::deserialize_from(reader).unwrap_or_default()
+}
+
+fn save_bin<T, P>(val: &T, path: P)
+where
+ P: AsRef<Path>,
+ T: Serialize,
+{
+ let f = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(path)
+ .unwrap();
+
+ bincode::serialize_into(f, val).unwrap();
+}
+pub struct Webgraph {
pub path: String,
- live_segment: Option<Segment<S>>,
- segments: Vec<Segment<S>>,
+ live_segment: LiveSegment,
+ segments: Vec<StoredSegment>,
executor: Arc<Executor>,
- node2id: CachedStore<Node, NodeID>,
- id2node: CachedStore<NodeID, Node>,
+ node2id: BTreeMap<Node, NodeID>,
+ id2node: BTreeMap<NodeID, Node>,
meta: Meta,
}
-impl<S: Store> Webgraph<S> {
+impl Webgraph {
fn meta<P: AsRef<Path>>(path: P) -> Meta {
let meta_path = path.as_ref().join("metadata.json");
let mut reader = BufReader::new(
@@ -316,7 +336,7 @@ impl<S: Store> Webgraph<S> {
fs::create_dir_all(path.as_ref().join("segments")).unwrap();
let mut segments = Vec::new();
for segment in &meta.comitted_segments {
- segments.push(Segment::open(
+ segments.push(StoredSegment::open(
path.as_ref().join("segments").join(segment),
segment.clone(),
));
@@ -324,41 +344,11 @@ impl<S: Store> Webgraph<S> {
Self {
path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
- live_segment: None,
+ live_segment: LiveSegment::default(),
segments,
executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),
- node2id: CachedStore::new(RocksDbStore::open(path.as_ref().join("node2id")), 100_000),
- id2node: CachedStore::new(RocksDbStore::open(path.as_ref().join("id2node")), 100_000),
- meta,
- }
- }
-
- fn open_read_only<P: AsRef<Path>>(path: P) -> Self {
- fs::create_dir_all(&path).unwrap();
- let meta = Self::meta(&path);
-
- fs::create_dir_all(path.as_ref().join("segments")).unwrap();
- let mut segments = Vec::new();
- for segment in &meta.comitted_segments {
- segments.push(Segment::open(
- path.as_ref().join("segments").join(segment),
- segment.clone(),
- ));
- }
-
- Self {
- path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
- live_segment: None,
- segments,
- executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),
- node2id: CachedStore::new(
- RocksDbStore::open_read_only(path.as_ref().join("node2id")),
- 100_000,
- ),
- id2node: CachedStore::new(
- RocksDbStore::open_read_only(path.as_ref().join("id2node")),
- 100_000,
- ),
+ node2id: open_bin(path.as_ref().join("node2id.bin")),
+ id2node: open_bin(path.as_ref().join("id2node.bin")),
meta,
}
}
@@ -371,7 +361,7 @@ impl<S: Store> Webgraph<S> {
fn id_or_assign(&mut self, node: &Node) -> NodeID {
match self.node2id(node) {
- Some(id) => id,
+ Some(id) => *id,
None => {
let id = self.id_and_increment();
@@ -390,30 +380,19 @@ impl<S: Store> Webgraph<S> {
let (from, to) = (from.into_host(), to.into_host());
let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));
- match &mut self.live_segment {
- Some(segment) => segment.insert(from_id, to_id, label),
- None => {
- let segment_id = Uuid::new_v4().to_string();
- let path = Path::new(&self.path).join("segments").join(&segment_id);
- let mut segment: Segment<S> = Segment::open(path, segment_id);
-
- segment.insert(from_id, to_id, label);
-
- self.live_segment = Some(segment);
- }
- }
+ self.live_segment.insert(from_id, to_id, label);
}
- pub fn merge(&mut self, mut other: Webgraph<S>) {
+ pub fn merge(&mut self, mut other: Webgraph) {
other.commit();
let mut mapping = Vec::new();
for (node, other_id) in other.node2id.iter() {
- match self.node2id(&node) {
- Some(this_id) => mapping.push((other_id, this_id)),
+ match self.node2id(node) {
+ Some(this_id) => mapping.push((*other_id, *this_id)),
None => {
- let new_id = self.id_or_assign(&node);
- mapping.push((other_id, new_id));
+ let new_id = self.id_or_assign(node);
+ mapping.push((*other_id, new_id));
}
}
}
@@ -425,30 +404,31 @@ impl<S: Store> Webgraph<S> {
)
.expect("failed to merge webgraphs");
- for mut segment in other.segments {
- segment.flush();
+ for segment in other.segments {
let id = segment.id();
let new_path = Path::new(&self.path).join("segments").join(segment.id());
std::fs::rename(segment.path(), &new_path).unwrap();
self.meta.comitted_segments.push(segment.id());
drop(segment);
- self.segments.push(Segment::open(new_path, id));
+ self.segments.push(StoredSegment::open(new_path, id));
}
self.commit();
}
pub fn commit(&mut self) {
- if let Some(mut segment) = self.live_segment.take() {
- segment.flush();
+ if !self.live_segment.is_empty() {
+ let live_segment = std::mem::take(&mut self.live_segment);
+ let segment = live_segment.commit(Path::new(&self.path).join("segments"));
+
self.meta.comitted_segments.push(segment.id());
self.segments.push(segment);
}
self.save_metadata();
- self.node2id.flush();
- self.id2node.flush();
+ save_bin(&self.node2id, Path::new(&self.path).join("node2id.bin"));
+ save_bin(&self.id2node, Path::new(&self.path).join("id2node.bin"));
if self.segments.len() > MAX_NUM_SEGMENTS {
self.merge_segments((MAX_NUM_SEGMENTS / 2).max(1));
@@ -457,12 +437,12 @@ impl<S: Store> Webgraph<S> {
pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
if let Some(node_id) = self.node2id(&node) {
- self.raw_ingoing_edges(&node_id)
+ self.inner_ingoing_edges(node_id, true)
.into_iter()
.map(|edge| FullEdge {
- from: self.id2node(&edge.from).unwrap(),
- to: self.id2node(&edge.to).unwrap(),
- label: edge.label,
+ from: self.id2node(&edge.from).unwrap().clone(),
+ to: self.id2node(&edge.to).unwrap().clone(),
+ label: edge.label.loaded().unwrap(),
})
.collect()
} else {
@@ -471,8 +451,15 @@ impl<S: Store> Webgraph<S> {
}
pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {
+ self.inner_ingoing_edges(node, false)
+ }
+
+ fn inner_ingoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {
self.executor
- .map(|segment| segment.ingoing_edges(node), self.segments.iter())
+ .map(
+ |segment| segment.ingoing_edges(node, load_labels),
+ self.segments.iter(),
+ )
.unwrap()
.into_iter()
.flatten()
@@ -480,8 +467,15 @@ impl<S: Store> Webgraph<S> {
}
pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {
+ self.inner_outgoing_edges(node, false)
+ }
+
+ fn inner_outgoing_edges(&self, node: &NodeID, load_labels: bool) -> Vec<Edge> {
self.executor
- .map(|segment| segment.outgoing_edges(node), self.segments.iter())
+ .map(
+ |segment| segment.outgoing_edges(node, load_labels),
+ self.segments.iter(),
+ )
.unwrap()
.into_iter()
.flatten()
@@ -490,12 +484,12 @@ impl<S: Store> Webgraph<S> {
pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {
if let Some(node_id) = self.node2id(&node) {
- self.raw_outgoing_edges(&node_id)
+ self.inner_outgoing_edges(node_id, true)
.into_iter()
.map(|edge| FullEdge {
- from: self.id2node(&edge.from).unwrap(),
- to: self.id2node(&edge.to).unwrap(),
- label: edge.label,
+ from: self.id2node(&edge.from).unwrap().clone(),
+ to: self.id2node(&edge.to).unwrap().clone(),
+ label: edge.label.loaded().unwrap(),
})
.collect()
} else {
@@ -503,23 +497,19 @@ impl<S: Store> Webgraph<S> {
}
}
- pub fn node2id(&self, node: &Node) -> Option<NodeID> {
- self.node2id
- .get(node)
- .map(|lock| lock.read().unwrap().0.into())
+ pub fn node2id(&self, node: &Node) -> Option<&NodeID> {
+ self.node2id.get(node)
}
- pub fn id2node(&self, id: &NodeID) -> Option<Node> {
- self.id2node
- .get(id)
- .map(|lock| lock.read().unwrap().clone())
+ pub fn id2node(&self, id: &NodeID) -> Option<&Node> {
+ self.id2node.get(id)
}
- pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {
- self.node2id.iter().map(|(_, id)| id)
+ pub fn nodes(&self) -> impl Iterator<Item = &NodeID> + '_ {
+ self.node2id.values()
}
- pub fn node_ids(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {
+ pub fn node_ids(&self) -> impl Iterator<Item = (&Node, &NodeID)> + '_ {
self.node2id.iter()
}
@@ -557,7 +547,7 @@ impl<S: Store> Webgraph<S> {
.map(
|mut candidate| {
for other in candidate.merges {
- candidate.segment.merge(other);
+ candidate.segment = candidate.segment.merge(other);
}
candidate.segment
@@ -699,19 +689,68 @@ mod test {
#[test]
fn merge() {
- let mut graph1 = WebgraphBuilder::new_memory().open();
+ let mut graphs = Vec::new();
+ for (from, to, label) in &[
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("C"), String::new()),
+ (Node::from("C"), Node::from("D"), String::new()),
+ (Node::from("D"), Node::from("E"), String::new()),
+ (Node::from("E"), Node::from("F"), String::new()),
+ (Node::from("F"), Node::from("G"), String::new()),
+ (Node::from("G"), Node::from("H"), String::new()),
+ ] {
+ let mut graph = WebgraphBuilder::new_memory().open();
+ graph.insert(from.clone(), to.clone(), label.clone());
+ graph.commit();
+ graphs.push(graph);
+ }
- graph1.insert(Node::from("A"), Node::from("B"), String::new());
- graph1.commit();
+ let mut graph = graphs.pop().unwrap();
- let mut graph2 = WebgraphBuilder::new_memory().open();
- graph2.insert(Node::from("B"), Node::from("C"), String::new());
- graph2.commit();
+ for other in graphs {
+ graph.merge(other);
+ }
- graph1.merge(graph2);
+ assert_eq!(
+ graph.distances(Node::from("A")).get(&Node::from("H")),
+ Some(&7)
+ );
+ graph.merge_segments(1);
assert_eq!(
- graph1.distances(Node::from("A")).get(&Node::from("C")),
+ graph.distances(Node::from("A")).get(&Node::from("H")),
+ Some(&7)
+ )
+ }
+ #[test]
+ fn merge_cycle() {
+ let mut graphs = Vec::new();
+ for (from, to, label) in &[
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("C"), String::new()),
+ (Node::from("C"), Node::from("A"), String::new()),
+ ] {
+ let mut graph = WebgraphBuilder::new_memory().open();
+ graph.insert(from.clone(), to.clone(), label.clone());
+ graph.commit();
+ graphs.push(graph);
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other);
+ }
+
+ assert_eq!(
+ graph.distances(Node::from("A")).get(&Node::from("C")),
+ Some(&2)
+ );
+
+ graph.merge_segments(1);
+ assert_eq!(
+ graph.distances(Node::from("A")).get(&Node::from("C")),
Some(&2)
)
}
@@ -721,6 +760,7 @@ mod test {
let graph = test_graph();
let path = graph.path.clone();
+ dbg!(&path);
let frozen: FrozenWebgraph = graph.into();
let bytes = bincode::serialize(&frozen).unwrap();
@@ -728,6 +768,7 @@ mod test {
let deserialized_frozen: FrozenWebgraph = bincode::deserialize(&bytes).unwrap();
let graph: Webgraph = deserialized_frozen.into();
+ dbg!(&graph.path);
let distances = graph.distances(Node::from("D"));
@@ -840,12 +881,6 @@ mod test {
graph.merge_segments(2);
assert_eq!(graph.segments.len(), 2);
- let distances = graph.distances(Node::from("D"));
-
- assert_eq!(distances.get(&Node::from("C")), Some(&1));
- assert_eq!(distances.get(&Node::from("A")), Some(&2));
- assert_eq!(distances.get(&Node::from("B")), Some(&3));
-
let mut res = graph.outgoing_edges(Node::from("A"));
res.sort_by(|a, b| a.to.name.cmp(&b.to.name));
@@ -888,5 +923,22 @@ mod test {
},
]
);
+
+ let res = graph.outgoing_edges(Node::from("D"));
+
+ assert_eq!(
+ res,
+ vec![FullEdge {
+ from: Node::from("D"),
+ to: Node::from("C"),
+ label: String::new()
+ },]
+ );
+
+ let distances = graph.distances(Node::from("D"));
+
+ assert_eq!(distances.get(&Node::from("C")), Some(&1));
+ assert_eq!(distances.get(&Node::from("A")), Some(&2));
+ assert_eq!(distances.get(&Node::from("B")), Some(&3));
}
}
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 92c49930..240d3aab 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -14,19 +14,31 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use rkyv::{Archive, Deserialize};
use std::{
collections::{BTreeMap, HashSet},
fmt::Debug,
- marker::PhantomData,
path::Path,
- sync::{Arc, RwLock},
};
-use super::{Edge, NodeID, Store, StoredEdge};
-use crate::kv::{rocksdb_store::RocksDbStore, Kv};
-
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+use super::{open_bin, save_bin, store::Store, Edge, Loaded, NodeID, StoredEdge};
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ serde::Serialize,
+ serde::Deserialize,
+ Archive,
+ rkyv::Serialize,
+ rkyv::Deserialize,
+ PartialEq,
+ Eq,
+ PartialOrd,
+ Ord,
+ Hash,
+)]
+#[archive_attr(derive(Eq, Hash, PartialEq, Debug))]
pub struct SegmentNodeID(u64);
impl From<u64> for SegmentNodeID {
@@ -35,336 +47,422 @@ impl From<u64> for SegmentNodeID {
}
}
-pub(crate) struct Adjacency {
- pub(crate) tree: CachedStore<SegmentNodeID, HashSet<StoredEdge>>,
+pub struct StoredSegment {
+ adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,
+ reversed_adjacency: Store<SegmentNodeID, HashSet<StoredEdge>>,
+ id_mapping: BTreeMap<NodeID, SegmentNodeID>,
+ rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,
+ meta: Meta,
+ id: String,
+ folder_path: String,
}
-impl Adjacency {
- fn insert(&mut self, from: SegmentNodeID, to: SegmentNodeID, label: String) {
- let edge = StoredEdge { other: to, label };
-
- if let Some(existing) = self.tree.get(&from) {
- existing.write().unwrap().insert(edge);
- } else {
- let mut set = HashSet::new();
- set.insert(edge);
- self.tree.insert(from, set);
+impl StoredSegment {
+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {
+ StoredSegment {
+ adjacency: Store::open(folder_path.as_ref().join("adjacency"), &id).unwrap(),
+ reversed_adjacency: Store::open(folder_path.as_ref().join("reversed_adjacency"), &id)
+ .unwrap(),
+ id_mapping: open_bin(folder_path.as_ref().join("id_mapping.bin")),
+ rev_id_mapping: open_bin(folder_path.as_ref().join("rev_id_mapping.bin")),
+ meta: open_bin(folder_path.as_ref().join("meta.bin")),
+ folder_path: folder_path
+ .as_ref()
+ .as_os_str()
+ .to_str()
+ .unwrap()
+ .to_string(),
+ id,
}
}
- fn edges(&self, node: &SegmentNodeID) -> Vec<StoredEdge> {
- self.tree
- .get(node)
- .map(|r| r.read().unwrap().clone())
- .unwrap_or_default()
- .into_iter()
- .collect()
+ pub fn num_nodes(&self) -> usize {
+ self.meta.num_nodes as usize
}
-}
-
-pub struct CachedStore<K, V>
-where
- K: Ord + Eq + Serialize + DeserializeOwned + Clone,
- V: Serialize + DeserializeOwned + Clone,
-{
- store: Box<dyn Kv<K, V> + Send + Sync>,
- cache: RwLock<BTreeMap<K, Arc<RwLock<V>>>>,
- cache_size: usize,
-}
-impl<K, V> CachedStore<K, V>
-where
- K: Ord + Eq + Serialize + DeserializeOwned + Clone,
- V: Serialize + DeserializeOwned + Clone + Debug,
-{
- pub fn new(store: Box<dyn Kv<K, V> + Send + Sync>, cache_size: usize) -> Self {
- Self {
- store,
- cache_size,
- cache: RwLock::new(BTreeMap::new()),
- }
+ fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {
+ self.id_mapping.get(node).copied()
}
- fn update_cache(&self, key: &K) {
- if !self.cache.read().unwrap().contains_key(key) {
- let mut cache = self.cache.write().unwrap();
-
- while cache.len() >= self.cache_size {
- cache.pop_first();
- }
-
- let val = self.store.get(key);
+ fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {
+ self.rev_id_mapping.get(node).copied()
+ }
+
+ pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {
+ self.id_mapping(node)
+ .and_then(|segment_id| {
+ self.adjacency.get(&segment_id).map(|edges| {
+ edges
+ .iter()
+ .map(move |edge| {
+ if load_label {
+ Edge {
+ from: *node,
+ to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),
+ label: Loaded::Some(
+ edge.label.deserialize(&mut rkyv::Infallible).unwrap(),
+ ),
+ }
+ } else {
+ Edge {
+ from: *node,
+ to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),
+ label: Loaded::NotYet,
+ }
+ }
+ })
+ .collect()
+ })
+ })
+ .unwrap_or_default()
+ }
- if let Some(val) = val {
- cache.insert(key.clone(), Arc::new(RwLock::new(val)));
- }
- }
+ pub fn ingoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {
+ self.id_mapping(node)
+ .and_then(|segment_id| {
+ self.reversed_adjacency.get(&segment_id).map(|edges| {
+ edges
+ .iter()
+ .map(move |edge| {
+ if load_label {
+ Edge {
+ from: self
+ .rev_id_mapping(&SegmentNodeID(edge.other.0))
+ .unwrap(),
+ to: *node,
+ label: Loaded::Some(
+ edge.label.deserialize(&mut rkyv::Infallible).unwrap(),
+ ),
+ }
+ } else {
+ Edge {
+ from: self
+ .rev_id_mapping(&SegmentNodeID(edge.other.0))
+ .unwrap(),
+ to: *node,
+ label: Loaded::NotYet,
+ }
+ }
+ })
+ .collect()
+ })
+ })
+ .unwrap_or_default()
}
- pub fn get(&self, key: &K) -> Option<Arc<RwLock<V>>> {
- self.update_cache(key);
- let guard = self.cache.read().unwrap();
- guard.get(key).cloned()
+ fn flush(&mut self) {
+ self.adjacency.flush().unwrap();
+ self.reversed_adjacency.flush().unwrap();
+ save_bin(&self.id_mapping, self.path().join("id_mapping.bin"));
+ save_bin(&self.rev_id_mapping, self.path().join("rev_id_mapping.bin"));
+ save_bin(&self.meta, self.path().join("meta.bin"));
}
- pub fn insert(&mut self, key: K, value: V) {
- let cache = self.cache.get_mut().unwrap();
+ pub fn id(&self) -> String {
+ self.id.clone()
+ }
- cache.insert(key, Arc::new(RwLock::new(value)));
+ pub fn path(&self) -> &Path {
+ Path::new(&self.folder_path)
+ }
- if cache.len() >= self.cache_size {
- while cache.len() > self.cache_size / 2 {
- if let Some((key, value)) = cache.pop_first() {
- self.store.insert(key, value.write().unwrap().clone());
+ pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {
+ self.adjacency.iter().flat_map(move |(node_id, edges)| {
+ let from = self.rev_id_mapping(node_id).unwrap();
+
+ edges.iter().map(move |stored_edge| Edge {
+ from,
+ to: self
+ .rev_id_mapping(&SegmentNodeID(stored_edge.other.0))
+ .unwrap(),
+ label: Loaded::NotYet,
+ })
+ })
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ fn merge_adjacency<F1, F2>(
+ &self,
+ self_edges_fn: F1,
+ other: &Self,
+ other_edges_fn: F2,
+ next_segment_id: &mut SegmentNodeID,
+ new_id_mapping: &mut BTreeMap<NodeID, SegmentNodeID>,
+ new_rev_id_mapping: &mut BTreeMap<SegmentNodeID, NodeID>,
+ new_adjacency: &mut Store<SegmentNodeID, HashSet<StoredEdge>>,
+ ) where
+ F1: Fn(&Self, &SegmentNodeID) -> HashSet<StoredEdge>,
+ F2: Fn(&Self, &SegmentNodeID) -> HashSet<StoredEdge>,
+ {
+ for (node_id, segment_id) in &self.id_mapping {
+ let mut edges = self_edges_fn(self, segment_id);
+
+ if let Some(other_segment_id) = other.id_mapping.get(node_id) {
+ let other_edges = other_edges_fn(other, other_segment_id);
+
+ for other_edge in other_edges {
+ let node = other.rev_id_mapping(&other_edge.other).unwrap();
+
+ if let Some(segment_id) = new_id_mapping.get(&node).copied() {
+ edges.insert(StoredEdge {
+ other: segment_id,
+ label: other_edge.label,
+ });
+ } else {
+ let r = new_id_mapping.insert(node, *next_segment_id);
+ debug_assert!(r.is_none());
+ let r = new_rev_id_mapping.insert(*next_segment_id, node);
+ debug_assert!(r.is_none());
+
+ edges.insert(StoredEdge {
+ other: *next_segment_id,
+ label: other_edge.label,
+ });
+
+ *next_segment_id = SegmentNodeID(next_segment_id.0 + 1);
+ }
}
}
- self.store.flush();
+ if !edges.is_empty() {
+ let segment_id = new_id_mapping.get(node_id).copied().unwrap();
+ println!("{:?} -> {:?}", node_id, segment_id);
+ new_adjacency.insert(segment_id, &edges).unwrap();
+ }
}
- }
- pub fn flush(&mut self) {
- let cache = self.cache.get_mut().unwrap();
-
- while let Some((key, value)) = cache.pop_first() {
- self.store.insert(key, value.write().unwrap().clone());
+ for (node_id, segment_id) in &other.id_mapping {
+ if self.id_mapping.get(node_id).is_none() {
+ let this_segment_id = if let Some(segment_id) = new_id_mapping.get(node_id) {
+ *segment_id
+ } else {
+ let r = new_id_mapping.insert(*node_id, *next_segment_id);
+ debug_assert!(r.is_none());
+
+ let r = new_rev_id_mapping.insert(*next_segment_id, *node_id);
+ debug_assert!(r.is_none());
+ let this = *next_segment_id;
+
+ *next_segment_id = SegmentNodeID(next_segment_id.0 + 1);
+
+ this
+ };
+
+ let edges = other_edges_fn(other, segment_id);
+
+ let edges: HashSet<_> = edges
+ .into_iter()
+ .map(|edge| {
+ let other_node = other.rev_id_mapping(&edge.other).unwrap();
+ if let Some(segment_id) = new_id_mapping.get(&other_node) {
+ StoredEdge {
+ other: *segment_id,
+ label: edge.label,
+ }
+ } else {
+ let new_segment_id = *next_segment_id;
+ *next_segment_id = SegmentNodeID(next_segment_id.0 + 1);
+
+ new_id_mapping.insert(other_node, new_segment_id);
+ new_rev_id_mapping.insert(new_segment_id, other_node);
+
+ StoredEdge {
+ other: new_segment_id,
+ label: edge.label,
+ }
+ }
+ })
+ .collect();
+
+ if !edges.is_empty() {
+ new_adjacency.insert(this_segment_id, &edges).unwrap();
+ }
+ }
}
-
- self.store.flush();
}
- pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
- self.store.iter()
- }
+ pub fn merge(self, other: StoredSegment) -> Self {
+ let new_segment_id = uuid::Uuid::new_v4().to_string();
- pub fn delete(&mut self, key: &K) {
- self.store.delete(key);
- self.cache.write().unwrap().remove(key);
- }
-}
-
-pub struct Segment<S: Store> {
- adjacency: Adjacency,
- reversed_adjacency: Adjacency,
- id_mapping: CachedStore<NodeID, SegmentNodeID>,
- rev_id_mapping: CachedStore<SegmentNodeID, NodeID>,
- meta: CachedStore<String, u64>,
- store: PhantomData<S>,
- id: String,
- path: String,
-}
+ let mut new_adjacency = Store::open(
+ self.path().join(&new_segment_id).join("adjacency"),
+ &new_segment_id,
+ )
+ .unwrap();
-impl<S: Store> Segment<S> {
- #[cfg(test)]
- pub(crate) fn temporary() -> Segment<S> {
- S::temporary()
- }
+ let mut next_segment_id = SegmentNodeID(self.meta.num_nodes);
- pub fn open<P: AsRef<Path>>(path: P, id: String) -> Self {
- S::open(path, id)
- }
+ let mut new_id_mapping = self.id_mapping.clone();
+ let mut new_rev_id_mapping = self.rev_id_mapping.clone();
- pub fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Self {
- S::open_read_only(path, id)
- }
+ self.merge_adjacency(
+ |_self: &StoredSegment, segment_id: &SegmentNodeID| {
+ _self
+ .adjacency
+ .get(segment_id)
+ .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())
+ .unwrap_or_default()
+ },
+ &other,
+ |_other: &StoredSegment, segment_id: &SegmentNodeID| {
+ _other
+ .adjacency
+ .get(segment_id)
+ .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())
+ .unwrap_or_default()
+ },
+ &mut next_segment_id,
+ &mut new_id_mapping,
+ &mut new_rev_id_mapping,
+ &mut new_adjacency,
+ );
- fn next_id(&self) -> SegmentNodeID {
- let key = &"next_id".to_string();
- self.meta
- .get(key)
- .map(|arc| *arc.read().unwrap())
- .unwrap_or(0)
- .into()
- }
+ let mut new_reversed_adjacency = Store::open(
+ self.path().join(&new_segment_id).join("reversed_adjacency"),
+ &new_segment_id,
+ )
+ .unwrap();
+
+ self.merge_adjacency(
+ |_self: &StoredSegment, segment_id: &SegmentNodeID| {
+ _self
+ .reversed_adjacency
+ .get(segment_id)
+ .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())
+ .unwrap_or_default()
+ },
+ &other,
+ |_other: &StoredSegment, segment_id: &SegmentNodeID| {
+ _other
+ .reversed_adjacency
+ .get(segment_id)
+ .map(|edges| edges.deserialize(&mut rkyv::Infallible).unwrap())
+ .unwrap_or_default()
+ },
+ &mut next_segment_id,
+ &mut new_id_mapping,
+ &mut new_rev_id_mapping,
+ &mut new_reversed_adjacency,
+ );
- pub fn num_nodes(&self) -> usize {
- self.next_id().0 as usize
- }
+ let mut res = Self {
+ adjacency: new_adjacency,
+ reversed_adjacency: new_reversed_adjacency,
+ id_mapping: new_id_mapping,
+ rev_id_mapping: new_rev_id_mapping,
+ meta: Meta {
+ num_nodes: next_segment_id.0,
+ },
+ id: new_segment_id,
+ folder_path: self.folder_path,
+ };
- fn increment_next_id(&mut self) {
- let current_id = self.next_id().0;
- let next_id = current_id + 1;
- self.meta.insert("next_id".to_string(), next_id);
- }
+ res.flush();
- fn id_and_increment(&mut self) -> SegmentNodeID {
- let id = self.next_id();
- self.increment_next_id();
- id
+ res
}
- fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {
- self.id_mapping.get(node).map(|lock| *lock.read().unwrap())
- }
+ pub fn update_id_mapping(&mut self, mapping: Vec<(NodeID, NodeID)>) {
+ let mut new_mappings = Vec::with_capacity(mapping.len());
- fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {
- self.rev_id_mapping
- .get(node)
- .map(|lock| *lock.read().unwrap())
- }
+ for (old_id, new_id) in mapping {
+ if let Some(segment_id) = self.id_mapping(&old_id) {
+ self.id_mapping.remove(&old_id);
+ self.rev_id_mapping.remove(&segment_id);
- pub fn outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {
- match self.id_mapping(node) {
- Some(segment_id) => self
- .adjacency
- .edges(&segment_id)
- .into_iter()
- .map(|edge| Edge {
- from: *node,
- to: self.rev_id_mapping(&edge.other).unwrap(),
- label: edge.label,
- })
- .collect(),
- None => Vec::new(),
+ new_mappings.push((segment_id, new_id));
+ }
}
- }
- pub fn ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {
- match self.id_mapping(node) {
- Some(segment_id) => self
- .reversed_adjacency
- .edges(&segment_id)
- .into_iter()
- .map(|edge| Edge {
- from: self.rev_id_mapping(&edge.other).unwrap(),
- to: *node,
- label: edge.label,
- })
- .collect(),
- None => Vec::new(),
+ for (segment_id, new_id) in new_mappings {
+ self.id_mapping.insert(new_id, segment_id);
+ self.rev_id_mapping.insert(segment_id, new_id);
}
+
+ self.flush();
}
+}
- fn id_or_assign(&mut self, node: &NodeID) -> SegmentNodeID {
- match self.id_mapping(node) {
- Some(id) => id,
- None => {
- let id = self.id_and_increment();
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]
+struct Meta {
+ num_nodes: u64,
+}
- self.id_mapping.insert(*node, id);
- self.rev_id_mapping.insert(id, *node);
+#[derive(Default)]
+pub struct LiveSegment {
+ adjacency: BTreeMap<SegmentNodeID, HashSet<StoredEdge>>,
+ reversed_adjacency: BTreeMap<SegmentNodeID, HashSet<StoredEdge>>,
+ id_mapping: BTreeMap<NodeID, SegmentNodeID>,
+ rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,
+ next_id: u64,
+}
- id
- }
- }
- }
+impl LiveSegment {
+ fn get_or_create_id(&mut self, id: NodeID) -> SegmentNodeID {
+ if let Some(segment_id) = self.id_mapping.get(&id) {
+ *segment_id
+ } else {
+ let segment_id = SegmentNodeID(self.next_id);
+ self.next_id += 1;
- pub fn insert(&mut self, from: NodeID, to: NodeID, label: String) {
- let from_id = self.id_or_assign(&from);
- let to_id = self.id_or_assign(&to);
+ self.id_mapping.insert(id, segment_id);
+ self.rev_id_mapping.insert(segment_id, id);
- self.adjacency.insert(from_id, to_id, label.clone());
- self.reversed_adjacency.insert(to_id, from_id, label);
+ segment_id
+ }
}
- pub fn flush(&mut self) {
- self.adjacency.tree.flush();
- self.reversed_adjacency.tree.flush();
- self.id_mapping.flush();
- self.rev_id_mapping.flush();
- self.meta.flush();
- }
+ pub fn insert(&mut self, from: NodeID, to: NodeID, label: String) {
+ let from = self.get_or_create_id(from);
+ let to = self.get_or_create_id(to);
- pub fn id(&self) -> String {
- self.id.clone()
- }
+ self.adjacency.entry(from).or_default().insert(StoredEdge {
+ other: to,
+ label: label.clone(),
+ });
- pub fn path(&self) -> &Path {
- Path::new(&self.path)
+ self.reversed_adjacency
+ .entry(to)
+ .or_default()
+ .insert(StoredEdge { other: from, label });
}
- pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {
- self.adjacency
- .tree
- .iter()
- .flat_map(move |(node_id, edges)| {
- let from = self.rev_id_mapping(&node_id).unwrap();
-
- edges.into_iter().map(move |stored_edge| Edge {
- from,
- to: self.rev_id_mapping(&stored_edge.other).unwrap(),
- label: stored_edge.label,
- })
- })
- }
+ pub fn commit<P: AsRef<Path>>(self, folder_path: P) -> StoredSegment {
+ let segment_id = uuid::Uuid::new_v4().to_string();
+ let path = folder_path.as_ref().join(&segment_id);
- pub fn merge(&mut self, mut other: Segment<S>) {
- other.flush();
+ let mut adjacency = Store::open(path.join("adjacency"), &segment_id).unwrap();
- for edge in other.edges() {
- self.insert(edge.from, edge.to, edge.label);
+ for (from, edges) in self.adjacency {
+ adjacency.insert(from, &edges).unwrap();
}
- self.flush();
- }
-
- pub fn update_id_mapping(&mut self, mapping: Vec<(NodeID, NodeID)>) {
- let mut new_mappings = Vec::with_capacity(mapping.len());
+ let mut reversed_adjacency =
+ Store::open(path.join("reversed_adjacency"), &segment_id).unwrap();
- for (old_id, new_id) in mapping {
- if let Some(segment_id) = self.id_mapping(&old_id) {
- self.id_mapping.delete(&old_id);
- self.rev_id_mapping.delete(&segment_id);
-
- new_mappings.push((segment_id, new_id));
- }
+ for (to, edges) in self.reversed_adjacency {
+ reversed_adjacency.insert(to, &edges).unwrap();
}
- for (segment_id, new_id) in new_mappings {
- self.id_mapping.insert(new_id, segment_id);
- self.rev_id_mapping.insert(segment_id, new_id);
- }
+ let mut stored_segment = StoredSegment {
+ adjacency,
+ reversed_adjacency,
+ id_mapping: self.id_mapping,
+ rev_id_mapping: self.rev_id_mapping,
+ meta: Meta {
+ num_nodes: self.next_id,
+ },
+ folder_path: path.as_os_str().to_str().unwrap().to_string(),
+ id: segment_id,
+ };
- self.flush();
- }
-}
+ stored_segment.flush();
-impl Store for RocksDbStore {
- fn open<P: AsRef<std::path::Path>>(path: P, id: String) -> Segment<Self> {
- let adjacency = RocksDbStore::open(path.as_ref().join("adjacency"));
- let reversed_adjacency = RocksDbStore::open(path.as_ref().join("reversed_adjacency"));
- let id_mapping = RocksDbStore::open(path.as_ref().join("id_mapping"));
- let rev_id_mapping = RocksDbStore::open(path.as_ref().join("rev_id_mapping"));
- let meta = RocksDbStore::open(path.as_ref().join("meta"));
-
- Segment {
- adjacency: Adjacency {
- tree: CachedStore::new(adjacency, 100),
- },
- reversed_adjacency: Adjacency {
- tree: CachedStore::new(reversed_adjacency, 100),
- },
- id_mapping: CachedStore::new(id_mapping, 100),
- rev_id_mapping: CachedStore::new(rev_id_mapping, 100),
- meta: CachedStore::new(meta, 1_000),
- store: Default::default(),
- path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
- id,
- }
+ stored_segment
}
- fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Segment<Self> {
- let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));
- let reversed_adjacency =
- RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));
- let id_mapping = RocksDbStore::open_read_only(path.as_ref().join("id_mapping"));
- let rev_id_mapping = RocksDbStore::open_read_only(path.as_ref().join("rev_id_mapping"));
- let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));
-
- Segment {
- adjacency: Adjacency {
- tree: CachedStore::new(adjacency, 100),
- },
- reversed_adjacency: Adjacency {
- tree: CachedStore::new(reversed_adjacency, 100),
- },
- id_mapping: CachedStore::new(id_mapping, 100),
- rev_id_mapping: CachedStore::new(rev_id_mapping, 100),
- meta: CachedStore::new(meta, 1_000),
- store: Default::default(),
- path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
- id,
- }
+ pub fn is_empty(&self) -> bool {
+ self.id_mapping.is_empty()
}
}
@@ -381,7 +479,7 @@ mod test {
// ▼ │ │
// 1─────►2◄┘
- let mut store: Segment<RocksDbStore> = Segment::temporary();
+ let mut store = LiveSegment::default();
let a = NodeID(0);
let b = NodeID(1);
@@ -392,10 +490,11 @@ mod test {
store.insert(c, a, String::new());
store.insert(a, c, String::new());
- store.flush();
+ let store: StoredSegment = store.commit(crate::gen_temp_path());
+
+ let mut out: Vec<Edge> = store.outgoing_edges(&a, false);
- let mut out = store.outgoing_edges(&a);
- out.sort();
+ out.sort_by(|a, b| a.to.cmp(&b.to));
assert_eq!(
out,
@@ -403,61 +502,74 @@ mod test {
Edge {
from: a,
to: b,
- label: String::new()
+ label: Loaded::NotYet
},
Edge {
from: a,
to: c,
- label: String::new()
+ label: Loaded::NotYet
},
]
);
- let mut out = store.outgoing_edges(&b);
- out.sort();
+ let mut out: Vec<Edge> = store.outgoing_edges(&b, false);
+ out.sort_by(|a, b| a.to.cmp(&b.to));
assert_eq!(
out,
vec![Edge {
from: b,
to: c,
- label: String::new()
+ label: Loaded::NotYet
},]
);
- let mut out = store.ingoing_edges(&c);
- out.sort();
+ let mut out: Vec<Edge> = store.outgoing_edges(&c, false);
+ out.sort_by(|a, b| a.to.cmp(&b.to));
assert_eq!(
out,
- vec![
- Edge {
- from: a,
- to: c,
- label: String::new()
- },
- Edge {
- from: b,
- to: c,
- label: String::new()
- },
- ]
+ vec![Edge {
+ from: c,
+ to: a,
+ label: Loaded::NotYet
+ },]
);
+ let out: Vec<Edge> = store.ingoing_edges(&a, false);
assert_eq!(
- store.ingoing_edges(&a),
+ out,
vec![Edge {
from: c,
to: a,
- label: String::new()
+ label: Loaded::NotYet
},]
);
+ let out: Vec<Edge> = store.ingoing_edges(&b, false);
assert_eq!(
- store.ingoing_edges(&b),
+ out,
vec![Edge {
from: a,
to: b,
- label: String::new()
+ label: Loaded::NotYet
},]
);
+
+ let mut out: Vec<Edge> = store.ingoing_edges(&c, false);
+ out.sort_by(|a, b| a.from.cmp(&b.from));
+ assert_eq!(
+ out,
+ vec![
+ Edge {
+ from: a,
+ to: c,
+ label: Loaded::NotYet
+ },
+ Edge {
+ from: b,
+ to: c,
+ label: Loaded::NotYet
+ },
+ ]
+ );
}
}
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
new file mode 100644
index 00000000..88b8a158
--- /dev/null
+++ b/core/src/webgraph/store.rs
@@ -0,0 +1,295 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ collections::BTreeMap,
+ fs::OpenOptions,
+ io::BufReader,
+ ops::Range,
+ path::{Path, PathBuf},
+};
+
+use rkyv::{ser::serializers::AllocSerializer, Archived};
+
+type Result<T> = std::result::Result<T, Error>;
+
+const SCRATCH_SIZE: usize = 4096;
+const LOCATIONS_NAME: &str = "locations.bin";
+const MMAP_NAME: &str = "data.bin";
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error("key already exists")]
+ KeyExists,
+
+ #[error("io error: {0}")]
+ Io(std::io::Error),
+
+ #[error("bincode error: {0}")]
+ Bincode(bincode::Error),
+}
+
+pub struct Store<K, V> {
+ locations: BTreeMap<K, Range<usize>>,
+ folder_path: PathBuf,
+ cur_end_range: usize,
+ map: memmap2::MmapMut,
+ has_new_inserts: bool,
+ segment_name: String,
+ _values: std::marker::PhantomData<V>,
+}
+
+impl<K, V> Store<K, V>
+where
+ K: serde::de::DeserializeOwned + serde::Serialize + Ord,
+ V: rkyv::Archive,
+{
+ pub fn open<P: AsRef<Path>>(folder_path: P, segment_name: &str) -> Result<Self> {
+ if folder_path.as_ref().is_file() {
+ return Err(Error::Io(std::io::Error::new(
+ std::io::ErrorKind::NotFound,
+ format!("{} is not a directory", folder_path.as_ref().display()),
+ )));
+ }
+
+ std::fs::create_dir_all(folder_path.as_ref()).map_err(Error::Io)?;
+
+ let locations_path = folder_path
+ .as_ref()
+ .join(format!("{segment_name}-{LOCATIONS_NAME}"));
+ let f = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(locations_path)
+ .map_err(Error::Io)?;
+
+ let mut reader = BufReader::new(f);
+ let locations: BTreeMap<K, Range<usize>> =
+ bincode::deserialize_from(&mut reader).unwrap_or_default();
+
+ let cur_end_range = locations.values().map(|range| range.end).max().unwrap_or(0);
+
+ let mmap_path = folder_path
+ .as_ref()
+ .join(format!("{segment_name}-{MMAP_NAME}"));
+ let map = unsafe {
+ let file = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(mmap_path)
+ .map_err(Error::Io)?;
+
+ memmap2::MmapOptions::new()
+ .map_mut(&file)
+ .map_err(Error::Io)?
+ };
+
+ Ok(Self {
+ locations,
+ cur_end_range,
+ map,
+ folder_path: folder_path.as_ref().to_path_buf(),
+ has_new_inserts: false,
+ _values: std::marker::PhantomData,
+ segment_name: segment_name.to_string(),
+ })
+ }
+
+ pub fn get(&self, key: &K) -> Option<&Archived<V>> {
+ let range = self.locations.get(key)?;
+ let bytes = &self.map[range.clone()];
+
+ let archived = unsafe { rkyv::archived_root::<V>(bytes) };
+ Some(archived)
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = (&K, &Archived<V>)> + '_ {
+ self.locations.iter().map(move |(key, range)| {
+ let bytes = &self.map[range.clone()];
+ let archived = unsafe { rkyv::archived_root::<V>(bytes) };
+ (key, archived)
+ })
+ }
+
+ pub fn insert(&mut self, key: K, value: &V) -> Result<()>
+ where
+ V: rkyv::Serialize<AllocSerializer<SCRATCH_SIZE>>,
+ {
+ if self.locations.contains_key(&key) {
+ return Err(Error::KeyExists);
+ }
+
+ let bytes = rkyv::to_bytes::<_, SCRATCH_SIZE>(value).unwrap();
+ self.insert_raw(key, &bytes)
+ }
+
+ fn insert_raw(&mut self, key: K, bytes: &[u8]) -> Result<()> {
+ let old_end = self.cur_end_range;
+ let new_end = bytes.len() + self.cur_end_range;
+
+ if new_end > self.map.len() {
+ self.resize(new_end)?;
+ }
+
+ self.map[old_end..new_end].copy_from_slice(bytes);
+ self.locations.insert(key, old_end..new_end);
+ self.cur_end_range = new_end;
+ self.has_new_inserts = true;
+
+ Ok(())
+ }
+
+ fn resize(&mut self, size: usize) -> Result<()> {
+ if size > self.map.len() {
+ self.flush_map()?;
+
+ let file = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(
+ self.folder_path
+ .join(format!("{}-{MMAP_NAME}", self.segment_name)),
+ )
+ .map_err(Error::Io)?;
+
+ file.set_len(size as u64).map_err(Error::Io)?;
+
+ self.map = unsafe {
+ memmap2::MmapOptions::new()
+ .map_mut(&file)
+ .map_err(Error::Io)?
+ };
+ }
+
+ Ok(())
+ }
+
+ fn flush_map(&mut self) -> Result<()> {
+ if self.has_new_inserts {
+ self.map.flush().map_err(Error::Io)?;
+ }
+
+ Ok(())
+ }
+
+ pub fn flush(&mut self) -> Result<()> {
+ if self.has_new_inserts {
+ self.flush_map()?;
+
+ let f = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(
+ self.folder_path
+ .join(format!("{}-{}", self.segment_name, LOCATIONS_NAME)),
+ )
+ .map_err(Error::Io)?;
+
+ bincode::serialize_into(f, &self.locations).map_err(Error::Bincode)?;
+ }
+ self.has_new_inserts = false;
+
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use rkyv::Archive;
+
+ use super::*;
+
+ #[derive(Debug, PartialEq, Eq, Clone, Archive, rkyv::Serialize, rkyv::Deserialize)]
+ #[archive(compare(PartialEq))]
+ #[archive_attr(derive(Debug))]
+ struct TestStruct {
+ a: String,
+ b: i32,
+ }
+
+ #[test]
+ fn test_insert() {
+ let mut kv =
+ Store::<String, TestStruct>::open(crate::gen_temp_path(), "test-segment").unwrap();
+
+ assert!(kv.get(&"test".to_string()).is_none());
+
+ let test_struct = TestStruct {
+ a: "test".to_string(),
+ b: 5,
+ };
+
+ kv.insert("test".to_string(), &test_struct).unwrap();
+ kv.flush().unwrap();
+
+ let archived = kv.get(&"test".to_string()).unwrap();
+ assert_eq!(&test_struct, archived);
+ }
+
+ #[test]
+ fn test_re_open() {
+ let path = crate::gen_temp_path();
+ let segment_name = "test-segment";
+ let test_struct = TestStruct {
+ a: "test".to_string(),
+ b: 5,
+ };
+
+ {
+ let mut kv = Store::<String, TestStruct>::open(path.clone(), segment_name).unwrap();
+
+ assert!(kv.get(&"test".to_string()).is_none());
+
+ kv.insert("test".to_string(), &test_struct).unwrap();
+ kv.flush().unwrap();
+ }
+
+ let kv = Store::<String, TestStruct>::open(path, segment_name).unwrap();
+
+ let archived = kv.get(&"test".to_string()).unwrap();
+ assert_eq!(&test_struct, archived);
+ }
+
+ #[test]
+ fn test_insert_existing_key() {
+ let mut kv =
+ Store::<String, TestStruct>::open(crate::gen_temp_path(), "test-segment").unwrap();
+
+ assert!(kv.get(&"test".to_string()).is_none());
+
+ let test_struct = TestStruct {
+ a: "test".to_string(),
+ b: 5,
+ };
+
+ kv.insert("test".to_string(), &test_struct).unwrap();
+ kv.flush().unwrap();
+
+ let archived = kv.get(&"test".to_string()).unwrap();
+ assert_eq!(&test_struct, archived);
+
+ let test_struct = TestStruct {
+ a: "test".to_string(),
+ b: 6,
+ };
+
+ assert!(kv.insert("test".to_string(), &test_struct).is_err());
+ }
+}
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index df84b831..593ba072 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -14,6 +14,38 @@ TOP_N_RESULTS = 10
openai.api_key = os.getenv("OPENAI_API_KEY")
+"""
+NEW PROMPT:
+
+You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a reputable source. The score should be between 0.0 and 1.0.
+
+query: england politics
+
+result: {domain: "https://abingdonrowcondos.com", title: "Gareth Southgate hails hat-trick hero Harry Kane’s return to form", snippet: "Plus, Nesbitt Realty will pay a down-payment assistance of $1,903 toward your funds for your down-payment. Please get in touch with Will Nesbitt for learn more about our first-time buyer's credit or 607 Bashford Ln #1 or any ."}
+
+score: 0.7
+
+query: q learning algorithm
+
+result: {domain: "datascience.stackexchange.com", title: "Q-learning why do we subtract the Q(s, a) term during update? - Data Science Stack Exchange", snippet: "The q-learning algorithm is an off-policy algorithm , unlike SARSA . The Bellman equation describes q-learning as follows: "The q value for action $a$ taken in state $s$ at time $t$ becomes equal to: that same q-value plus small amount of: currently"}
+
+score: 1.0
+
+query: learn python
+
+result: {domain: "jqxxj.fulltime-abbigliamento.it", title: "Vim plugins python", snippet: "In fact, VimL can be learned fast, but using. omplete Python syntax highlighter for Vim. shows what I get with default Vim that illustrates … Upgrade to PyCharm, the leading Python IDE: best in class debugging, code navigation, refactoring."}
+
+score: 0.1
+
+query: savings interest formula
+
+result: {domain: "readingrockets.org", title: "Choosing Words to Teach | Reading Rockets", snippet: "15. Oct. 2020 Only a sobering revelation (that essential detail) saves him, his marriage, and his wife. A couple of points should be emphasized here. The words were selected not so much because they are essential to comprehension of the story but because they seem..."}
+
+score:
+
+"""
+
+
def search(q):
url = f"http://{STRACT_ENDPOINT}/beta/api/search"
r = requests.post(
diff --git a/optics/testcases b/optics/testcases
index afa95e5d..d80320e3 160000
--- a/optics/testcases
+++ b/optics/testcases
@@ -1 +1 @@
-Subproject commit afa95e5d018e7c4b53c48bdc02cf7712de6d21ba
+Subproject commit d80320e33b78ddb233cfec56af498f0703d34b12