Skip to content

web search done right

Huge refactor of webgraph to split graph into multiple segments. This allows for more parallelism

Mikkel Denkere4a27f27a0a0
diff --git a/Cargo.lock b/Cargo.lock
index d1cdee35..a4f69b8d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -142,7 +142,7 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
- "hermit-abi",
+ "hermit-abi 0.1.19",
"libc",
"winapi",
]
@@ -602,9 +602,9 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
-version = "0.5.4"
+version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
dependencies = [
"cfg-if",
"crossbeam-utils",
@@ -727,6 +727,7 @@ dependencies = [
"chrono",
"clap 3.1.18",
"criterion",
+ "crossbeam-channel",
"csv",
"encoding_rs",
"flate2",
@@ -743,6 +744,7 @@ dependencies = [
"md5",
"memmap",
"min-max-heap",
+ "num_cpus",
"once_cell",
"optics",
"parse_wiki_text",
@@ -1248,6 +1250,15 @@ dependencies = [
"libc",
]
+[[package]]
+name = "hermit-abi"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "hex"
version = "0.4.3"
@@ -1947,11 +1958,11 @@ dependencies = [
[[package]]
name = "num_cpus"
-version = "1.13.1"
+version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
dependencies = [
- "hermit-abi",
+ "hermit-abi 0.2.6",
"libc",
]
diff --git a/Cargo.toml b/Cargo.toml
index 90b6d062..684df09b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -63,3 +63,5 @@ bitvec = "1.0.1"
criterion = "0.3.6"
maplit = "1.0.2"
rand = "0.8.5"
+crossbeam-channel = "0.5.6"
+num_cpus = "1.15.0"
\ No newline at end of file
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 3ed652fa..6586779f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -67,6 +67,8 @@ bitvec = { workspace = true }
tower-http = { workspace = true }
once_cell = { workspace = true }
optics = { path = "../optics" }
+crossbeam-channel = { workspace = true }
+num_cpus = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
@@ -86,5 +88,5 @@ name = "harmonic-centrality"
harness = false
[[bench]]
-name = "approximate-harmonic"
-harness = false
\ No newline at end of file
+name = "online-harmonic"
+harness = false
diff --git a/core/benches/approximate-harmonic.rs b/core/benches/approximate-harmonic.rs
deleted file mode 100644
index b8a26353..00000000
--- a/core/benches/approximate-harmonic.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-use criterion::{criterion_group, criterion_main, Criterion};
-use cuely::webgraph::{
- centrality::approximate_harmonic::ApproximatedHarmonicCentrality, WebgraphBuilder,
-};
-
-const WEBGRAPH_PATH: &str = "data/webgraph";
-
-pub fn criterion_benchmark(c: &mut Criterion) {
- let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).with_host_graph().open();
- c.bench_function("Approximated harmonic centrality calculation", |b| {
- b.iter(|| {
- for _ in 0..10 {
- ApproximatedHarmonicCentrality::new(&webgraph);
- }
- })
- });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
diff --git a/core/benches/harmonic-centrality.rs b/core/benches/harmonic-centrality.rs
index 5bd8d058..7540b0d3 100644
--- a/core/benches/harmonic-centrality.rs
+++ b/core/benches/harmonic-centrality.rs
@@ -4,7 +4,7 @@ use cuely::webgraph::{centrality::harmonic::HarmonicCentrality, WebgraphBuilder}
const WEBGRAPH_PATH: &str = "data/webgraph";
pub fn criterion_benchmark(c: &mut Criterion) {
- let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).with_host_graph().open();
+ let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).open();
c.bench_function("Harmonic centrality calculation", |b| {
b.iter(|| {
for _ in 0..10 {
diff --git a/core/benches/online-harmonic.rs b/core/benches/online-harmonic.rs
new file mode 100644
index 00000000..0dacbc7b
--- /dev/null
+++ b/core/benches/online-harmonic.rs
@@ -0,0 +1,18 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use cuely::webgraph::{centrality::online_harmonic::OnlineHarmonicCentrality, WebgraphBuilder};
+
+const WEBGRAPH_PATH: &str = "data/webgraph";
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+ let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).open();
+ c.bench_function("Online harmonic centrality calculation", |b| {
+ b.iter(|| {
+ for _ in 0..10 {
+ OnlineHarmonicCentrality::new(&webgraph);
+ }
+ })
+ });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/core/examples/print_approx_harmonic_centrality.rs b/core/examples/print_approx_harmonic_centrality.rs
index 078e1a8f..b11d723e 100644
--- a/core/examples/print_approx_harmonic_centrality.rs
+++ b/core/examples/print_approx_harmonic_centrality.rs
@@ -21,7 +21,7 @@ use std::{
use cuely::{
ranking::centrality_store::CentralityStore,
- webgraph::{centrality::approximate_harmonic::Scorer, Node, WebgraphBuilder},
+ webgraph::{centrality::online_harmonic::Scorer, Node, NodeID, WebgraphBuilder},
webpage::Url,
};
@@ -50,13 +50,13 @@ impl Ord for ScoredNode {
impl Eq for ScoredNode {}
-fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[u64]) -> Vec<ScoredNode> {
+fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[NodeID]) -> Vec<ScoredNode> {
let mut top_nodes = BinaryHeap::with_capacity(top_n);
for node in nodes {
top_nodes.push(cmp::Reverse(ScoredNode {
score: scorer.score(*node),
- id: *node,
+ id: node.0,
}));
while top_nodes.len() > top_n {
@@ -72,12 +72,12 @@ fn get_top_nodes(scorer: Scorer, top_n: usize, nodes: &[u64]) -> Vec<ScoredNode>
nodes
}
-fn print_nodes(nodes: &[ScoredNode], id2node: &HashMap<u64, Node>) {
+fn print_nodes(nodes: &[ScoredNode], id2node: &HashMap<NodeID, Node>) {
for (i, node) in nodes.iter().rev().enumerate() {
println!(
"{i} \t {:.3} \t {} ",
node.score,
- id2node.get(&node.id).unwrap().name
+ id2node.get(&(node.id.into())).unwrap().name
);
}
}
@@ -86,15 +86,15 @@ fn print_top_nodes(
liked_sites: &[&str],
top_n: usize,
store: &CentralityStore,
- id2node: &HashMap<u64, Node>,
- nodes: &[u64],
+ id2node: &HashMap<NodeID, Node>,
+ nodes: &[NodeID],
) {
let liked_nodes: Vec<_> = liked_sites
.iter()
.map(|host| Node::from_url(&Url::from(host.to_string())))
.collect();
- let scorer = store.approx_harmonic.scorer(&liked_nodes, &[]);
+ let scorer = store.online_harmonic.scorer(&liked_nodes, &[]);
let top_nodes = get_top_nodes(scorer, top_n, nodes);
println!(
"top {} sites for these liked sites {:?}",
@@ -108,20 +108,17 @@ fn print_top_nodes(
pub fn main() {
const TOP_N: usize = 50;
- let graph = WebgraphBuilder::new("data/webgraph")
- .with_host_graph()
- .read_only(true)
- .open();
- let nodes: Vec<_> = graph.host.as_ref().unwrap().nodes().collect();
+ let graph = WebgraphBuilder::new("data/webgraph").read_only(true).open();
+ let nodes: Vec<_> = graph.nodes().collect();
let id2node: HashMap<_, _> = nodes
.iter()
- .map(|id| (*id, graph.host.as_ref().unwrap().id2node(id).unwrap()))
+ .map(|id| (*id, graph.id2node(id).unwrap()))
.collect();
let store = CentralityStore::open("data/centrality");
let mut proxy_nodes = Vec::new();
- for node in &store.approx_harmonic.proxy_nodes {
+ for node in &store.online_harmonic.proxy_nodes {
proxy_nodes.push(id2node.get(&node.id).unwrap().name.clone());
}
diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rs
index 0a1a4f6a..0b11f933 100644
--- a/core/examples/print_inbound_links.rs
+++ b/core/examples/print_inbound_links.rs
@@ -17,10 +17,7 @@
use cuely::webgraph::{Node, WebgraphBuilder};
pub fn main() {
- let graph = WebgraphBuilder::new("data/webgraph")
- .with_host_graph()
- .read_only(true)
- .open();
+ let graph = WebgraphBuilder::new("data/webgraph").read_only(true).open();
for host in [
"plapcatesq.ga",
@@ -29,7 +26,7 @@ pub fn main() {
"GroomersNew.ga",
] {
println!("{}:", host);
- for edge in graph.ingoing_edges_host(Node::from(host)) {
+ for edge in graph.ingoing_edges(Node::from(host)) {
println!("{} -> {} ({})", edge.from.name, edge.to.name, edge.label);
}
println!();
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index c6aafa36..d306b1e3 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -22,10 +22,7 @@ pub struct Centrality {}
impl Centrality {
pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
- let graph = WebgraphBuilder::new(webgraph_path)
- .read_only(true)
- .with_host_graph()
- .open();
+ let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();
CentralityStore::build(&graph, output_path);
}
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index ad7468d6..2fab4029 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -91,7 +91,6 @@ fn create_webgraph() -> Result<()> {
}),
warc_paths: vec![warc_path.to_str().unwrap().to_string()],
graph_base_path: out_path_tmp.to_str().unwrap().to_string(),
- create_full_graph: true,
};
let graph = webgraph::process_job(&job);
@@ -196,9 +195,9 @@ fn create_topic_centrality() {
.to_str()
.unwrap()
.to_string();
- let approximate_harmonic_path = Path::new(DATA_PATH)
+ let online_harmonic_path = Path::new(DATA_PATH)
.join("centrality")
- .join("approx_harmonic")
+ .join("online_harmonic")
.to_str()
.unwrap()
.to_string();
@@ -212,7 +211,7 @@ fn create_topic_centrality() {
index_path,
topics_path,
webgraph_path,
- approximate_harmonic_path,
+ online_harmonic_path,
output_path,
);
}
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 848c5d6a..d8fd2b7e 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -72,13 +72,7 @@ impl IndexingWorker {
) -> Self {
Self {
centrality_store: CentralityStore::open(centrality_store_path),
- webgraph: webgraph_path.map(|path| {
- WebgraphBuilder::new(path)
- .with_full_graph()
- .with_host_graph()
- .read_only(true)
- .open()
- }),
+ webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).read_only(true).open()),
crawl_stabilty: crawl_stability_path.map(CrawlStability::open),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
}
@@ -176,10 +170,10 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
let node_id = worker
.centrality_store
- .approx_harmonic
+ .online_harmonic
.node2id
.get(&Node::from_url(html.url()).into_host())
- .copied();
+ .cloned();
let mut host_topic = None;
let mut dmoz_description = None;
diff --git a/core/src/entrypoint/topic_centrality.rs b/core/src/entrypoint/topic_centrality.rs
index b925dcb4..9c2a7cbc 100644
--- a/core/src/entrypoint/topic_centrality.rs
+++ b/core/src/entrypoint/topic_centrality.rs
@@ -18,9 +18,7 @@ use crate::{
human_website_annotations,
index::Index,
webgraph::{
- centrality::{
- approximate_harmonic::ApproximatedHarmonicCentrality, topic::TopicCentrality,
- },
+ centrality::{online_harmonic::OnlineHarmonicCentrality, topic::TopicCentrality},
WebgraphBuilder,
},
};
@@ -29,15 +27,15 @@ pub fn run(
index_path: String,
topics_path: String,
webgraph_path: String,
- approximate_harmonic_path: String,
+ online_harmonic_path: String,
output_path: String,
) {
let index = Index::open(index_path).unwrap();
let topics = human_website_annotations::Mapper::open(topics_path).unwrap();
- let webgraph = WebgraphBuilder::new(webgraph_path).with_host_graph().open();
- let approx = ApproximatedHarmonicCentrality::open(approximate_harmonic_path).unwrap();
+ let webgraph = WebgraphBuilder::new(webgraph_path).open();
+ let harmonic = OnlineHarmonicCentrality::open(online_harmonic_path).unwrap();
- let centrality: TopicCentrality = TopicCentrality::build(&index, topics, webgraph, approx);
+ let centrality: TopicCentrality = TopicCentrality::build(&index, topics, webgraph, harmonic);
centrality.save(output_path).unwrap();
}
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 8fcde02e..f81d4915 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -32,7 +32,6 @@ use tracing::{info, trace};
#[derive(Debug, Serialize, Deserialize, Clone)]
struct GraphPointer {
path: String,
- with_full_path: bool,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -46,17 +45,10 @@ pub struct Job {
pub config: JobConfig,
pub warc_paths: Vec<String>,
pub graph_base_path: String,
- pub create_full_graph: bool,
}
-fn open_graph<P: AsRef<Path>>(path: P, with_full_graph: bool) -> webgraph::Webgraph {
- let mut builder = WebgraphBuilder::new(path).with_host_graph();
-
- if with_full_graph {
- builder = builder.with_full_graph()
- }
-
- builder.open()
+fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
+ WebgraphBuilder::new(path).open()
}
async fn async_process_job(job: &Job) -> webgraph::Webgraph {
@@ -64,10 +56,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {
info!("processing {}", name);
- let mut graph = open_graph(
- Path::new(&job.graph_base_path).join(name),
- job.create_full_graph,
- );
+ let mut graph = open_graph(Path::new(&job.graph_base_path).join(name));
let source = match job.config.clone() {
JobConfig::Http(config) => WarcSource::HTTP(config),
@@ -104,7 +93,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {
}
}
- graph.flush();
+ graph.commit();
std::fs::remove_file(path).unwrap();
}
@@ -132,10 +121,7 @@ impl Map<StatelessWorker, FrozenWebgraph> for Job {
impl Map<StatelessWorker, GraphPointer> for Job {
fn map(&self, _worker: &StatelessWorker) -> GraphPointer {
let graph = process_job(self);
- GraphPointer {
- path: graph.path,
- with_full_path: self.create_full_graph,
- }
+ GraphPointer { path: graph.path }
}
}
@@ -162,8 +148,8 @@ impl Reduce<GraphPointer> for GraphPointer {
let self_clone = self.clone();
{
- let graph = open_graph(self.path, self.with_full_path);
- let other_graph = open_graph(other.path, self.with_full_path);
+ let graph = open_graph(self.path);
+ let other_graph = open_graph(other.path);
let _ = graph.reduce(other_graph);
}
@@ -174,7 +160,7 @@ impl Reduce<GraphPointer> for GraphPointer {
impl Reduce<GraphPointer> for webgraph::Webgraph {
fn reduce(self, other: GraphPointer) -> Self {
- let other = open_graph(other.path, other.with_full_path);
+ let other = open_graph(other.path);
self.reduce(other)
}
}
@@ -215,7 +201,6 @@ impl Webgraph {
.graph_base_path
.clone()
.unwrap_or_else(|| "data/webgraph".to_string()),
- create_full_graph: config.create_full_graph.unwrap_or(true),
})
.collect::<Vec<_>>()
.into_iter(),
@@ -276,7 +261,6 @@ impl Webgraph {
.graph_base_path
.clone()
.unwrap_or_else(|| "data/webgraph".to_string()),
- create_full_graph: config.create_full_graph.unwrap_or(true),
})
.collect_vec()
.into_par_iter()
@@ -285,7 +269,8 @@ impl Webgraph {
if graphs.len() > 1 {
let pointer = graphs.pop().unwrap();
- let mut graph = open_graph(pointer.path, pointer.with_full_path);
+ let mut graph = open_graph(pointer.path);
+
for other in graphs {
graph = graph.reduce(other);
}
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 5ddbe4c0..9016d33a 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -224,7 +224,7 @@ impl InvertedIndex {
if id == u64::MAX {
Ok(None)
} else {
- Ok(Some(id))
+ Ok(Some(id.into()))
}
}
diff --git a/core/src/kv/mod.rs b/core/src/kv/mod.rs
index 1311325e..cd7a32d4 100644
--- a/core/src/kv/mod.rs
+++ b/core/src/kv/mod.rs
@@ -32,6 +32,14 @@ where
fn flush(&self);
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a>;
+ fn delete_raw(&mut self, key: &[u8]);
+
+ fn delete(&mut self, key: &K) {
+ let key_bytes = bincode::serialize(key).expect("failed to serialize key");
+
+ self.delete_raw(&key_bytes);
+ }
+
fn get(&self, key: &K) -> Option<V> {
let key_bytes = bincode::serialize(key).expect("failed to serialize key");
@@ -75,6 +83,10 @@ where
unimplemented!()
}
+ fn delete_raw(&mut self, key: &[u8]) {
+ unimplemented!()
+ }
+
fn flush(&self) {}
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a> {
@@ -88,4 +100,8 @@ where
fn insert(&self, key: K, val: V) {
self.0.write().unwrap().insert(key, val);
}
+
+ fn delete(&mut self, key: &K) {
+ self.0.write().unwrap().remove(key);
+ }
}
diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rs
index 41c7cccf..d2efae6f 100644
--- a/core/src/kv/rocksdb_store.rs
+++ b/core/src/kv/rocksdb_store.rs
@@ -36,7 +36,13 @@ impl RocksDbStore {
fs::create_dir_all(path.as_ref()).expect("faild to create dir");
}
- Box::new(DB::open_default(path).expect("unable to open rocks db"))
+ let mut options = Options::default();
+
+ options.create_if_missing(true);
+ options.set_max_open_files(1);
+ options.set_max_file_opening_threads(1);
+
+ Box::new(DB::open(&options, path).expect("unable to open rocks db"))
}
pub fn open_read_only<K, V, P>(path: P) -> Box<dyn Kv<K, V> + Send + Sync>
@@ -50,7 +56,10 @@ impl RocksDbStore {
}
let mut options = Options::default();
+
options.create_if_missing(true);
+ options.set_max_open_files(1);
+ options.set_max_file_opening_threads(1);
Box::new(DB::open_for_read_only(&options, path, false).expect("unable to open rocks db"))
}
@@ -87,6 +96,10 @@ where
value: PhantomData::default(),
})
}
+
+ fn delete_raw(&mut self, key: &[u8]) {
+ rocksdb::DB::delete(self, key).unwrap();
+ }
}
pub struct IntoIter<'a, K, V>
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 51c7d4a0..389190c2 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -99,7 +99,6 @@ pub struct WebgraphMasterConfig {
workers: Vec<String>,
graph_base_path: Option<String>,
batch_size: Option<usize>,
- create_full_graph: Option<bool>,
}
#[derive(Debug, Deserialize, Clone)]
@@ -108,7 +107,6 @@ pub struct WebgraphLocalConfig {
warc_source: WarcSource,
graph_base_path: Option<String>,
batch_size: Option<usize>,
- create_full_graph: Option<bool>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -238,6 +236,12 @@ pub enum Error {
#[error("The stackoverflow schema was not structured as expected")]
InvalidStackoverflowSchema,
+
+ #[error("Rayong thread pool builder")]
+ RayonThreadPool(#[from] rayon::ThreadPoolBuildError),
+
+ #[error("Internal error")]
+ InternalError(String),
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/core/src/main.rs b/core/src/main.rs
index 445fbb41..f09e84c9 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -73,7 +73,7 @@ enum Commands {
index_path: String,
topics_path: String,
webgraph_path: String,
- approximate_harmonic_path: String,
+ online_harmonic_path: String,
output_path: String,
},
#[cfg(feature = "dev")]
@@ -227,13 +227,13 @@ fn main() -> Result<()> {
index_path,
topics_path,
webgraph_path,
- approximate_harmonic_path,
+ online_harmonic_path,
output_path,
} => entrypoint::topic_centrality::run(
index_path,
topics_path,
webgraph_path,
- approximate_harmonic_path,
+ online_harmonic_path,
output_path,
),
}
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 67b512fb..eee53d67 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -594,10 +594,7 @@ mod tests {
fn liked_sites() {
let mut index = Index::temporary().expect("Unable to open index");
- let mut graph = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph = WebgraphBuilder::new_memory().open();
graph.insert(
Node::from("https://www.a.com").into_host(),
@@ -611,7 +608,7 @@ mod tests {
String::new(),
);
- graph.flush();
+ graph.commit();
let centrality_store = CentralityStore::build(&graph, gen_temp_path());
@@ -645,7 +642,7 @@ mod tests {
dmoz_description: None,
node_id: Some(
*centrality_store
- .approx_harmonic
+ .online_harmonic
.node2id
.get(&Node::from("www.a.com").into_host())
.unwrap(),
@@ -681,7 +678,7 @@ mod tests {
host_topic: None,
node_id: Some(
*centrality_store
- .approx_harmonic
+ .online_harmonic
.node2id
.get(&Node::from("www.b.com").into_host())
.unwrap(),
@@ -717,7 +714,7 @@ mod tests {
dmoz_description: None,
node_id: Some(
*centrality_store
- .approx_harmonic
+ .online_harmonic
.node2id
.get(&Node::from("www.c.com").into_host())
.unwrap(),
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 81ccbaff..e91ff2fd 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -19,9 +19,7 @@ use std::{fs::File, path::Path};
use crate::{
kv::{rocksdb_store::RocksDbStore, Kv},
webgraph::{
- centrality::{
- approximate_harmonic::ApproximatedHarmonicCentrality, harmonic::HarmonicCentrality,
- },
+ centrality::{harmonic::HarmonicCentrality, online_harmonic::OnlineHarmonicCentrality},
Webgraph,
},
};
@@ -47,7 +45,7 @@ impl HarmonicCentralityStore {
pub struct CentralityStore {
pub harmonic: HarmonicCentralityStore,
- pub approx_harmonic: ApproximatedHarmonicCentrality,
+ pub online_harmonic: OnlineHarmonicCentrality,
pub base_path: String,
}
@@ -55,39 +53,13 @@ impl CentralityStore {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
Self {
harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),
- approx_harmonic: ApproximatedHarmonicCentrality::open(
- path.as_ref().join("approx_harmonic"),
- )
- .ok()
- .unwrap_or_default(),
+ online_harmonic: OnlineHarmonicCentrality::open(path.as_ref().join("online_harmonic"))
+ .ok()
+ .unwrap_or_default(),
base_path: path.as_ref().to_str().unwrap().to_string(),
}
}
- fn store_full<P: AsRef<Path>>(
- output_path: P,
- store: &mut CentralityStore,
- harmonic_centrality: &HarmonicCentrality,
- ) {
- let csv_file = File::options()
- .write(true)
- .create(true)
- .truncate(true)
- .open(output_path.as_ref().join("harmonic_full.csv"))
- .unwrap();
- let mut wtr = csv::Writer::from_writer(csv_file);
-
- let mut full: Vec<_> = harmonic_centrality.full.clone().into_iter().collect();
- full.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
- for (node, centrality) in full {
- store.harmonic.full.insert(node.name.clone(), centrality);
- wtr.write_record(&[node.name, centrality.to_string()])
- .unwrap();
- }
-
- wtr.flush().unwrap();
- }
-
fn store_host<P: AsRef<Path>>(
output_path: P,
store: &mut CentralityStore,
@@ -116,15 +88,9 @@ impl CentralityStore {
let harmonic_centrality = HarmonicCentrality::calculate(graph);
- if graph.full.is_some() {
- Self::store_full(&output_path, &mut store, &harmonic_centrality)
- }
-
- if graph.host.is_some() {
- Self::store_host(&output_path, &mut store, &harmonic_centrality)
- }
+ Self::store_host(&output_path, &mut store, &harmonic_centrality);
- store.approx_harmonic = ApproximatedHarmonicCentrality::new(graph);
+ store.online_harmonic = OnlineHarmonicCentrality::new(graph);
store.flush();
@@ -134,8 +100,8 @@ impl CentralityStore {
pub fn flush(&self) {
self.harmonic.flush();
- self.approx_harmonic
- .save(Path::new(&self.base_path).join("approx_harmonic"))
+ self.online_harmonic
+ .save(Path::new(&self.base_path).join("online_harmonic"))
.unwrap();
}
}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 2de09e87..2ccea2c0 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -34,7 +34,7 @@ use crate::{
searcher::NUM_RESULTS_PER_PAGE,
webgraph::{
centrality::{
- approximate_harmonic::{self, ApproximatedHarmonicCentrality, Scorer},
+ online_harmonic::{self, OnlineHarmonicCentrality, Scorer},
topic,
},
Node,
@@ -141,14 +141,14 @@ impl Ranker {
self.aggregator.set_topic_scorer(topic_scorer);
}
- pub fn set_query_centrality(&mut self, approx: approximate_harmonic::Scorer) {
- self.aggregator.set_query_centrality(approx);
+ pub fn set_query_centrality(&mut self, harmonic: online_harmonic::Scorer) {
+ self.aggregator.set_query_centrality(harmonic);
}
}
pub fn online_centrality_scorer(
site_rankings: &SiteRankings,
- approx_harmonic: &ApproximatedHarmonicCentrality,
+ harmonic: &OnlineHarmonicCentrality,
) -> Scorer {
let mut liked_nodes = Vec::new();
let mut disliked_nodes = Vec::new();
@@ -161,7 +161,7 @@ pub fn online_centrality_scorer(
disliked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());
}
- approx_harmonic.scorer(&liked_nodes, &disliked_nodes)
+ harmonic.scorer(&liked_nodes, &disliked_nodes)
}
#[cfg(test)]
@@ -173,9 +173,7 @@ mod tests {
index::Index,
searcher::{LocalSearcher, SearchQuery},
webgraph::{
- centrality::{
- approximate_harmonic::ApproximatedHarmonicCentrality, topic::TopicCentrality,
- },
+ centrality::{online_harmonic::OnlineHarmonicCentrality, topic::TopicCentrality},
Node, WebgraphBuilder,
},
webpage::{Html, Link, Webpage},
@@ -1152,7 +1150,7 @@ mod tests {
page_centrality: 0.0,
primary_image: None,
host_topic: Some(topic_a.clone()),
- node_id: Some(1),
+ node_id: Some(1.into()),
dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -1187,7 +1185,7 @@ mod tests {
crawl_stability: 0.0,
primary_image: None,
host_topic: Some(topic_b.clone()),
- node_id: Some(2),
+ node_id: Some(2.into()),
dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -1222,7 +1220,7 @@ mod tests {
crawl_stability: 0.0,
primary_image: None,
host_topic: None,
- node_id: Some(0),
+ node_id: Some(0.into()),
dmoz_description: None,
})
.expect("failed to insert webpage");
@@ -1245,9 +1243,7 @@ mod tests {
);
let topics = human_website_annotations::Mapper::from(mapper);
- let mut webgraph = WebgraphBuilder::new(crate::gen_temp_path())
- .with_host_graph()
- .open();
+ let mut webgraph = WebgraphBuilder::new(crate::gen_temp_path()).open();
webgraph.insert(
Node::from("www.wrong.com"),
@@ -1260,11 +1256,11 @@ mod tests {
String::new(),
);
- webgraph.flush();
+ webgraph.commit();
- let approx = ApproximatedHarmonicCentrality::new(&webgraph);
+ let harmonic = OnlineHarmonicCentrality::new(&webgraph);
- let topic_centrality = TopicCentrality::build(&index, topics, webgraph, approx);
+ let topic_centrality = TopicCentrality::build(&index, topics, webgraph, harmonic);
let mut searcher = LocalSearcher::new(index);
searcher.set_topic_centrality(topic_centrality);
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index f677abec..66cd855c 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -69,10 +69,7 @@ mod tests {
fn site_rankings() {
let mut index = Index::temporary().expect("Unable to open index");
- let mut graph = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph = WebgraphBuilder::new_memory().open();
graph.insert(
Node::from("https://www.first.com").into_host(),
@@ -85,7 +82,7 @@ mod tests {
String::new(),
);
- graph.flush();
+ graph.commit();
index
.insert(Webpage {
@@ -116,9 +113,6 @@ mod tests {
host_topic: None,
node_id: Some(
graph
- .host
- .as_ref()
- .unwrap()
.node2id(&Node::from("https://www.first.com").into_host())
.unwrap(),
),
@@ -153,9 +147,6 @@ mod tests {
dmoz_description: None,
node_id: Some(
graph
- .host
- .as_ref()
- .unwrap()
.node2id(&Node::from("https://www.second.com").into_host())
.unwrap(),
),
@@ -190,9 +181,6 @@ mod tests {
host_topic: None,
node_id: Some(
graph
- .host
- .as_ref()
- .unwrap()
.node2id(&Node::from("https://www.third.com").into_host())
.unwrap(),
),
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index d001b5dc..f35e9751 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -83,7 +83,7 @@ impl RankingWebsite {
value.value.as_u64().unwrap() as f64 / FLOAT_SCALING as f64
}
Field::Fast(FastField::HostNodeID) => {
- let node = value.value.as_u64().unwrap();
+ let node = value.value.as_u64().unwrap().into();
res.personal_centrality = aggregator.personal_centrality(node);
res.topic_centrality = aggregator.topic_centrality(node).unwrap_or_default();
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 83937be1..3ef45e9c 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -17,9 +17,12 @@
use crate::{
fastfield_cache,
schema::{FastField, TextField},
- webgraph::centrality::{
- approximate_harmonic::{self, SHIFT},
- topic,
+ webgraph::{
+ centrality::{
+ online_harmonic::{self, SHIFT},
+ topic,
+ },
+ NodeID,
},
webpage::Webpage,
};
@@ -257,12 +260,12 @@ impl FieldBoost {
pub struct SignalAggregator {
fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,
signal_coefficients: SignalCoefficient,
- personal_centrality: Vec<Arc<approximate_harmonic::Scorer>>,
+ personal_centrality: Vec<Arc<online_harmonic::Scorer>>,
field_boost: FieldBoost,
fetch_time_ms_cache: [f64; 1000],
update_time_cache: Vec<f64>,
topic_scorer: Option<topic::Scorer>,
- query_centrality: Option<Arc<approximate_harmonic::Scorer>>,
+ query_centrality: Option<Arc<online_harmonic::Scorer>>,
}
impl std::fmt::Debug for SignalAggregator {
@@ -314,27 +317,27 @@ impl SignalAggregator {
self.topic_scorer = Some(topic_scorer);
}
- pub fn set_query_centrality(&mut self, query_centrality: approximate_harmonic::Scorer) {
+ pub fn set_query_centrality(&mut self, query_centrality: online_harmonic::Scorer) {
self.query_centrality = Some(Arc::new(query_centrality));
}
- pub fn add_personal_harmonic(&mut self, personal_centrality: approximate_harmonic::Scorer) {
+ pub fn add_personal_harmonic(&mut self, personal_centrality: online_harmonic::Scorer) {
self.personal_centrality.push(Arc::new(personal_centrality))
}
- pub fn topic_centrality(&self, host_id: u64) -> Option<f64> {
+ pub fn topic_centrality(&self, host_id: NodeID) -> Option<f64> {
self.topic_scorer
.as_ref()
.map(|scorer| scorer.score(host_id))
}
- pub fn query_centrality(&self, host_id: u64) -> Option<f64> {
+ pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {
self.query_centrality
.as_ref()
.map(|scorer| scorer.score(host_id) - SHIFT)
}
- pub fn personal_centrality(&self, host_id: u64) -> f64 {
+ pub fn personal_centrality(&self, host_id: NodeID) -> f64 {
self.personal_centrality
.iter()
.map(|scorer| scorer.score(host_id))
@@ -349,10 +352,12 @@ impl SignalAggregator {
current_timestamp: usize,
selected_region: Option<Region>,
) -> Score {
- let host_id = self
- .fastfield_cache
- .as_ref()
- .and_then(|cache| cache.get_doc_cache(&FastField::HostNodeID).get_u64(&doc));
+ let host_id = self.fastfield_cache.as_ref().and_then(|cache| {
+ cache
+ .get_doc_cache(&FastField::HostNodeID)
+ .get_u64(&doc)
+ .map(NodeID::from)
+ });
let topic_score = host_id.and_then(|host_id| self.topic_centrality(host_id));
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 94a88888..2b842c14 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -93,16 +93,14 @@ impl LocalSearcher {
})
.unwrap_or_default();
- if let (Some(optic), Some(approx_harmonic)) = (
+ if let (Some(optic), Some(harmonic)) = (
optic,
self.centrality_store
.as_ref()
- .map(|store| &store.approx_harmonic),
+ .map(|store| &store.online_harmonic),
) {
- query_aggregator.add_personal_harmonic(online_centrality_scorer(
- &optic.site_rankings,
- approx_harmonic,
- ));
+ query_aggregator
+ .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));
}
let mut parsed_query = Query::parse(
@@ -156,15 +154,13 @@ impl LocalSearcher {
.and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))
.unwrap_or_default();
- if let Some(approx_harmonic) = self
+ if let Some(harmonic) = self
.centrality_store
.as_ref()
- .map(|store| &store.approx_harmonic)
+ .map(|store| &store.online_harmonic)
{
- aggregator.add_personal_harmonic(online_centrality_scorer(
- &optic.site_rankings,
- approx_harmonic,
- ));
+ aggregator
+ .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));
}
let mut ranker = Ranker::new(
@@ -193,10 +189,10 @@ impl LocalSearcher {
let top_host_nodes = self.index.top_nodes(&parsed_query, ranker.collector())?;
if !top_host_nodes.is_empty() {
- let approx = centrality_store
- .approx_harmonic
+ let harmonic = centrality_store
+ .online_harmonic
.scorer_from_ids(&top_host_nodes, &[]);
- ranker.set_query_centrality(approx);
+ ranker.set_query_centrality(harmonic);
}
}
@@ -456,8 +452,6 @@ mod tests {
assert!(!urls.is_empty());
- dbg!(&urls);
-
for (i, url) in urls.into_iter().enumerate() {
assert_eq!(
url,
diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rs
index 77bb88e2..924972cd 100644
--- a/core/src/webgraph/centrality/betweenness.rs
+++ b/core/src/webgraph/centrality/betweenness.rs
@@ -24,15 +24,15 @@ use serde::{Deserialize, Serialize};
use crate::{
intmap::IntMap,
- webgraph::{graph_store::GraphStore, Node, NodeID, Store, Webgraph},
+ webgraph::{Node, NodeID, Webgraph},
};
-fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<Node, f64>, i32) {
+fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32) {
let mut centrality: HashMap<NodeID, f64> = HashMap::new();
let mut n = 0;
let mut max_dist = 0;
- let nodes: Vec<_> = store.nodes().take(100_000).collect();
+ let nodes: Vec<_> = graph.nodes().take(100_000).collect();
let pb =
if with_progress {
@@ -60,38 +60,38 @@ fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<N
let mut sigma = IntMap::new();
- sigma.insert(s, 1);
+ sigma.insert(s.0, 1);
let mut distances = IntMap::new();
- distances.insert(s, 0);
+ distances.insert(s.0, 0);
let mut q = VecDeque::new();
q.push_back(s);
while let Some(v) = q.pop_front() {
stack.push(v);
- for edge in store.outgoing_edges(v) {
+ for edge in graph.raw_outgoing_edges(&v) {
let w = edge.to;
- if !distances.contains(&w) {
- let dist_v = distances.get(&v).unwrap();
+ if !distances.contains(&w.0) {
+ let dist_v = distances.get(&v.0).unwrap();
q.push_back(w);
- distances.insert(w, dist_v + 1);
+ distances.insert(w.0, dist_v + 1);
}
- if *distances.get(&w).unwrap() == distances.get(&v).unwrap() + 1 {
- let sigma_v = *sigma.get(&v).unwrap_or(&0);
+ if *distances.get(&w.0).unwrap() == distances.get(&v.0).unwrap() + 1 {
+ let sigma_v = *sigma.get(&v.0).unwrap_or(&0);
- if !sigma.contains(&w) {
- sigma.insert(w, 0);
+ if !sigma.contains(&w.0) {
+ sigma.insert(w.0, 0);
}
- *sigma.get_mut(&w).unwrap() += sigma_v;
+ *sigma.get_mut(&w.0).unwrap() += sigma_v;
- if !predecessors.contains(&w) {
- predecessors.insert(w, Vec::new());
+ if !predecessors.contains(&w.0) {
+ predecessors.insert(w.0, Vec::new());
}
- predecessors.get_mut(&w).unwrap().push(v);
+ predecessors.get_mut(&w.0).unwrap().push(v.0);
}
}
}
@@ -100,20 +100,20 @@ fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<N
let mut delta = IntMap::new();
while let Some(w) = stack.pop() {
- if let Some(pred) = predecessors.get(&w) {
+ if let Some(pred) = predecessors.get(&w.0) {
for v in pred {
let dv = delta.get(v).copied().unwrap_or(0.0);
delta.insert(
*v,
- dv + (*sigma.get(v).unwrap() as f64 / *sigma.get(&w).unwrap() as f64)
- * (1.0 + delta.get(&w).unwrap_or(&0.0)),
+ dv + (*sigma.get(v).unwrap() as f64 / *sigma.get(&w.0).unwrap() as f64)
+ * (1.0 + delta.get(&w.0).unwrap_or(&0.0)),
);
}
}
if w != s {
- *centrality.entry(w).or_insert(0.0) += *delta.get(&w).unwrap_or(&0.0);
+ *centrality.entry(w).or_insert(0.0) += *delta.get(&w.0).unwrap_or(&0.0);
}
}
}
@@ -128,7 +128,7 @@ fn calculate<S: Store>(store: &GraphStore<S>, with_progress: bool) -> (HashMap<N
(
centrality
.into_iter()
- .map(|(id, centrality)| (store.id2node(&id).unwrap(), centrality / norm))
+ .map(|(id, centrality)| (graph.id2node(&id).unwrap(), centrality / norm))
.collect(),
max_dist,
)
@@ -143,34 +143,18 @@ pub struct Betweenness {
impl Betweenness {
#[allow(unused)]
pub fn calculate(graph: &Webgraph) -> Self {
- match &graph.host {
- Some(store) => {
- let (host, max_dist) = calculate(store, false);
- Self {
- centrality: host,
- max_dist: max_dist.max(0) as usize,
- }
- }
- None => Self {
- centrality: HashMap::new(),
- max_dist: 0,
- },
+ let (host, max_dist) = calculate(graph, false);
+ Self {
+ centrality: host,
+ max_dist: max_dist.max(0) as usize,
}
}
pub fn calculate_with_progress(graph: &Webgraph) -> Self {
- match &graph.host {
- Some(store) => {
- let (host, max_dist) = calculate(store, true);
- Self {
- centrality: host,
- max_dist: max_dist.max(0) as usize,
- }
- }
- None => Self {
- centrality: HashMap::new(),
- max_dist: 0,
- },
+ let (host, max_dist) = calculate(graph, true);
+ Self {
+ centrality: host,
+ max_dist: max_dist.max(0) as usize,
}
}
}
@@ -195,10 +179,7 @@ mod tests {
// │
// D
- let mut graph = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph = WebgraphBuilder::new_memory().open();
for i in 0..n - 1 {
graph.insert(
@@ -208,7 +189,7 @@ mod tests {
);
}
- graph.flush();
+ graph.commit();
graph
}
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index 44c58ec0..f8b7525b 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -23,7 +23,7 @@ use crate::{
hyperloglog::HyperLogLog,
intmap::IntMap,
kahan_sum::KahanSum,
- webgraph::{graph_store::GraphStore, Node, Store, Webgraph},
+ webgraph::{Node, NodeID, Webgraph},
};
const HYPERLOGLOG_COUNTERS: usize = 64;
@@ -63,14 +63,10 @@ impl JankyBloomFilter {
}
pub struct HarmonicCentrality {
- pub full: HashMap<Node, f64>,
pub host: HashMap<Node, f64>,
}
-fn calculate_centrality<S>(graph: &GraphStore<S>) -> HashMap<Node, f64>
-where
- S: Store + Sync,
-{
+fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {
let nodes: Vec<_> = graph.nodes().collect();
info!("Found {} nodes in the graph", nodes.len());
let norm_factor = (nodes.len() - 1) as f64;
@@ -79,9 +75,9 @@ where
.iter()
.map(|node| {
let mut counter = HyperLogLog::default();
- counter.add(*node);
+ counter.add(node.0);
- (*node, counter)
+ (node.0, counter)
})
.collect();
@@ -89,12 +85,12 @@ where
let mut t = 0;
let mut centralities: IntMap<KahanSum> = nodes
.iter()
- .map(|node| (*node, KahanSum::default()))
+ .map(|node| (node.0, KahanSum::default()))
.collect();
let mut changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);
for node in &nodes {
- changed_nodes.insert(*node);
+ changed_nodes.insert(node.0);
}
loop {
@@ -108,12 +104,12 @@ where
let mut new_changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);
for edge in graph.edges() {
- if !changed_nodes.contains(&edge.from) {
+ if !changed_nodes.contains(&edge.from.0) {
continue;
}
if let (Some(counter_to), Some(counter_from)) =
- (new_counters.get_mut(&edge.to), counters.get(&edge.from))
+ (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0))
{
if counter_to
.registers()
@@ -122,7 +118,7 @@ where
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to);
+ new_changed_nodes.insert(edge.to.0);
counter_changes += 1;
}
}
@@ -152,31 +148,23 @@ where
.into_iter()
.map(|(node_id, sum)| (node_id, f64::from(sum)))
.filter(|(_, centrality)| *centrality > 0.0)
- .map(|(node_id, centrality)| (graph.id2node(&node_id).unwrap(), centrality / norm_factor))
+ .map(|(node_id, centrality)| {
+ (
+ graph.id2node(&NodeID::from(node_id)).unwrap(),
+ centrality / norm_factor,
+ )
+ })
.collect()
}
-fn calculate_full(graph: &Webgraph) -> HashMap<Node, f64> {
- graph
- .full
- .as_ref()
- .map(calculate_centrality)
- .unwrap_or_default()
-}
-
fn calculate_host(graph: &Webgraph) -> HashMap<Node, f64> {
- graph
- .host
- .as_ref()
- .map(calculate_centrality)
- .unwrap_or_default()
+ calculate_centrality(&graph)
}
impl HarmonicCentrality {
pub fn calculate(graph: &Webgraph) -> Self {
Self {
host: calculate_host(graph),
- full: calculate_full(graph),
}
}
}
@@ -198,7 +186,7 @@ mod tests {
// │
// D
- let mut graph = WebgraphBuilder::new_memory().with_host_graph().open();
+ let mut graph = WebgraphBuilder::new_memory().open();
graph.insert(Node::from("A"), Node::from("B"), String::new());
graph.insert(Node::from("B"), Node::from("C"), String::new());
@@ -206,17 +194,14 @@ mod tests {
graph.insert(Node::from("C"), Node::from("A"), String::new());
graph.insert(Node::from("D"), Node::from("C"), String::new());
- graph.flush();
+ graph.commit();
graph
}
#[test]
fn host_harmonic_centrality() {
- let mut graph = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph = WebgraphBuilder::new_memory().open();
graph.insert(Node::from("A.com/1"), Node::from("A.com/2"), String::new());
graph.insert(Node::from("A.com/1"), Node::from("A.com/3"), String::new());
@@ -233,13 +218,9 @@ mod tests {
graph.insert(Node::from("C.com"), Node::from("B.com"), String::new());
graph.insert(Node::from("D.com"), Node::from("B.com"), String::new());
- graph.flush();
+ graph.commit();
let centrality = HarmonicCentrality::calculate(&graph);
- assert!(
- centrality.full.get(&Node::from("A.com/1")).unwrap()
- > centrality.full.get(&Node::from("B.com")).unwrap()
- );
assert!(
centrality.host.get(&Node::from("B.com")).unwrap()
@@ -266,17 +247,14 @@ mod tests {
#[test]
fn www_subdomain_ignored() {
- let mut graph = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph = WebgraphBuilder::new_memory().open();
graph.insert(Node::from("B.com"), Node::from("A.com"), String::new());
graph.insert(Node::from("B.com"), Node::from("www.A.com"), String::new());
graph.insert(Node::from("C.com"), Node::from("A.com"), String::new());
graph.insert(Node::from("C.com"), Node::from("www.A.com"), String::new());
- graph.flush();
+ graph.commit();
let centrality = HarmonicCentrality::calculate(&graph);
assert!(centrality.host.get(&Node::from("A.com")).is_some());
@@ -297,10 +275,10 @@ mod tests {
graph.insert(Node::from("A"), Node::from("B"), "6".to_string());
graph.insert(Node::from("A"), Node::from("B"), "7".to_string());
- graph.flush();
+ graph.commit();
let centrality_extra = HarmonicCentrality::calculate(&graph);
- assert_eq!(centrality.full, centrality_extra.full);
+ assert_eq!(centrality.host, centrality_extra.host);
}
}
diff --git a/core/src/webgraph/centrality/mod.rs b/core/src/webgraph/centrality/mod.rs
index 15c13d7a..2343dec6 100644
--- a/core/src/webgraph/centrality/mod.rs
+++ b/core/src/webgraph/centrality/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-pub mod approximate_harmonic;
pub mod betweenness;
pub mod harmonic;
+pub mod online_harmonic;
pub mod topic;
diff --git a/core/src/webgraph/centrality/approximate_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
similarity index 92%
rename from core/src/webgraph/centrality/approximate_harmonic.rs
rename to core/src/webgraph/centrality/online_harmonic.rs
index 6893c290..173c7422 100644
--- a/core/src/webgraph/centrality/approximate_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -34,7 +34,7 @@ use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use crate::intmap::IntMap;
-use crate::webgraph::Node;
+use crate::webgraph::{Node, ShortestPaths};
use crate::webgraph::{NodeID, Webgraph};
use crate::Result;
@@ -52,8 +52,8 @@ pub struct ProxyNode {
impl ProxyNode {
fn dist(&self, from: &NodeID, to: &NodeID) -> Option<usize> {
- if let Some(from_node_to_proxy) = self.dist_from_node.get(from) {
- if let Some(from_proxy_to_node) = self.dist_to_node.get(to) {
+ if let Some(from_node_to_proxy) = self.dist_from_node.get(&from.0) {
+ if let Some(from_proxy_to_node) = self.dist_to_node.get(&to.0) {
return Some(from_node_to_proxy + from_proxy_to_node);
}
}
@@ -183,7 +183,7 @@ impl UserNode {
fn new(id: NodeID, proxy_nodes: &[Arc<ProxyNode>]) -> Self {
let mut heap = BinaryHeap::with_capacity(BEST_PROXY_NODES_PER_USER_NODE);
for proxy in proxy_nodes {
- if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id) {
+ if let Some(dist_to_proxy) = proxy.dist_from_node.get(&id.0) {
let weighted_node = WeightedProxyNode {
node: Arc::clone(proxy),
dist: *dist_to_proxy,
@@ -229,12 +229,12 @@ impl UserNode {
}
#[derive(Serialize, Deserialize, Default)]
-pub struct ApproximatedHarmonicCentrality {
+pub struct OnlineHarmonicCentrality {
pub node2id: HashMap<Node, NodeID>,
pub proxy_nodes: Vec<Arc<ProxyNode>>,
}
-impl ApproximatedHarmonicCentrality {
+impl OnlineHarmonicCentrality {
pub fn new(graph: &Webgraph) -> Self {
Self::new_with_num_proxy(graph, NUM_PROXY_NODES)
}
@@ -246,15 +246,13 @@ impl ApproximatedHarmonicCentrality {
// betweenness centrality, but I don't know how we can approximate betweenness
// on a graph that cannot be in memory
let mut nodes = Vec::new();
- if let Some(store) = graph.host.as_ref() {
- for id in store.nodes() {
- if let Some(node) = store.id2node(&id) {
- if nodes.len() < num_proxy_nodes {
- nodes.push(node.clone());
- }
-
- node2id.insert(node, id);
+ for id in graph.nodes() {
+ if let Some(node) = graph.id2node(&id) {
+ if nodes.len() < num_proxy_nodes {
+ nodes.push(node.clone());
}
+
+ node2id.insert(node, id);
}
}
@@ -263,10 +261,15 @@ impl ApproximatedHarmonicCentrality {
.take(num_proxy_nodes)
.filter(|node| node2id.contains_key(node))
.map(|node| {
- let dist_to_node = graph.raw_host_distances(node.clone()).into_iter().collect();
+ let dist_to_node = graph
+ .raw_distances(node.clone())
+ .into_iter()
+ .map(|(n, v)| (n.0, v))
+ .collect();
let dist_from_node = graph
- .raw_host_reversed_distances(node.clone())
+ .raw_reversed_distances(node.clone())
.into_iter()
+ .map(|(n, v)| (n.0, v))
.collect();
Arc::new(ProxyNode {
@@ -287,13 +290,13 @@ impl ApproximatedHarmonicCentrality {
let liked_nodes = liked_nodes
.iter()
.map(|node| node.clone().into_host())
- .filter_map(|node| self.node2id.get(&node).copied())
+ .filter_map(|node| self.node2id.get(&node).cloned())
.collect_vec();
let disliked_nodes = disliked_nodes
.iter()
.map(|node| node.clone().into_host())
- .filter_map(|node| self.node2id.get(&node).copied())
+ .filter_map(|node| self.node2id.get(&node).cloned())
.collect_vec();
self.scorer_from_ids(&liked_nodes, &disliked_nodes)
@@ -349,10 +352,7 @@ mod tests {
B ----► D ◄------- C
*/
- let mut graph = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph = WebgraphBuilder::new_memory().open();
graph.insert(Node::from("A"), Node::from("D"), String::new());
graph.insert(Node::from("B"), Node::from("D"), String::new());
@@ -364,7 +364,7 @@ mod tests {
graph.insert(Node::from("F"), Node::from("G"), String::new());
graph.insert(Node::from("G"), Node::from("E"), String::new());
- graph.flush();
+ graph.commit();
graph
}
@@ -372,7 +372,7 @@ mod tests {
#[test]
fn ordering() {
let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);
+ let centrality = OnlineHarmonicCentrality::new_with_num_proxy(&graph, 5);
let liked_nodes = vec![Node::from("B".to_string()), Node::from("E".to_string())];
@@ -423,7 +423,7 @@ mod tests {
#[test]
fn disliked_nodes_centrality() {
let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);
+ let centrality = OnlineHarmonicCentrality::new_with_num_proxy(&graph, 5);
let disliked_nodes = vec![Node::from("D".to_string()), Node::from("E".to_string())];
@@ -437,7 +437,7 @@ mod tests {
#[test]
fn ordering_with_dislikes() {
let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);
+ let centrality = OnlineHarmonicCentrality::new_with_num_proxy(&graph, 5);
let liked_nodes = vec![Node::from("B".to_string()), Node::from("E".to_string())];
let disliked_nodes = vec![Node::from("F".to_string())];
diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rs
index 44639ead..5ca803e6 100644
--- a/core/src/webgraph/centrality/topic.rs
+++ b/core/src/webgraph/centrality/topic.rs
@@ -37,11 +37,11 @@ use crate::{
index::Index,
ranking::{initial::InitialScoreTweaker, SignalAggregator},
schema::{FastField, Field, TextField},
- webgraph::{centrality::approximate_harmonic::SHIFT, Webgraph},
+ webgraph::{centrality::online_harmonic::SHIFT, NodeID, Webgraph},
};
use crate::{query::Query, Result};
-use super::approximate_harmonic::ApproximatedHarmonicCentrality;
+use super::online_harmonic::OnlineHarmonicCentrality;
const TOP_TERMS: usize = 1_000_000;
pub const NUM_TOPICS: usize = 50;
@@ -53,8 +53,8 @@ pub struct Scorer {
}
impl Scorer {
- pub fn score(&self, host: u64) -> f64 {
- match self.host_centrality.get(host as usize) {
+ pub fn score(&self, host: NodeID) -> f64 {
+ match self.host_centrality.get(host.0 as usize) {
Some(host_score) => host_score
.iter()
.zip_eq(self.term_weights.iter())
@@ -76,7 +76,7 @@ impl TopicCentrality {
index: &Index,
topics: Mapper,
webgraph: Webgraph,
- approx: ApproximatedHarmonicCentrality,
+ harmonic: OnlineHarmonicCentrality,
) -> Self {
let body_field = index
.schema()
@@ -148,7 +148,7 @@ impl TopicCentrality {
let collector = TopDocs::with_limit(1000, index.inverted_index.fastfield_cache())
.tweak_score(score_tweaker);
- let mut nodes: Vec<_> = webgraph.host.as_ref().unwrap().nodes().collect();
+ let mut nodes: Vec<_> = webgraph.nodes().collect();
nodes.sort();
let mut node_scores = vec![vec![0.0; NUM_TOPICS]; nodes.len()];
@@ -168,7 +168,8 @@ impl TopicCentrality {
if id == u64::MAX {
None
} else {
- Some(webgraph.host.as_ref().unwrap().id2node(&id).unwrap())
+ let id = id.into();
+ Some(webgraph.id2node(&id).unwrap())
}
})
.collect();
@@ -177,10 +178,10 @@ impl TopicCentrality {
assert!(!top_sites.is_empty());
- let scorer = approx.scorer(&top_sites, &[]);
+ let scorer = harmonic.scorer(&top_sites, &[]);
for node in &nodes {
- node_scores[*node as usize][i] = scorer.score(*node) - SHIFT;
+ node_scores[node.0 as usize][i] = scorer.score(*node) - SHIFT;
}
}
diff --git a/core/src/webgraph/executor.rs b/core/src/webgraph/executor.rs
new file mode 100644
index 00000000..a5041424
--- /dev/null
+++ b/core/src/webgraph/executor.rs
@@ -0,0 +1,153 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{Error, Result};
+
+use crossbeam_channel::unbounded;
+use rayon::{ThreadPool, ThreadPoolBuilder};
+
+pub enum Executor {
+ #[allow(unused)]
+ SingleThread,
+ ThreadPool(ThreadPool),
+}
+
+impl Executor {
+ #[allow(unused)]
+ pub fn single_thread() -> Executor {
+ Executor::SingleThread
+ }
+
+ pub fn multi_thread(prefix: &'static str) -> Result<Executor> {
+ Self::with_threads(num_cpus::get(), prefix)
+ }
+
+ pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {
+ let pool = ThreadPoolBuilder::new()
+ .num_threads(num_threads)
+ .thread_name(move |num| format!("{}{}", prefix, num))
+ .build()?;
+ Ok(Executor::ThreadPool(pool))
+ }
+
+ pub fn map<J: Send, R: Send, JIterator: Iterator<Item = J>, F: Sized + Sync + Fn(J) -> R>(
+ &self,
+ f: F,
+ jobs: JIterator,
+ ) -> Result<Vec<R>> {
+ match self {
+ Executor::SingleThread => Ok(jobs.map(f).collect::<_>()),
+ Executor::ThreadPool(pool) => {
+ let jobs: Vec<J> = jobs.collect();
+ let num_jobs = jobs.len();
+ let rx = {
+ let (tx, rx) = unbounded();
+ pool.scope(|scope| {
+ for (idx, arg) in jobs.into_iter().enumerate() {
+ let tx_ref = &tx;
+ let f_ref = &f;
+ scope.spawn(move |_| {
+ let res = f_ref(arg);
+ if let Err(err) = tx_ref.send((idx, res)) {
+ tracing::error!(
+ "Failed to execute job. It probably means all executor \
+ threads have panicked. {:?}",
+ err
+ );
+ panic!();
+ }
+ });
+ }
+ });
+ rx
+ // This ends the scope of tx.
+ // This is important as it makes it possible for the rx iteration to
+ // terminate.
+ };
+ let mut result_placeholders: Vec<Option<R>> =
+ std::iter::repeat_with(|| None).take(num_jobs).collect();
+
+ for (pos, res) in rx {
+ result_placeholders[pos] = Some(res);
+ }
+ let results: Vec<R> = result_placeholders.into_iter().flatten().collect();
+
+ if results.len() != num_jobs {
+ return Err(Error::InternalError(
+ "At least one of the scheduled jobs failed.".to_string(),
+ ));
+ }
+
+ Ok(results)
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+
+ use super::Executor;
+
+ #[test]
+ #[should_panic(expected = "panic should propagate")]
+ fn test_panic_propagates_single_thread() {
+ let _result: Vec<usize> = Executor::single_thread()
+ .map(
+ |_| {
+ panic!("panic should propagate");
+ },
+ vec![0].into_iter(),
+ )
+ .unwrap();
+ }
+
+ #[test]
+ #[should_panic] //< unfortunately the panic message is not propagated
+ fn test_panic_propagates_multi_thread() {
+ let _result: Vec<usize> = Executor::with_threads(1, "search-test")
+ .unwrap()
+ .map(
+ |_| {
+ panic!("panic should propagate");
+ },
+ vec![0].into_iter(),
+ )
+ .unwrap();
+ }
+
+ #[test]
+ fn test_map_singlethread() {
+ let result: Vec<usize> = Executor::single_thread().map(|i| i * 2, 0..1_000).unwrap();
+ assert_eq!(result.len(), 1_000);
+
+ for (i, r) in result.iter().enumerate() {
+ assert_eq!(*r, i * 2);
+ }
+ }
+
+ #[test]
+ fn test_map_multithread() {
+ let result: Vec<usize> = Executor::with_threads(3, "search-test")
+ .unwrap()
+ .map(|i| i * 2, 0..10)
+ .unwrap();
+ assert_eq!(result.len(), 10);
+ for (i, r) in result.iter().enumerate() {
+ assert_eq!(*r, i * 2);
+ }
+ }
+}
diff --git a/core/src/webgraph/graph_store.rs b/core/src/webgraph/graph_store.rs
deleted file mode 100644
index 08292caa..00000000
--- a/core/src/webgraph/graph_store.rs
+++ /dev/null
@@ -1,420 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{
- collections::{BTreeMap, HashSet},
- fmt::Debug,
- marker::PhantomData,
- path::Path,
- sync::{Arc, RwLock},
-};
-
-use serde::{de::DeserializeOwned, Serialize};
-
-use super::{Edge, Node, NodeID, Store, StoredEdge};
-use crate::kv::{rocksdb_store::RocksDbStore, Kv};
-pub(crate) struct Adjacency {
- pub(crate) tree: CachedTree<NodeID, HashSet<StoredEdge>>,
-}
-
-impl Adjacency {
- fn insert(&mut self, from: NodeID, to: NodeID, label: String) {
- let edge = StoredEdge { other: to, label };
-
- if let Some(existing) = self.tree.get(&from) {
- existing.write().unwrap().insert(edge);
- } else {
- let mut set = HashSet::new();
- set.insert(edge);
- self.tree.insert(from, set);
- }
- }
-
- fn edges(&self, node: NodeID) -> Vec<StoredEdge> {
- self.tree
- .get(&node)
- .map(|r| r.read().unwrap().clone())
- .unwrap_or_default()
- .into_iter()
- .collect()
- }
-}
-
-pub(crate) struct CachedTree<K, V>
-where
- K: Ord + Eq + Serialize + DeserializeOwned + Clone,
- V: Serialize + DeserializeOwned + Clone,
-{
- store: Box<dyn Kv<K, V> + Send + Sync>,
- cache: RwLock<BTreeMap<K, Arc<RwLock<V>>>>,
- cache_size: usize,
-}
-
-impl<K, V> CachedTree<K, V>
-where
- K: Ord + Eq + Serialize + DeserializeOwned + Clone,
- V: Serialize + DeserializeOwned + Clone + Debug,
-{
- pub(crate) fn new(store: Box<dyn Kv<K, V> + Send + Sync>, cache_size: usize) -> Self {
- Self {
- store,
- cache_size,
- cache: RwLock::new(BTreeMap::new()),
- }
- }
-
- fn update_cache(&self, key: &K) {
- if !self.cache.read().unwrap().contains_key(key) {
- let val = self.store.get(key);
-
- if let Some(val) = val {
- self.cache
- .write()
- .unwrap()
- .insert(key.clone(), Arc::new(RwLock::new(val)));
- }
- }
- }
-
- pub(crate) fn get(&self, key: &K) -> Option<Arc<RwLock<V>>> {
- self.update_cache(key);
- let guard = self.cache.read().unwrap();
- guard.get(key).cloned()
- }
-
- fn insert(&mut self, key: K, value: V) {
- let cache = self.cache.get_mut().unwrap();
-
- cache.insert(key, Arc::new(RwLock::new(value)));
-
- if cache.len() >= self.cache_size {
- while cache.len() > self.cache_size / 2 {
- if let Some((key, value)) = cache.pop_first() {
- self.store.insert(key, value.write().unwrap().clone());
- }
- }
-
- self.store.flush();
- }
- }
-
- pub(crate) fn flush(&mut self) {
- let cache = self.cache.get_mut().unwrap();
-
- while let Some((key, value)) = cache.pop_first() {
- self.store.insert(key, value.write().unwrap().clone());
- }
-
- self.store.flush();
- }
-
- pub(crate) fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
- self.store.iter()
- }
-}
-
-pub struct GraphStore<S> {
- pub(crate) adjacency: Adjacency,
- pub(crate) reversed_adjacency: Adjacency,
- pub(crate) node2id: CachedTree<Node, NodeID>,
- pub(crate) id2node: CachedTree<NodeID, Node>,
- pub(crate) meta: CachedTree<String, u64>,
- pub(crate) store: PhantomData<S>,
-}
-
-impl<S: Store> GraphStore<S> {
- #[cfg(test)]
- pub(crate) fn temporary() -> GraphStore<S> {
- S::temporary()
- }
-
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- S::open(path)
- }
-
- pub fn open_read_only<P: AsRef<Path>>(path: P) -> Self {
- S::open_read_only(path)
- }
-
- fn next_id(&self) -> NodeID {
- let key = &"next_id".to_string();
- self.meta
- .get(key)
- .map(|arc| *arc.read().unwrap())
- .unwrap_or(0)
- }
-
- fn increment_next_id(&mut self) {
- let current_id = self.next_id();
- let next_id = current_id + 1;
- self.meta.insert("next_id".to_string(), next_id);
- }
-
- fn id_and_increment(&mut self) -> NodeID {
- let id = self.next_id();
- self.increment_next_id();
- id
- }
-
- fn assign_id(&mut self, node: Node, id: NodeID) {
- self.node2id.insert(node.clone(), id);
- self.id2node.insert(id, node);
- }
-
- fn id_or_assign(&mut self, node: Node) -> NodeID {
- if let Some(id) = self.node2id.get(&node) {
- return *id.read().unwrap();
- }
- let id = self.id_and_increment();
- self.assign_id(node, id);
- id
- }
-
- #[allow(unused)]
- pub fn outgoing_edges(&self, node: NodeID) -> Vec<Edge> {
- self.adjacency
- .edges(node)
- .into_iter()
- .map(|edge| Edge {
- from: node,
- to: edge.other,
- label: edge.label,
- })
- .collect()
- }
-
- pub fn ingoing_edges(&self, node: NodeID) -> Vec<Edge> {
- self.reversed_adjacency
- .edges(node)
- .into_iter()
- .map(|edge| Edge {
- from: edge.other,
- to: node,
- label: edge.label,
- })
- .collect()
- }
-
- pub fn nodes(&self) -> impl Iterator<Item = NodeID> {
- self.node2id
- .iter()
- .map(|(_, id)| id)
- .collect::<Vec<u64>>()
- .into_iter()
- }
-
- pub fn insert(&mut self, from: Node, to: Node, label: String) {
- let from_id = self.id_or_assign(from);
- let to_id = self.id_or_assign(to);
-
- self.adjacency.insert(from_id, to_id, label.clone());
- self.reversed_adjacency.insert(to_id, from_id, label);
- }
-
- pub fn node2id(&self, node: &Node) -> Option<NodeID> {
- self.node2id.get(node).map(|arc| *arc.read().unwrap())
- }
-
- pub fn id2node(&self, id: &NodeID) -> Option<Node> {
- self.id2node.get(id).map(|r| r.read().unwrap().clone())
- }
-
- pub fn flush(&mut self) {
- self.adjacency.tree.flush();
- self.reversed_adjacency.tree.flush();
- self.node2id.flush();
- self.id2node.flush();
- self.meta.flush();
- }
-
- pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {
- self.adjacency.tree.iter().flat_map(|(node_id, edges)| {
- edges.into_iter().map(move |stored_edge| Edge {
- from: node_id,
- to: stored_edge.other,
- label: stored_edge.label,
- })
- })
- }
-
- pub fn append(&mut self, mut other: GraphStore<S>) {
- other.flush();
-
- for edge in other.edges() {
- let from = other.id2node(&edge.from).expect("node not found");
- let to = other.id2node(&edge.to).expect("node not found");
-
- self.insert(from, to, edge.label);
- }
-
- self.flush();
- }
-}
-
-impl Store for RocksDbStore {
- fn open<P: AsRef<std::path::Path>>(path: P) -> GraphStore<Self> {
- let adjacency = RocksDbStore::open(path.as_ref().join("adjacency"));
- let reversed_adjacency = RocksDbStore::open(path.as_ref().join("reversed_adjacency"));
- let node2id = RocksDbStore::open(path.as_ref().join("node2id"));
- let id2node = RocksDbStore::open(path.as_ref().join("id2node"));
- let meta = RocksDbStore::open(path.as_ref().join("meta"));
-
- GraphStore {
- adjacency: Adjacency {
- tree: CachedTree::new(adjacency, 50_000),
- },
- reversed_adjacency: Adjacency {
- tree: CachedTree::new(reversed_adjacency, 50_000),
- },
- node2id: CachedTree::new(node2id, 100_000),
- id2node: CachedTree::new(id2node, 100_000),
- meta: CachedTree::new(meta, 1_000),
- store: Default::default(),
- }
- }
-
- fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self> {
- let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));
- let reversed_adjacency =
- RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));
- let node2id = RocksDbStore::open_read_only(path.as_ref().join("node2id"));
- let id2node = RocksDbStore::open_read_only(path.as_ref().join("id2node"));
- let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));
-
- GraphStore {
- adjacency: Adjacency {
- tree: CachedTree::new(adjacency, 50_000),
- },
- reversed_adjacency: Adjacency {
- tree: CachedTree::new(reversed_adjacency, 50_000),
- },
- node2id: CachedTree::new(node2id, 100_000),
- id2node: CachedTree::new(id2node, 100_000),
- meta: CachedTree::new(meta, 1_000),
- store: Default::default(),
- }
- }
-}
-
-#[cfg(test)]
-mod test {
- use super::*;
-
- #[test]
- fn simple_triangle_graph() {
- // ┌────┐
- // │ │
- // ┌───A◄─┐ │
- // │ │ │
- // ▼ │ │
- // B─────►C◄┘
-
- let mut store: GraphStore<RocksDbStore> = GraphStore::temporary();
-
- let a = Node {
- name: "A".to_string(),
- };
- let b = Node {
- name: "B".to_string(),
- };
- let c = Node {
- name: "C".to_string(),
- };
-
- store.insert(a.clone(), b.clone(), String::new());
- store.insert(b.clone(), c.clone(), String::new());
- store.insert(c.clone(), a.clone(), String::new());
- store.insert(a.clone(), c.clone(), String::new());
-
- store.flush();
-
- let nodes: Vec<Node> = store
- .nodes()
- .map(|id| store.id2node(&id).unwrap())
- .collect();
- assert_eq!(nodes, vec![a.clone(), b.clone(), c.clone()]);
-
- let a_id = store.node2id(&a).unwrap();
- let b_id = store.node2id(&b).unwrap();
- let c_id = store.node2id(&c).unwrap();
-
- let mut out = store.outgoing_edges(a_id);
- out.sort();
-
- assert_eq!(
- out,
- vec![
- Edge {
- from: a_id,
- to: b_id,
- label: String::new()
- },
- Edge {
- from: a_id,
- to: c_id,
- label: String::new()
- },
- ]
- );
-
- let mut out = store.outgoing_edges(b_id);
- out.sort();
- assert_eq!(
- out,
- vec![Edge {
- from: b_id,
- to: c_id,
- label: String::new()
- },]
- );
-
- let mut out = store.ingoing_edges(c_id);
- out.sort();
- assert_eq!(
- out,
- vec![
- Edge {
- from: a_id,
- to: c_id,
- label: String::new()
- },
- Edge {
- from: b_id,
- to: c_id,
- label: String::new()
- },
- ]
- );
-
- assert_eq!(
- store.ingoing_edges(a_id),
- vec![Edge {
- from: c_id,
- to: a_id,
- label: String::new()
- },]
- );
-
- assert_eq!(
- store.ingoing_edges(b_id),
- vec![Edge {
- from: a_id,
- to: b_id,
- label: String::new()
- },]
- );
- }
-}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index a127eedb..1075cd31 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -13,14 +13,18 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-mod graph_store;
+mod segment;
use serde::{Deserialize, Serialize};
-use std::collections::{BinaryHeap, HashMap};
+use std::collections::{BinaryHeap, HashMap, HashSet};
+use std::fs::File;
+use std::io::{BufReader, BufWriter, Read, Write};
use std::path::Path;
+use std::sync::Arc;
use std::{cmp, fs};
+use uuid::Uuid;
-use graph_store::GraphStore;
+use segment::Segment;
use crate::directory::{self, DirEntry};
use crate::webpage::Url;
@@ -28,12 +32,22 @@ use crate::webpage::Url;
use crate::kv::rocksdb_store::RocksDbStore;
pub mod centrality;
+mod executor;
+use self::executor::Executor;
+use self::segment::{CachedStore, SegmentNodeID};
-pub type NodeID = u64;
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct NodeID(pub u64);
+
+impl From<u64> for NodeID {
+ fn from(val: u64) -> Self {
+ Self(val)
+ }
+}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub(crate) struct StoredEdge {
- other: NodeID,
+ other: SegmentNodeID,
label: String,
}
@@ -107,8 +121,6 @@ pub struct FullEdge {
pub struct WebgraphBuilder {
path: Box<Path>,
- full_graph_path: Option<Box<Path>>,
- host_graph_path: Option<Box<Path>>,
read_only: bool,
}
@@ -124,22 +136,10 @@ impl WebgraphBuilder {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
Self {
path: path.as_ref().into(),
- full_graph_path: None,
- host_graph_path: None,
read_only: false,
}
}
- pub fn with_full_graph(mut self) -> Self {
- self.full_graph_path = Some(self.path.join("full").into());
- self
- }
-
- pub fn with_host_graph(mut self) -> Self {
- self.host_graph_path = Some(self.path.join("host").into());
- self
- }
-
pub fn read_only(mut self, read_only: bool) -> Self {
self.read_only = read_only;
@@ -148,273 +148,443 @@ impl WebgraphBuilder {
pub fn open(self) -> Webgraph {
if self.read_only {
- Webgraph {
- full: self.full_graph_path.map(GraphStore::open_read_only),
- host: self.host_graph_path.map(GraphStore::open_read_only),
- path: self.path.to_str().unwrap().to_string(),
- }
+ Webgraph::open_read_only(self.path)
} else {
- Webgraph {
- full: self.full_graph_path.map(GraphStore::open),
- host: self.host_graph_path.map(GraphStore::open),
- path: self.path.to_str().unwrap().to_string(),
- }
+ Webgraph::open(self.path)
}
}
}
pub trait Store
where
- Self: Sized,
+ Self: Sized + Send + Sync,
{
- fn open<P: AsRef<Path>>(path: P) -> GraphStore<Self>;
- fn open_read_only<P: AsRef<Path>>(path: P) -> GraphStore<Self>;
+ fn open<P: AsRef<Path>>(path: P, id: String) -> Segment<Self>;
+ fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Segment<Self>;
+
+ fn temporary() -> Segment<Self> {
+ Self::open(crate::gen_temp_path(), "id123".to_string())
+ }
+}
+
+pub trait ShortestPaths {
+ fn distances(&self, source: Node) -> HashMap<Node, usize>;
+ fn raw_distances(&self, source: Node) -> HashMap<NodeID, usize>;
+ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize>;
+ fn reversed_distances(&self, source: Node) -> HashMap<Node, usize>;
+}
+
+fn dijkstra<F1, F2>(
+ source: Node,
+ node_edges: F1,
+ edge_node: F2,
+ graph: &Webgraph,
+) -> HashMap<NodeID, usize>
+where
+ F1: Fn(NodeID) -> Vec<Edge>,
+ F2: Fn(&Edge) -> NodeID,
+{
+ const MAX_DIST: usize = 5;
+ let source_id = graph.node2id(&source);
+ if source_id.is_none() {
+ return HashMap::new();
+ }
+
+ let source_id = source_id.unwrap();
+ let mut distances: HashMap<NodeID, usize> = HashMap::default();
+
+ let mut queue = BinaryHeap::new();
+
+ queue.push(cmp::Reverse((0_usize, source_id)));
+ distances.insert(source_id, 0);
+
+ while let Some(state) = queue.pop() {
+ let (cost, v) = state.0;
+
+ if cost >= MAX_DIST {
+ continue;
+ }
+
+ let current_dist = distances.get(&v).unwrap_or(&usize::MAX);
+
+ if cost > *current_dist {
+ continue;
+ }
+
+ for edge in node_edges(v) {
+ if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&usize::MAX) {
+ let d = cost + 1;
+
+ if d > MAX_DIST {
+ continue;
+ }
+
+ let next = cmp::Reverse((d, edge_node(&edge)));
+ queue.push(next);
+ distances.insert(edge_node(&edge), d);
+ }
+ }
+ }
+
+ distances
+}
+
+impl ShortestPaths for Webgraph {
+ fn distances(&self, source: Node) -> HashMap<Node, usize> {
+ self.raw_distances(source)
+ .into_iter()
+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))
+ .collect()
+ }
+
+ fn raw_distances(&self, source: Node) -> HashMap<NodeID, usize> {
+ dijkstra(
+ source,
+ |node| self.raw_outgoing_edges(&node),
+ |edge| edge.to,
+ self,
+ )
+ }
+
+ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {
+ dijkstra(
+ source,
+ |node| self.raw_ingoing_edges(&node),
+ |edge| edge.from,
+ self,
+ )
+ }
- fn temporary() -> GraphStore<Self> {
- Self::open(crate::gen_temp_path())
+ fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {
+ self.raw_reversed_distances(source)
+ .into_iter()
+ .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))
+ .collect()
}
}
+type SegmentID = String;
+
+#[derive(Serialize, Deserialize, Default)]
+struct Meta {
+ comitted_segments: Vec<SegmentID>,
+ next_node_id: u64,
+}
+
+struct SegmentMergeCandidate<S: Store> {
+ segment: Segment<S>,
+ merges: Vec<Segment<S>>,
+}
+
pub struct Webgraph<S: Store = RocksDbStore> {
pub path: String,
- pub full: Option<GraphStore<S>>,
- pub host: Option<GraphStore<S>>,
+ live_segment: Option<Segment<S>>,
+ segments: Vec<Segment<S>>,
+ executor: Arc<Executor>,
+ node2id: CachedStore<Node, NodeID>,
+ id2node: CachedStore<NodeID, Node>,
+ meta: Meta,
}
impl<S: Store> Webgraph<S> {
- pub fn insert(&mut self, mut from: Node, mut to: Node, label: String) {
- from.remove_protocol();
- to.remove_protocol();
+ fn meta<P: AsRef<Path>>(path: P) -> Meta {
+ let meta_path = path.as_ref().join("metadata.json");
+ let mut reader = BufReader::new(
+ File::options()
+ .create(true)
+ .write(true)
+ .read(true)
+ .open(meta_path)
+ .unwrap(),
+ );
+ let mut buf = String::new();
+ reader.read_to_string(&mut buf).unwrap();
+ serde_json::from_str(&buf).unwrap_or_default()
+ }
+
+ fn save_metadata(&mut self) {
+ let path = Path::new(&self.path).join("metadata.json");
+ let mut writer = BufWriter::new(
+ File::options()
+ .create(true)
+ .write(true)
+ .read(true)
+ .open(path)
+ .unwrap(),
+ );
+
+ let json = serde_json::to_string_pretty(&self.meta).unwrap();
+ writer.write_all(json.as_bytes()).unwrap();
+ }
+
+ fn open<P: AsRef<Path>>(path: P) -> Self {
+ fs::create_dir_all(&path).unwrap();
+ let meta = Self::meta(&path);
- if let Some(full_graph) = &mut self.full {
- full_graph.insert(from.clone(), to.clone(), label.clone());
+ let mut segments = Vec::new();
+ for segment in &meta.comitted_segments {
+ segments.push(Segment::open(
+ path.as_ref().join("segments").join(segment),
+ segment.clone(),
+ ));
}
- if let Some(host_graph) = &mut self.host {
- host_graph.insert(from.into_host(), to.into_host(), label);
+ Self {
+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
+ live_segment: None,
+ segments,
+ executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),
+ node2id: CachedStore::new(RocksDbStore::open(path.as_ref().join("node2id")), 100_000),
+ id2node: CachedStore::new(RocksDbStore::open(path.as_ref().join("id2node")), 100_000),
+ meta,
}
}
- pub fn merge(&mut self, other: Webgraph<S>) {
- match (&mut self.full, other.full) {
- (Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),
- (None, Some(other_graph)) => self.full = Some(other_graph),
- (Some(_), None) | (None, None) => {}
- }
+ fn open_read_only<P: AsRef<Path>>(path: P) -> Self {
+ let meta = Self::meta(&path);
- match (&mut self.host, other.host) {
- (Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),
- (None, Some(other_graph)) => self.host = Some(other_graph),
- (Some(_), None) | (None, None) => {}
- }
+ let mut segments = Vec::new();
- self.flush();
- }
-
- fn dijkstra<F1, F2>(
- source: Node,
- node_edges: F1,
- edge_node: F2,
- store: &GraphStore<S>,
- ) -> HashMap<NodeID, usize>
- where
- F1: Fn(NodeID) -> Vec<Edge>,
- F2: Fn(&Edge) -> NodeID,
- {
- const MAX_DIST: usize = 5;
- let source_id = store.node2id(&source);
- if source_id.is_none() {
- return HashMap::new();
+ for segment in &meta.comitted_segments {
+ segments.push(Segment::open(
+ path.as_ref().join("segments").join(segment),
+ segment.clone(),
+ ));
}
- let source_id = source_id.unwrap();
- let mut distances: HashMap<NodeID, usize> = HashMap::default();
+ Self {
+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
+ live_segment: None,
+ segments,
+ executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),
+ node2id: CachedStore::new(
+ RocksDbStore::open_read_only(path.as_ref().join("node2id")),
+ 100_000,
+ ),
+ id2node: CachedStore::new(
+ RocksDbStore::open_read_only(path.as_ref().join("id2node")),
+ 100_000,
+ ),
+ meta,
+ }
+ }
- let mut queue = BinaryHeap::new();
+ fn id_and_increment(&mut self) -> NodeID {
+ let id = self.meta.next_node_id.into();
+ self.meta.next_node_id += 1;
+ id
+ }
- queue.push(cmp::Reverse((0_usize, source_id)));
- distances.insert(source_id, 0);
+ fn id_or_assign(&mut self, node: &Node) -> NodeID {
+ match self.node2id(node) {
+ Some(id) => id,
+ None => {
+ let id = self.id_and_increment();
- while let Some(state) = queue.pop() {
- let (cost, v) = state.0;
+ self.node2id.insert(node.clone(), id);
+ self.id2node.insert(id, node.clone());
- if cost >= MAX_DIST {
- continue;
+ id
}
+ }
+ }
- let current_dist = distances.get(&v).unwrap_or(&usize::MAX);
+ pub fn insert(&mut self, mut from: Node, mut to: Node, label: String) {
+ from.remove_protocol();
+ to.remove_protocol();
- if cost > *current_dist {
- continue;
- }
+ let (from, to) = (from.into_host(), to.into_host());
+ let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));
- for edge in node_edges(v) {
- if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&usize::MAX) {
- let d = cost + 1;
+ match &mut self.live_segment {
+ Some(segment) => segment.insert(from_id, to_id, label),
+ None => {
+ let segment_id = Uuid::new_v4().to_string();
+ let path = Path::new(&self.path).join("segments").join(&segment_id);
+ let mut segment: Segment<S> = Segment::open(path, segment_id);
- if d > MAX_DIST {
- continue;
- }
+ segment.insert(from_id, to_id, label);
- let next = cmp::Reverse((d, edge_node(&edge)));
- queue.push(next);
- distances.insert(edge_node(&edge), d);
- }
+ self.live_segment = Some(segment);
}
}
-
- distances
}
- pub fn distances(&self, source: Node) -> HashMap<Node, usize> {
- self.full
- .as_ref()
- .map(|full_graph| {
- let distances = Webgraph::dijkstra(
- source,
- |node_id| full_graph.outgoing_edges(node_id),
- |edge| edge.to,
- full_graph,
- );
+ pub fn merge(&mut self, mut other: Webgraph<S>) {
+ other.commit();
+ let mut mapping = Vec::new();
- distances
- .into_iter()
- .map(|(id, dist)| (full_graph.id2node(&id).expect("unknown node"), dist))
- .collect()
- })
- .unwrap_or_default()
- }
+ for (node, other_id) in other.node2id.iter() {
+ match self.node2id(&node) {
+ Some(this_id) => mapping.push((other_id, this_id)),
+ None => {
+ let new_id = self.id_or_assign(&node);
+ mapping.push((other_id, new_id));
+ }
+ }
+ }
- #[allow(unused)]
- fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {
- self.full
- .as_ref()
- .map(|full_graph| {
- Webgraph::dijkstra(
- source,
- |node| full_graph.ingoing_edges(node),
- |edge| edge.from,
- full_graph,
- )
- })
- .unwrap_or_default()
- }
-
- #[allow(unused)]
- pub fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {
- self.full
- .as_ref()
- .map(|full_graph| {
- self.raw_reversed_distances(source)
- .into_iter()
- .map(|(id, dist)| (full_graph.id2node(&id).expect("unknown node"), dist))
- .collect()
- })
- .unwrap_or_default()
- }
-
- #[allow(unused)]
- pub fn host_distances(&self, source: Node) -> HashMap<Node, usize> {
- self.host
- .as_ref()
- .map(|host_graph| {
- self.raw_host_distances(source)
- .into_iter()
- .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))
- .collect()
- })
- .unwrap_or_default()
- }
-
- pub fn raw_host_distances(&self, source: Node) -> HashMap<NodeID, usize> {
- self.host
- .as_ref()
- .map(|host_graph| {
- Webgraph::dijkstra(
- source,
- |node| host_graph.outgoing_edges(node),
- |edge| edge.to,
- host_graph,
- )
- })
- .unwrap_or_default()
- }
-
- pub fn raw_host_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {
- self.host
- .as_ref()
- .map(|host_graph| {
- Webgraph::dijkstra(
- source,
- |node| host_graph.ingoing_edges(node),
- |edge| edge.from,
- host_graph,
- )
- })
- .unwrap_or_default()
- }
-
- #[allow(unused)]
- pub fn host_reversed_distances(&self, source: Node) -> HashMap<Node, usize> {
- self.host
- .as_ref()
- .map(|host_graph| {
- self.raw_host_reversed_distances(source)
- .into_iter()
- .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))
- .collect()
- })
- .unwrap_or_default()
- }
-
- pub fn flush(&mut self) {
- if let Some(full_graph) = &mut self.full {
- full_graph.flush();
+ self.executor
+ .map(
+ |segment| segment.update_id_mapping(mapping.clone()),
+ other.segments.iter_mut(),
+ )
+ .expect("failed to merge webgraphs");
+
+ for mut segment in other.segments {
+ segment.flush();
+ let id = segment.id();
+ let new_path = Path::new(&self.path).join("segments").join(segment.id());
+ std::fs::rename(segment.path(), &new_path).unwrap();
+
+ self.meta.comitted_segments.push(segment.id());
+ drop(segment);
+ self.segments.push(Segment::open(new_path, id));
}
- if let Some(host_graph) = &mut self.host {
- host_graph.flush();
+
+ self.commit();
+ }
+
+ pub fn commit(&mut self) {
+ if let Some(mut segment) = self.live_segment.take() {
+ segment.flush();
+ self.meta.comitted_segments.push(segment.id());
+ self.segments.push(segment);
+ self.save_metadata();
}
+
+ self.node2id.flush();
+ self.id2node.flush();
}
pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
- if let Some(graph) = &self.full {
- if let Some(node_id) = graph.node2id(&node) {
- graph
- .ingoing_edges(node_id)
- .into_iter()
- .map(|edge| FullEdge {
- from: graph.id2node(&edge.from).unwrap(),
- to: graph.id2node(&edge.to).unwrap(),
- label: edge.label,
- })
- .collect()
- } else {
- Vec::new()
- }
+ if let Some(node_id) = self.node2id(&node) {
+ self.raw_ingoing_edges(&node_id)
+ .into_iter()
+ .map(|edge| FullEdge {
+ from: self.id2node(&edge.from).unwrap(),
+ to: self.id2node(&edge.to).unwrap(),
+ label: edge.label,
+ })
+ .collect()
} else {
Vec::new()
}
}
- pub fn ingoing_edges_host(&self, node: Node) -> Vec<FullEdge> {
- if let Some(graph) = &self.host {
- if let Some(node_id) = graph.node2id(&node) {
- graph
- .ingoing_edges(node_id)
- .into_iter()
- .map(|edge| FullEdge {
- from: graph.id2node(&edge.from).unwrap(),
- to: graph.id2node(&edge.to).unwrap(),
- label: edge.label,
- })
- .collect()
- } else {
- Vec::new()
- }
+ pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {
+ self.executor
+ .map(|segment| segment.ingoing_edges(node), self.segments.iter())
+ .unwrap()
+ .into_iter()
+ .flatten()
+ .collect()
+ }
+
+ pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {
+ self.executor
+ .map(|segment| segment.outgoing_edges(node), self.segments.iter())
+ .unwrap()
+ .into_iter()
+ .flatten()
+ .collect()
+ }
+
+ pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {
+ if let Some(node_id) = self.node2id(&node) {
+ self.raw_outgoing_edges(&node_id)
+ .into_iter()
+ .map(|edge| FullEdge {
+ from: self.id2node(&edge.from).unwrap(),
+ to: self.id2node(&edge.to).unwrap(),
+ label: edge.label,
+ })
+ .collect()
} else {
Vec::new()
}
}
+
+ pub fn node2id(&self, node: &Node) -> Option<NodeID> {
+ self.node2id
+ .get(node)
+ .map(|lock| lock.read().unwrap().0.into())
+ }
+
+ pub fn id2node(&self, id: &NodeID) -> Option<Node> {
+ self.id2node
+ .get(id)
+ .map(|lock| lock.read().unwrap().clone())
+ }
+
+ pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {
+ self.node2id.iter().map(|(_, id)| id)
+ }
+
+ pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {
+ self.segments.iter().flat_map(|segment| segment.edges())
+ }
+
+ pub fn merge_segments(&mut self, num_segments: usize) {
+ if num_segments >= self.segments.len() {
+ return;
+ }
+
+ self.segments
+ .sort_by_key(|b| std::cmp::Reverse(b.num_nodes()));
+
+ let mut candidates = Vec::with_capacity(num_segments);
+
+ for segment in self.segments.drain(0..num_segments) {
+ candidates.push(SegmentMergeCandidate {
+ segment,
+ merges: Vec::new(),
+ });
+ }
+
+ let num_candidates = candidates.len();
+
+ for (next_candidate, segment) in self.segments.drain(0..).enumerate() {
+ candidates[next_candidate % num_candidates]
+ .merges
+ .push(segment);
+ }
+
+ let segments = self
+ .executor
+ .map(
+ |mut candidate| {
+ for other in candidate.merges {
+ candidate.segment.merge(other);
+ }
+
+ candidate.segment
+ },
+ candidates.into_iter(),
+ )
+ .unwrap();
+
+ self.segments = segments;
+
+ self.meta.comitted_segments = self.segments.iter().map(|segment| segment.id()).collect();
+
+ self.garbage_collect();
+ }
+
+ fn garbage_collect(&self) {
+ let path = Path::new(&self.path).join("segments");
+ let segments: HashSet<_> = self.meta.comitted_segments.iter().cloned().collect();
+
+ for path in fs::read_dir(path).unwrap() {
+ let path = path.unwrap();
+ let file_name = path.file_name();
+ let name = file_name.as_os_str().to_str().unwrap();
+
+ if !segments.contains(name) {
+ fs::remove_dir_all(path.path()).unwrap();
+ }
+ }
+ }
}
impl From<FrozenWebgraph> for Webgraph {
@@ -435,48 +605,40 @@ impl From<FrozenWebgraph> for Webgraph {
directory::recreate_folder(&frozen.root).unwrap();
- let mut builder = WebgraphBuilder::new(path);
-
- if frozen.has_full {
- builder = builder.with_full_graph();
- }
-
- if frozen.has_host {
- builder = builder.with_host_graph();
- }
-
- builder.open()
+ WebgraphBuilder::new(path).open()
}
}
impl From<Webgraph> for FrozenWebgraph {
fn from(mut graph: Webgraph) -> Self {
- graph.flush();
+ graph.commit();
let path = graph.path.clone();
- let has_full = graph.full.is_some();
- let has_host = graph.host.is_some();
drop(graph);
let root = directory::scan_folder(path).unwrap();
- Self {
- root,
- has_full,
- has_host,
- }
+ Self { root }
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct FrozenWebgraph {
pub root: DirEntry,
- has_full: bool,
- has_host: bool,
}
#[cfg(test)]
mod test {
use super::*;
+ fn test_edges() -> Vec<(Node, Node, String)> {
+ vec![
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("C"), String::new()),
+ (Node::from("A"), Node::from("C"), String::new()),
+ (Node::from("C"), Node::from("A"), String::new()),
+ (Node::from("D"), Node::from("C"), String::new()),
+ ]
+ }
+
fn test_graph() -> Webgraph {
// ┌────┐
// │ │
@@ -489,18 +651,13 @@ mod test {
// │
// D
- let mut graph = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph = WebgraphBuilder::new_memory().open();
- graph.insert(Node::from("A"), Node::from("B"), String::new());
- graph.insert(Node::from("B"), Node::from("C"), String::new());
- graph.insert(Node::from("A"), Node::from("C"), String::new());
- graph.insert(Node::from("C"), Node::from("A"), String::new());
- graph.insert(Node::from("D"), Node::from("C"), String::new());
+ for (from, to, label) in test_edges() {
+ graph.insert(from, to, label);
+ }
- graph.flush();
+ graph.commit();
graph
}
@@ -542,18 +699,14 @@ mod test {
#[test]
fn merge() {
- let mut graph1 = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph1 = WebgraphBuilder::new_memory().open();
graph1.insert(Node::from("A"), Node::from("B"), String::new());
+ graph1.commit();
- let mut graph2 = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph2 = WebgraphBuilder::new_memory().open();
graph2.insert(Node::from("B"), Node::from("C"), String::new());
+ graph2.commit();
graph1.merge(graph2);
@@ -566,6 +719,7 @@ mod test {
#[test]
fn serialize_deserialize_bincode() {
let graph = test_graph();
+
let path = graph.path.clone();
let frozen: FrozenWebgraph = graph.into();
let bytes = bincode::serialize(&frozen).unwrap();
@@ -601,10 +755,7 @@ mod test {
assert_eq!(&n.name, "www.example.com?test");
- let mut graph = WebgraphBuilder::new_memory()
- .with_full_graph()
- .with_host_graph()
- .open();
+ let mut graph = WebgraphBuilder::new_memory().open();
graph.insert(
Node::from("http://A"),
@@ -612,7 +763,7 @@ mod test {
String::new(),
);
- graph.flush();
+ graph.commit();
let distances = graph.distances(Node::from("A"));
assert_eq!(distances.get(&Node::from("B")), Some(&1));
@@ -620,4 +771,122 @@ mod test {
let distances = graph.distances(Node::from("http://A"));
assert!(distances.is_empty());
}
+
+ #[test]
+ fn merge_segments() {
+ let mut graph = WebgraphBuilder::new_memory().open();
+
+ let edges = test_edges();
+ let num_edges = edges.len();
+
+ for (from, to, label) in test_edges() {
+ graph.insert(from, to, label);
+ graph.commit();
+ }
+
+ graph.commit();
+
+ assert_eq!(num_edges, graph.segments.len());
+
+ let distances = graph.distances(Node::from("D"));
+
+ assert_eq!(distances.get(&Node::from("C")), Some(&1));
+ assert_eq!(distances.get(&Node::from("A")), Some(&2));
+ assert_eq!(distances.get(&Node::from("B")), Some(&3));
+
+ let mut res = graph.outgoing_edges(Node::from("A"));
+ res.sort_by(|a, b| a.to.name.cmp(&b.to.name));
+
+ assert_eq!(
+ res,
+ vec![
+ FullEdge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ label: String::new()
+ },
+ FullEdge {
+ from: Node::from("A"),
+ to: Node::from("C"),
+ label: String::new()
+ }
+ ]
+ );
+
+ let mut res = graph.ingoing_edges(Node::from("C"));
+ res.sort_by(|a, b| a.from.name.cmp(&b.from.name));
+
+ assert_eq!(
+ res,
+ vec![
+ FullEdge {
+ from: Node::from("A"),
+ to: Node::from("C"),
+ label: String::new()
+ },
+ FullEdge {
+ from: Node::from("B"),
+ to: Node::from("C"),
+ label: String::new()
+ },
+ FullEdge {
+ from: Node::from("D"),
+ to: Node::from("C"),
+ label: String::new()
+ },
+ ]
+ );
+
+ graph.merge_segments(2);
+ assert_eq!(graph.segments.len(), 2);
+
+ let distances = graph.distances(Node::from("D"));
+
+ assert_eq!(distances.get(&Node::from("C")), Some(&1));
+ assert_eq!(distances.get(&Node::from("A")), Some(&2));
+ assert_eq!(distances.get(&Node::from("B")), Some(&3));
+
+ let mut res = graph.outgoing_edges(Node::from("A"));
+ res.sort_by(|a, b| a.to.name.cmp(&b.to.name));
+
+ assert_eq!(
+ res,
+ vec![
+ FullEdge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ label: String::new()
+ },
+ FullEdge {
+ from: Node::from("A"),
+ to: Node::from("C"),
+ label: String::new()
+ }
+ ]
+ );
+
+ let mut res = graph.ingoing_edges(Node::from("C"));
+ res.sort_by(|a, b| a.from.name.cmp(&b.from.name));
+
+ assert_eq!(
+ res,
+ vec![
+ FullEdge {
+ from: Node::from("A"),
+ to: Node::from("C"),
+ label: String::new()
+ },
+ FullEdge {
+ from: Node::from("B"),
+ to: Node::from("C"),
+ label: String::new()
+ },
+ FullEdge {
+ from: Node::from("D"),
+ to: Node::from("C"),
+ label: String::new()
+ },
+ ]
+ );
+ }
}
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
new file mode 100644
index 00000000..aa20d7a0
--- /dev/null
+++ b/core/src/webgraph/segment.rs
@@ -0,0 +1,460 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use std::{
+ collections::{BTreeMap, HashSet},
+ fmt::Debug,
+ marker::PhantomData,
+ path::Path,
+ sync::{Arc, RwLock},
+};
+
+use super::{Edge, NodeID, Store, StoredEdge};
+use crate::kv::{rocksdb_store::RocksDbStore, Kv};
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct SegmentNodeID(u64);
+
+impl From<u64> for SegmentNodeID {
+ fn from(val: u64) -> Self {
+ Self(val)
+ }
+}
+
+pub(crate) struct Adjacency {
+ pub(crate) tree: CachedStore<SegmentNodeID, HashSet<StoredEdge>>,
+}
+
+impl Adjacency {
+ fn insert(&mut self, from: SegmentNodeID, to: SegmentNodeID, label: String) {
+ let edge = StoredEdge { other: to, label };
+
+ if let Some(existing) = self.tree.get(&from) {
+ existing.write().unwrap().insert(edge);
+ } else {
+ let mut set = HashSet::new();
+ set.insert(edge);
+ self.tree.insert(from, set);
+ }
+ }
+
+ fn edges(&self, node: &SegmentNodeID) -> Vec<StoredEdge> {
+ self.tree
+ .get(node)
+ .map(|r| r.read().unwrap().clone())
+ .unwrap_or_default()
+ .into_iter()
+ .collect()
+ }
+}
+
+pub struct CachedStore<K, V>
+where
+ K: Ord + Eq + Serialize + DeserializeOwned + Clone,
+ V: Serialize + DeserializeOwned + Clone,
+{
+ store: Box<dyn Kv<K, V> + Send + Sync>,
+ cache: RwLock<BTreeMap<K, Arc<RwLock<V>>>>,
+ cache_size: usize,
+}
+
+impl<K, V> CachedStore<K, V>
+where
+ K: Ord + Eq + Serialize + DeserializeOwned + Clone,
+ V: Serialize + DeserializeOwned + Clone + Debug,
+{
+ pub fn new(store: Box<dyn Kv<K, V> + Send + Sync>, cache_size: usize) -> Self {
+ Self {
+ store,
+ cache_size,
+ cache: RwLock::new(BTreeMap::new()),
+ }
+ }
+
+ fn update_cache(&self, key: &K) {
+ if !self.cache.read().unwrap().contains_key(key) {
+ let val = self.store.get(key);
+
+ if let Some(val) = val {
+ self.cache
+ .write()
+ .unwrap()
+ .insert(key.clone(), Arc::new(RwLock::new(val)));
+ }
+ }
+ }
+
+ pub fn get(&self, key: &K) -> Option<Arc<RwLock<V>>> {
+ self.update_cache(key);
+ let guard = self.cache.read().unwrap();
+ guard.get(key).cloned()
+ }
+
+ pub fn insert(&mut self, key: K, value: V) {
+ let cache = self.cache.get_mut().unwrap();
+
+ cache.insert(key, Arc::new(RwLock::new(value)));
+
+ if cache.len() >= self.cache_size {
+ while cache.len() > self.cache_size / 2 {
+ if let Some((key, value)) = cache.pop_first() {
+ self.store.insert(key, value.write().unwrap().clone());
+ }
+ }
+
+ self.store.flush();
+ }
+ }
+
+ pub fn flush(&mut self) {
+ let cache = self.cache.get_mut().unwrap();
+
+ while let Some((key, value)) = cache.pop_first() {
+ self.store.insert(key, value.write().unwrap().clone());
+ }
+
+ self.store.flush();
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
+ self.store.iter()
+ }
+
+ pub fn delete(&mut self, key: &K) {
+ self.store.delete(key);
+ self.cache.write().unwrap().remove(key);
+ }
+}
+
+pub struct Segment<S: Store> {
+ adjacency: Adjacency,
+ reversed_adjacency: Adjacency,
+ id_mapping: CachedStore<NodeID, SegmentNodeID>,
+ rev_id_mapping: CachedStore<SegmentNodeID, NodeID>,
+ meta: CachedStore<String, u64>,
+ store: PhantomData<S>,
+ id: String,
+ path: String,
+}
+
+impl<S: Store> Segment<S> {
+ #[cfg(test)]
+ pub(crate) fn temporary() -> Segment<S> {
+ S::temporary()
+ }
+
+ pub fn open<P: AsRef<Path>>(path: P, id: String) -> Self {
+ S::open(path, id)
+ }
+
+ pub fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Self {
+ S::open_read_only(path, id)
+ }
+
+ fn next_id(&self) -> SegmentNodeID {
+ let key = &"next_id".to_string();
+ self.meta
+ .get(key)
+ .map(|arc| *arc.read().unwrap())
+ .unwrap_or(0)
+ .into()
+ }
+
+ pub fn num_nodes(&self) -> usize {
+ self.next_id().0 as usize
+ }
+
+ fn increment_next_id(&mut self) {
+ let current_id = self.next_id().0;
+ let next_id = current_id + 1;
+ self.meta.insert("next_id".to_string(), next_id);
+ }
+
+ fn id_and_increment(&mut self) -> SegmentNodeID {
+ let id = self.next_id();
+ self.increment_next_id();
+ id
+ }
+
+ fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {
+ self.id_mapping.get(node).map(|lock| *lock.read().unwrap())
+ }
+
+ fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {
+ self.rev_id_mapping
+ .get(node)
+ .map(|lock| *lock.read().unwrap())
+ }
+
+ pub fn outgoing_edges(&self, node: &NodeID) -> Vec<Edge> {
+ match self.id_mapping(node) {
+ Some(segment_id) => self
+ .adjacency
+ .edges(&segment_id)
+ .into_iter()
+ .map(|edge| Edge {
+ from: *node,
+ to: self.rev_id_mapping(&edge.other).unwrap(),
+ label: edge.label,
+ })
+ .collect(),
+ None => Vec::new(),
+ }
+ }
+
+ pub fn ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {
+ match self.id_mapping(node) {
+ Some(segment_id) => self
+ .reversed_adjacency
+ .edges(&segment_id)
+ .into_iter()
+ .map(|edge| Edge {
+ from: self.rev_id_mapping(&edge.other).unwrap(),
+ to: *node,
+ label: edge.label,
+ })
+ .collect(),
+ None => Vec::new(),
+ }
+ }
+
+ fn id_or_assign(&mut self, node: &NodeID) -> SegmentNodeID {
+ match self.id_mapping(node) {
+ Some(id) => id,
+ None => {
+ let id = self.id_and_increment();
+
+ self.id_mapping.insert(*node, id);
+ self.rev_id_mapping.insert(id, *node);
+
+ id
+ }
+ }
+ }
+
+ pub fn insert(&mut self, from: NodeID, to: NodeID, label: String) {
+ let from_id = self.id_or_assign(&from);
+ let to_id = self.id_or_assign(&to);
+
+ self.adjacency.insert(from_id, to_id, label.clone());
+ self.reversed_adjacency.insert(to_id, from_id, label);
+ }
+
+ pub fn flush(&mut self) {
+ self.adjacency.tree.flush();
+ self.reversed_adjacency.tree.flush();
+ self.id_mapping.flush();
+ self.rev_id_mapping.flush();
+ self.meta.flush();
+ }
+
+ pub fn id(&self) -> String {
+ self.id.clone()
+ }
+
+ pub fn path(&self) -> &Path {
+ Path::new(&self.path)
+ }
+
+ pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {
+ self.adjacency
+ .tree
+ .iter()
+ .flat_map(move |(node_id, edges)| {
+ let from = self.rev_id_mapping(&node_id).unwrap();
+
+ edges.into_iter().map(move |stored_edge| Edge {
+ from,
+ to: self.rev_id_mapping(&stored_edge.other).unwrap(),
+ label: stored_edge.label,
+ })
+ })
+ }
+
+ pub fn merge(&mut self, mut other: Segment<S>) {
+ other.flush();
+
+ for edge in other.edges() {
+ self.insert(edge.from, edge.to, edge.label);
+ }
+
+ self.flush();
+ }
+
+ pub fn update_id_mapping(&mut self, mapping: Vec<(NodeID, NodeID)>) {
+ let mut new_mappings = Vec::with_capacity(mapping.len());
+
+ for (old_id, new_id) in mapping {
+ if let Some(segment_id) = self.id_mapping(&old_id) {
+ self.id_mapping.delete(&old_id);
+ self.rev_id_mapping.delete(&segment_id);
+
+ new_mappings.push((segment_id, new_id));
+ }
+ }
+
+ for (segment_id, new_id) in new_mappings {
+ self.id_mapping.insert(new_id, segment_id);
+ self.rev_id_mapping.insert(segment_id, new_id);
+ }
+
+ self.flush();
+ }
+}
+
+impl Store for RocksDbStore {
+ fn open<P: AsRef<std::path::Path>>(path: P, id: String) -> Segment<Self> {
+ let adjacency = RocksDbStore::open(path.as_ref().join("adjacency"));
+ let reversed_adjacency = RocksDbStore::open(path.as_ref().join("reversed_adjacency"));
+ let id_mapping = RocksDbStore::open(path.as_ref().join("id_mapping"));
+ let rev_id_mapping = RocksDbStore::open(path.as_ref().join("rev_id_mapping"));
+ let meta = RocksDbStore::open(path.as_ref().join("meta"));
+
+ Segment {
+ adjacency: Adjacency {
+ tree: CachedStore::new(adjacency, 50_000),
+ },
+ reversed_adjacency: Adjacency {
+ tree: CachedStore::new(reversed_adjacency, 50_000),
+ },
+ id_mapping: CachedStore::new(id_mapping, 100_000),
+ rev_id_mapping: CachedStore::new(rev_id_mapping, 100_000),
+ meta: CachedStore::new(meta, 1_000),
+ store: Default::default(),
+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
+ id,
+ }
+ }
+
+ fn open_read_only<P: AsRef<Path>>(path: P, id: String) -> Segment<Self> {
+ let adjacency = RocksDbStore::open_read_only(path.as_ref().join("adjacency"));
+ let reversed_adjacency =
+ RocksDbStore::open_read_only(path.as_ref().join("reversed_adjacency"));
+ let id_mapping = RocksDbStore::open_read_only(path.as_ref().join("id_mapping"));
+ let rev_id_mapping = RocksDbStore::open_read_only(path.as_ref().join("rev_id_mapping"));
+ let meta = RocksDbStore::open_read_only(path.as_ref().join("meta"));
+
+ Segment {
+ adjacency: Adjacency {
+ tree: CachedStore::new(adjacency, 50_000),
+ },
+ reversed_adjacency: Adjacency {
+ tree: CachedStore::new(reversed_adjacency, 50_000),
+ },
+ id_mapping: CachedStore::new(id_mapping, 100_000),
+ rev_id_mapping: CachedStore::new(rev_id_mapping, 100_000),
+ meta: CachedStore::new(meta, 1_000),
+ store: Default::default(),
+ path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
+ id,
+ }
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ #[test]
+ fn simple_triangle_graph() {
+ // ┌────┐
+ // │ │
+ // ┌───0◄─┐ │
+ // │ │ │
+ // ▼ │ │
+ // 1─────►2◄┘
+
+ let mut store: Segment<RocksDbStore> = Segment::temporary();
+
+ let a = NodeID(0);
+ let b = NodeID(1);
+ let c = NodeID(2);
+
+ store.insert(a, b, String::new());
+ store.insert(b, c, String::new());
+ store.insert(c, a, String::new());
+ store.insert(a, c, String::new());
+
+ store.flush();
+
+ let mut out = store.outgoing_edges(&a);
+ out.sort();
+
+ assert_eq!(
+ out,
+ vec![
+ Edge {
+ from: a,
+ to: b,
+ label: String::new()
+ },
+ Edge {
+ from: a,
+ to: c,
+ label: String::new()
+ },
+ ]
+ );
+
+ let mut out = store.outgoing_edges(&b);
+ out.sort();
+ assert_eq!(
+ out,
+ vec![Edge {
+ from: b,
+ to: c,
+ label: String::new()
+ },]
+ );
+
+ let mut out = store.ingoing_edges(&c);
+ out.sort();
+ assert_eq!(
+ out,
+ vec![
+ Edge {
+ from: a,
+ to: c,
+ label: String::new()
+ },
+ Edge {
+ from: b,
+ to: c,
+ label: String::new()
+ },
+ ]
+ );
+
+ assert_eq!(
+ store.ingoing_edges(&a),
+ vec![Edge {
+ from: c,
+ to: a,
+ label: String::new()
+ },]
+ );
+
+ assert_eq!(
+ store.ingoing_edges(&b),
+ vec![Edge {
+ from: a,
+ to: b,
+ label: String::new()
+ },]
+ );
+ }
+}
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index d428cbab..032768c2 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -19,6 +19,7 @@ use crate::{
schema::{FastField, TextField},
simhash,
tokenizer::{self, FlattenedJson},
+ webgraph::NodeID,
Error, Result,
};
use chrono::{DateTime, FixedOffset, Utc};
@@ -131,7 +132,7 @@ pub struct Webpage {
pub fetch_time_ms: u64,
pub pre_computed_score: f64,
pub primary_image: Option<StoredPrimaryImage>,
- pub node_id: Option<u64>,
+ pub node_id: Option<NodeID>,
pub crawl_stability: f64,
pub host_topic: Option<Topic>,
pub dmoz_description: Option<String>,
@@ -249,7 +250,7 @@ impl Webpage {
schema
.get_field(Field::Fast(FastField::HostNodeID).name())
.expect("Failed to get node_id field"),
- self.node_id.unwrap_or(u64::MAX),
+ self.node_id.map(|n| n.0).unwrap_or(u64::MAX),
);
doc.add_u64(
diff --git a/scripts/export_crossencoder b/scripts/export_crossencoder
new file mode 100755
index 00000000..fba0becb
--- /dev/null
+++ b/scripts/export_crossencoder
@@ -0,0 +1,4 @@
+#!/bin/bash
+pip3 install transformers[onnx] torch
+python3 -m transformers.onnx --model=cross-encoder/ms-marco-MiniLM-L-6-v2 data/
+mv data/model.onnx data/crossencoder.onnx
diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.sh
index f7c69417..0a8564b1 100644
--- a/scripts/run_frontend.sh
+++ b/scripts/run_frontend.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#!/bin/bash
trap 'kill $(jobs -p)' EXIT
cargo run -- search-server configs/search_server.toml &
cargo run -- frontend configs/frontend.toml