Skip to content

web search done right

Split centrality building into separate processes. This is a hotfix to reduce the memory for each step

Mikkel Denker8a6751cf244d
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index d306b1e3..f5f5c32a 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -21,9 +21,18 @@ use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuilde
pub struct Centrality {}
impl Centrality {
- pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
+ pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();
+ CentralityStore::build_harmonic(&graph, output_path);
+ }
- CentralityStore::build(&graph, output_path);
+ pub fn build_online<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
+ let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();
+ CentralityStore::build_online(&graph, output_path);
+ }
+
+ pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
+ let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();
+ CentralityStore::build_similarity(&graph, output_path);
}
}
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 2fab4029..ec50c64e 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -104,7 +104,10 @@ fn calculate_centrality() {
debug!("Calculating centrality");
let webgraph_path = Path::new(DATA_PATH).join("webgraph");
let out_path = Path::new(DATA_PATH).join("centrality");
- Centrality::run(webgraph_path, out_path);
+
+ Centrality::build_harmonic(&webgraph_path, &out_path);
+ Centrality::build_online(&webgraph_path, &out_path);
+ Centrality::build_similarity(&webgraph_path, &out_path);
}
fn create_inverted_index() -> Result<()> {
diff --git a/core/src/main.rs b/core/src/main.rs
index 4ca73b97..c5c1a546 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -43,6 +43,8 @@ enum Commands {
options: IndexingOptions,
},
Centrality {
+ #[clap(subcommand)]
+ mode: CentralityType,
webgraph_path: String,
output_path: String,
},
@@ -84,6 +86,13 @@ enum Commands {
},
}
+#[derive(Subcommand)]
+enum CentralityType {
+ Harmonic,
+ OnlineHarmonic,
+ Similarity,
+}
+
#[derive(Subcommand)]
enum WebgraphOptions {
Master {
@@ -179,9 +188,20 @@ fn main() -> Result<()> {
)?,
},
Commands::Centrality {
+ mode,
webgraph_path,
output_path,
- } => entrypoint::Centrality::run(webgraph_path, output_path),
+ } => match mode {
+ CentralityType::Harmonic => {
+ entrypoint::Centrality::build_harmonic(webgraph_path, output_path)
+ }
+ CentralityType::OnlineHarmonic => {
+ entrypoint::Centrality::build_online(webgraph_path, output_path)
+ }
+ CentralityType::Similarity => {
+ entrypoint::Centrality::build_similarity(webgraph_path, output_path)
+ }
+ },
Commands::Webgraph { options } => match options {
WebgraphOptions::Master { config_path } => {
let config = load_toml_config(config_path);
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 81af4842..82825937 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -162,21 +162,39 @@ impl CentralityStore {
}
pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
+ Self::build_harmonic(graph, &output_path);
+ Self::build_online(graph, &output_path);
+ Self::build_similarity(graph, &output_path)
+ }
+
+ pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
let mut store = CentralityStore::open(output_path.as_ref());
store.node2id = graph.node_ids().collect();
let harmonic_centrality = HarmonicCentrality::calculate(graph);
-
Self::store_host(&output_path, &mut store, harmonic_centrality);
+ store.flush();
+ store
+ }
+
+ pub fn build_online<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
+ let mut store = CentralityStore::open(output_path.as_ref());
+
debug!("Begin online harmonic");
store.online_harmonic = OnlineHarmonicCentrality::new(graph, &store.harmonic);
+ store.flush();
+ store
+ }
+
+ pub fn build_similarity<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
+ let mut store = CentralityStore::open(output_path.as_ref());
+
debug!("Begin inbound similarity index construction");
store.inbound_similarity = InboundSimilarity::build(graph, &store.harmonic);
store.flush();
-
store
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 2fcb0611..50ef44ec 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -32,8 +32,8 @@ use crate::{
};
use super::{bitvec_similarity, centrality_store::HarmonicCentralityStore};
-const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_INBOUND: usize = 1000;
-const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES: usize = 1000;
+const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_INBOUND: usize = 2000;
+const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES: usize = 10000;
const SCORE_SCALE: f64 = 5.0;
pub struct Scorer {
diff --git a/justfile b/justfile
index 005fed6b..88efc4dc 100644
--- a/justfile
+++ b/justfile
@@ -20,9 +20,14 @@
@configure *ARGS:
pip3 install -r scripts/requirements.txt
./scripts/export_crossencoder
+ ./scripts/export_qa_model
cd frontend; npm install; npm run build
cargo run --release --all-features -- configure {{ARGS}}
+@centrality webgraph output:
+ ./scripts/build_harmonic {{webgraph}} {{output}}
+
+
@entity:
rm -rf data/entity
cargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entity
diff --git a/scripts/build_harmonic b/scripts/build_harmonic
new file mode 100755
index 00000000..0de84d75
--- /dev/null
+++ b/scripts/build_harmonic
@@ -0,0 +1,4 @@
+#!/bin/bash
+cargo run --release centrality $1 $2 harmonic
+cargo run --release centrality $1 $2 online-harmonic
+cargo run --release centrality $1 $2 similarity
\ No newline at end of file