web search done right
Split centrality building into separate processes. This is a hotfix to reduce the memory for each step
Mikkel Denker8a6751cf244d
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex d306b1e3..f5f5c32a 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -21,9 +21,18 @@ use crate::{ranking::centrality_store::CentralityStore, webgraph::WebgraphBuildepub struct Centrality {}impl Centrality {- pub fn run<P: AsRef<Path>>(webgraph_path: P, output_path: P) {+ pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, output_path: P) {let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();+ CentralityStore::build_harmonic(&graph, output_path);+ }- CentralityStore::build(&graph, output_path);+ pub fn build_online<P: AsRef<Path>>(webgraph_path: P, output_path: P) {+ let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();+ CentralityStore::build_online(&graph, output_path);+ }++ pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {+ let graph = WebgraphBuilder::new(webgraph_path).read_only(true).open();+ CentralityStore::build_similarity(&graph, output_path);}}diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 2fab4029..ec50c64e 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -104,7 +104,10 @@ fn calculate_centrality() {debug!("Calculating centrality");let webgraph_path = Path::new(DATA_PATH).join("webgraph");let out_path = Path::new(DATA_PATH).join("centrality");- Centrality::run(webgraph_path, out_path);++ Centrality::build_harmonic(&webgraph_path, &out_path);+ Centrality::build_online(&webgraph_path, &out_path);+ Centrality::build_similarity(&webgraph_path, &out_path);}fn create_inverted_index() -> Result<()> {diff --git a/core/src/main.rs b/core/src/main.rsindex 4ca73b97..c5c1a546 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -43,6 +43,8 @@ enum Commands {options: IndexingOptions,},Centrality {+ #[clap(subcommand)]+ mode: CentralityType,webgraph_path: String,output_path: String,},@@ -84,6 +86,13 @@ enum Commands {},}+#[derive(Subcommand)]+enum CentralityType {+ Harmonic,+ OnlineHarmonic,+ Similarity,+}+#[derive(Subcommand)]enum WebgraphOptions {Master {@@ -179,9 +188,20 @@ fn main() -> Result<()> {)?,},Commands::Centrality {+ mode,webgraph_path,output_path,- } => entrypoint::Centrality::run(webgraph_path, output_path),+ } => match mode {+ CentralityType::Harmonic => {+ entrypoint::Centrality::build_harmonic(webgraph_path, output_path)+ }+ CentralityType::OnlineHarmonic => {+ entrypoint::Centrality::build_online(webgraph_path, output_path)+ }+ CentralityType::Similarity => {+ entrypoint::Centrality::build_similarity(webgraph_path, output_path)+ }+ },Commands::Webgraph { options } => match options {WebgraphOptions::Master { config_path } => {let config = load_toml_config(config_path);diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 81af4842..82825937 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -162,21 +162,39 @@ impl CentralityStore {}pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {+ Self::build_harmonic(graph, &output_path);+ Self::build_online(graph, &output_path);+ Self::build_similarity(graph, &output_path)+ }++ pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {let mut store = CentralityStore::open(output_path.as_ref());store.node2id = graph.node_ids().collect();let harmonic_centrality = HarmonicCentrality::calculate(graph);-Self::store_host(&output_path, &mut store, harmonic_centrality);+ store.flush();+ store+ }++ pub fn build_online<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {+ let mut store = CentralityStore::open(output_path.as_ref());+debug!("Begin online harmonic");store.online_harmonic = OnlineHarmonicCentrality::new(graph, &store.harmonic);+ store.flush();+ store+ }++ pub fn build_similarity<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {+ let mut store = CentralityStore::open(output_path.as_ref());+debug!("Begin inbound similarity index construction");store.inbound_similarity = InboundSimilarity::build(graph, &store.harmonic);store.flush();-store}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 2fcb0611..50ef44ec 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -32,8 +32,8 @@ use crate::{};use super::{bitvec_similarity, centrality_store::HarmonicCentralityStore};-const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_INBOUND: usize = 1000;-const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES: usize = 1000;+const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_INBOUND: usize = 2000;+const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES: usize = 10000;const SCORE_SCALE: f64 = 5.0;pub struct Scorer {diff --git a/justfile b/justfileindex 005fed6b..88efc4dc 100644--- a/justfile+++ b/justfile@@ -20,9 +20,14 @@@configure *ARGS:pip3 install -r scripts/requirements.txt./scripts/export_crossencoder+ ./scripts/export_qa_modelcd frontend; npm install; npm run buildcargo run --release --all-features -- configure {{ARGS}}+@centrality webgraph output:+ ./scripts/build_harmonic {{webgraph}} {{output}}++@entity:rm -rf data/entitycargo run --release -- indexer entity data/enwiki_subset.xml.bz2 data/entitydiff --git a/scripts/build_harmonic b/scripts/build_harmonicnew file mode 100755index 00000000..0de84d75--- /dev/null+++ b/scripts/build_harmonic@@ -0,0 +1,4 @@+#!/bin/bash+cargo run --release centrality $1 $2 harmonic+cargo run --release centrality $1 $2 online-harmonic+cargo run --release centrality $1 $2 similarity\ No newline at end of file