Skip to content

web search done right

way faster similarity index creation. Also improves inverted index merging by dividing the merge into num_cpu merges where each merge merges into a single segment

Mikkel Denker2b36327fa159
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.toml
index d0474276..9d3a11c2 100644
--- a/configs/webgraph/local.toml
+++ b/configs/webgraph/local.toml
@@ -3,16 +3,6 @@ graph_base_path = "data/webgraph"
batch_size = 1
[warc_source]
-# type = "HTTP"
-# base_url = "https://data.commoncrawl.org/"
-# warc_paths_file = "warc.paths"
-type = "Local"
-folder = "./data/warc_files"
-names = [
- "CC-MAIN-20220116093137-20220116123137-00044.warc.gz",
- "CC-MAIN-20220116093137-20220116123137-00045.warc.gz",
- "CC-MAIN-20220116093137-20220116123137-00046.warc.gz",
- "CC-MAIN-20220116093137-20220116123137-00047.warc.gz",
- "CC-MAIN-20220116093137-20220116123137-00048.warc.gz",
- "CC-MAIN-20220116093137-20220116123137-00049.warc.gz",
- ]
\ No newline at end of file
+type = "HTTP"
+base_url = "https://data.commoncrawl.org/"
+warc_paths_file = "data/warc.paths"
\ No newline at end of file
diff --git a/core/Cargo.toml b/core/Cargo.toml
index fad948c6..4b3fc5de 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -104,4 +104,8 @@ harness = false
[[bench]]
name = "abstractive-summarizer"
+harness = false
+
+[[bench]]
+name = "build-similarity"
harness = false
\ No newline at end of file
diff --git a/core/benches/build-similarity.rs b/core/benches/build-similarity.rs
new file mode 100644
index 00000000..14d201be
--- /dev/null
+++ b/core/benches/build-similarity.rs
@@ -0,0 +1,21 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use stract::{
+ ranking::{centrality_store::CentralityStore, inbound_similarity::InboundSimilarity},
+ webgraph::WebgraphBuilder,
+};
+
+const WEBGRAPH_PATH: &str = "data/webgraph";
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+ let graph = WebgraphBuilder::new(WEBGRAPH_PATH).open();
+ let store = CentralityStore::build_harmonic(&graph, stract::gen_temp_path());
+
+ c.bench_function("Inbound similarity creation", |b| {
+ b.iter(|| {
+ InboundSimilarity::build(&graph, &store.harmonic);
+ })
+ });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 252ccd86..26e1ec12 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -122,7 +122,6 @@ fn create_inverted_index() -> Result<()> {
warc_paths: vec![warc_path.to_str().unwrap().to_string()],
base_path: out_path_tmp.to_str().unwrap().to_string(),
host_centrality_threshold: None,
- max_num_segments: 1,
minimum_clean_words: Some(40),
};
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 792b553a..f1c15e37 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -1,4 +1,3 @@
-use chrono::Utc;
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
@@ -14,9 +13,11 @@ use chrono::Utc;
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use chrono::Utc;
use futures::StreamExt;
use std::net::SocketAddr;
use std::path::Path;
+use std::thread;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
@@ -55,7 +56,6 @@ pub struct Job {
pub base_path: String,
pub host_centrality_threshold: Option<f64>,
pub minimum_clean_words: Option<usize>,
- pub max_num_segments: u32,
}
pub struct IndexingWorker {
@@ -231,10 +231,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
std::fs::remove_file(path).unwrap();
}
- index
- .inverted_index
- .merge_into_max_segments(job.max_num_segments)
- .unwrap();
+ index.inverted_index.merge_into_max_segments(1).unwrap();
info!("{} done", name);
@@ -339,7 +336,6 @@ impl Indexer {
.index_base_path
.clone()
.unwrap_or_else(|| "data/index".to_string()),
- max_num_segments: config.final_num_segments.unwrap_or(20),
minimum_clean_words: config.minimum_clean_words,
})
.collect_vec()
@@ -354,7 +350,7 @@ impl Indexer {
index
.inverted_index
- .merge_into_max_segments(config.final_num_segments.unwrap_or(20))
+ .merge_into_max_segments(num_cpus::get() as u32)
.unwrap();
});
@@ -425,27 +421,55 @@ impl Indexer {
.output_path
.clone()
.unwrap_or_else(|| "data/index".to_string()),
- max_num_segments: config.final_num_segments.unwrap_or(20),
minimum_clean_words: config.minimum_clean_words,
}),
)
.unwrap_or_default();
- Self::merge(indexes, config.final_num_segments.unwrap_or(20))?;
+ Self::merge(indexes)?;
Ok(())
}
- pub fn merge(indexes: Vec<IndexPointer>, num_segments: u32) -> Result<()> {
+ pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {
+ let num_indexes = indexes.len();
let mut it = indexes.into_iter();
- let mut index = Index::open(it.next().unwrap().0)?;
+ let num_cores = num_cpus::get();
+
+ let mut threads = Vec::new();
+
+ for _ in 0..(num_cores + 1) {
+ let indexes = it.by_ref().take(num_indexes / num_cores).collect_vec();
+
+ if indexes.is_empty() {
+ break;
+ }
+
+ threads.push(thread::spawn(move || {
+ let mut it = indexes.into_iter();
+ let mut index = Index::open(it.next().unwrap().0).unwrap();
+
+ for other in it {
+ let other_path = other.0;
+ let other = Index::open(&other_path).unwrap();
+ index = index.merge(other);
+ std::fs::remove_dir_all(other_path).unwrap();
+ index.inverted_index.merge_into_max_segments(1).unwrap();
+ }
+
+ index
+ }));
+ }
+
+ let mut indexes = Vec::new();
+ for thread in threads {
+ indexes.push(thread.join().unwrap());
+ }
+
+ let mut it = indexes.into_iter();
+ let mut index = it.next().unwrap();
for other in it {
- let other_path = other.0;
- let other = Index::open(&other_path)?;
index = index.merge(other);
-
- std::fs::remove_dir_all(other_path)?;
- index.inverted_index.merge_into_max_segments(num_segments)?;
}
Ok(())
diff --git a/core/src/lib.rs b/core/src/lib.rs
index e1de85c8..1b66f0db 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -81,7 +81,6 @@ mod widgets;
pub struct IndexingMasterConfig {
limit_warc_files: Option<usize>,
skip_num_warc_files: Option<usize>,
- final_num_segments: Option<u32>,
warc_source: WarcSource,
workers: Vec<String>,
batch_size: Option<usize>,
@@ -96,7 +95,6 @@ pub struct IndexingLocalConfig {
crawl_stability_path: Option<String>,
limit_warc_files: Option<usize>,
skip_num_warc_files: Option<usize>,
- final_num_segments: Option<u32>,
warc_source: WarcSource,
batch_size: Option<usize>,
webgraph_path: Option<String>,
diff --git a/core/src/main.rs b/core/src/main.rs
index 6c9dce60..39a8e153 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -88,6 +88,7 @@ enum Commands {
#[derive(Subcommand)]
enum CentralityType {
+ All,
Harmonic,
OnlineHarmonic,
Similarity,
@@ -132,7 +133,6 @@ enum IndexingOptions {
output_path: String,
},
Merge {
- num_segments: u32,
indexes: Vec<String>,
},
}
@@ -179,13 +179,9 @@ fn main() -> Result<()> {
wikipedia_dump_path,
output_path,
} => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,
- IndexingOptions::Merge {
- num_segments,
- indexes,
- } => entrypoint::Indexer::merge(
- indexes.into_iter().map(IndexPointer::from).collect(),
- num_segments,
- )?,
+ IndexingOptions::Merge { indexes } => {
+ entrypoint::Indexer::merge(indexes.into_iter().map(IndexPointer::from).collect())?
+ }
},
Commands::Centrality {
mode,
@@ -201,6 +197,11 @@ fn main() -> Result<()> {
CentralityType::Similarity => {
entrypoint::Centrality::build_similarity(webgraph_path, output_path)
}
+ CentralityType::All => {
+ entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);
+ entrypoint::Centrality::build_online(&webgraph_path, &output_path);
+ entrypoint::Centrality::build_similarity(&webgraph_path, &output_path);
+ }
},
Commands::Webgraph { options } => match options {
WebgraphOptions::Master { config_path } => {
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rs
index a608c222..db85e20c 100644
--- a/core/src/ranking/bitvec_similarity.rs
+++ b/core/src/ranking/bitvec_similarity.rs
@@ -14,7 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use itertools::Itertools;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug, Clone)]
@@ -25,33 +24,15 @@ struct Block {
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BitVec {
- blocks: Vec<Block>,
+ ranks: Vec<usize>,
sqrt_len: f64,
}
impl BitVec {
- pub fn new(data: &[bool]) -> Self {
- let mut blocks = Vec::new();
- let mut len = 0;
-
- for (offset, chunk) in data.iter().chunks(64).into_iter().enumerate() {
- let mut data = 0u64;
- for (i, b) in chunk.into_iter().enumerate() {
- if *b {
- data |= 1u64 << (63 - i);
- len += 1;
- }
- }
-
- if data == 0 {
- continue;
- }
-
- blocks.push(Block { data, offset })
- }
-
+ pub fn new(ranks: Vec<usize>) -> Self {
+ let len = ranks.len();
Self {
- blocks,
+ ranks,
sqrt_len: (len as f64).sqrt(),
}
}
@@ -62,12 +43,12 @@ impl BitVec {
let mut dot: u64 = 0;
- while i < self.blocks.len() && j < other.blocks.len() {
- match self.blocks[i].offset.cmp(&other.blocks[j].offset) {
+ while i < self.ranks.len() && j < other.ranks.len() {
+ match self.ranks[i].cmp(&other.ranks[j]) {
std::cmp::Ordering::Less => i += 1,
std::cmp::Ordering::Greater => j += 1,
std::cmp::Ordering::Equal => {
- dot += (self.blocks[i].data & other.blocks[j].data).count_ones() as u64;
+ dot += 1;
i += 1;
j += 1;
}
@@ -85,8 +66,17 @@ impl BitVec {
#[cfg(test)]
mod tests {
use super::*;
+ use itertools::Itertools;
use std::iter::repeat;
+ fn into_ranks(a: &[bool]) -> Vec<usize> {
+ a.iter()
+ .enumerate()
+ .filter(|(_, b)| **b)
+ .map(|(i, _)| i)
+ .collect()
+ }
+
fn naive_sim(a: &[bool], b: &[bool]) -> f64 {
let dot = a
.iter()
@@ -117,8 +107,8 @@ mod tests {
assert!(expected > 0.894);
- let a = BitVec::new(&a);
- let b = BitVec::new(&b);
+ let a = BitVec::new(into_ranks(&a));
+ let b = BitVec::new(into_ranks(&b));
assert!((expected - a.sim(&b)).abs() < 0.00001);
}
@@ -129,8 +119,8 @@ mod tests {
let b: Vec<_> = repeat(true).take(1000).collect();
- let a = BitVec::new(&a);
- let b = BitVec::new(&b);
+ let a = BitVec::new(into_ranks(&a));
+ let b = BitVec::new(into_ranks(&b));
assert_eq!(a.sim(&b), 0.0);
}
@@ -140,8 +130,8 @@ mod tests {
let a: Vec<_> = Vec::new();
let b: Vec<_> = Vec::new();
- let a = BitVec::new(&a);
- let b = BitVec::new(&b);
+ let a = BitVec::new(into_ranks(&a));
+ let b = BitVec::new(into_ranks(&b));
assert_eq!(a.sim(&b), 0.0);
}
@@ -163,8 +153,8 @@ mod tests {
assert!(expected < 0.01);
- let a = BitVec::new(&a);
- let b = BitVec::new(&b);
+ let a = BitVec::new(into_ranks(&a));
+ let b = BitVec::new(into_ranks(&b));
assert!((expected - a.sim(&b)).abs() < 0.00001);
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index d4a1e3b7..81e3cd5f 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -121,7 +121,6 @@ impl InboundSimilarity {
num_top_nodes: usize,
) -> Self {
let mut vectors = IntMap::new();
- let nodes: Vec<_> = graph.nodes().collect();
let mut top_nodes: BinaryHeap<Reverse<ScoredNode>> =
BinaryHeap::with_capacity(num_top_nodes);
@@ -143,19 +142,14 @@ impl InboundSimilarity {
}
}
- if let Some(max_node) = nodes.iter().max().copied() {
- let mut buf = vec![false; max_node.0 as usize + 1];
+ for node_id in top_nodes.into_iter().map(|n| n.0.node) {
+ let mut ranks = Vec::new();
- for node_id in top_nodes.into_iter().map(|n| n.0.node) {
- buf.clear();
- buf.resize(max_node.0 as usize + 1, false);
-
- for edge in graph.raw_ingoing_edges(&node_id) {
- buf[edge.from.0 as usize] = true;
- }
-
- vectors.insert(node_id.0, bitvec_similarity::BitVec::new(&buf));
+ for edge in graph.raw_ingoing_edges(&node_id) {
+ ranks.push(edge.from.0 as usize);
}
+
+ vectors.insert(node_id.0, bitvec_similarity::BitVec::new(ranks));
}
Self {