web search done right
way faster similarity index creation. Also improves inverted index merging by dividing the merge into num_cpu merges where each merge merges into a single segment
Mikkel Denker2b36327fa159
diff --git a/configs/webgraph/local.toml b/configs/webgraph/local.tomlindex d0474276..9d3a11c2 100644--- a/configs/webgraph/local.toml+++ b/configs/webgraph/local.toml@@ -3,16 +3,6 @@ graph_base_path = "data/webgraph"batch_size = 1[warc_source]-# type = "HTTP"-# base_url = "https://data.commoncrawl.org/"-# warc_paths_file = "warc.paths"-type = "Local"-folder = "./data/warc_files"-names = [- "CC-MAIN-20220116093137-20220116123137-00044.warc.gz",- "CC-MAIN-20220116093137-20220116123137-00045.warc.gz",- "CC-MAIN-20220116093137-20220116123137-00046.warc.gz",- "CC-MAIN-20220116093137-20220116123137-00047.warc.gz",- "CC-MAIN-20220116093137-20220116123137-00048.warc.gz",- "CC-MAIN-20220116093137-20220116123137-00049.warc.gz",- ]\ No newline at end of file+type = "HTTP"+base_url = "https://data.commoncrawl.org/"+warc_paths_file = "data/warc.paths"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex fad948c6..4b3fc5de 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -104,4 +104,8 @@ harness = false[[bench]]name = "abstractive-summarizer"+harness = false++[[bench]]+name = "build-similarity"harness = false\ No newline at end of filediff --git a/core/benches/build-similarity.rs b/core/benches/build-similarity.rsnew file mode 100644index 00000000..14d201be--- /dev/null+++ b/core/benches/build-similarity.rs@@ -0,0 +1,21 @@+use criterion::{criterion_group, criterion_main, Criterion};+use stract::{+ ranking::{centrality_store::CentralityStore, inbound_similarity::InboundSimilarity},+ webgraph::WebgraphBuilder,+};++const WEBGRAPH_PATH: &str = "data/webgraph";++pub fn criterion_benchmark(c: &mut Criterion) {+ let graph = WebgraphBuilder::new(WEBGRAPH_PATH).open();+ let store = CentralityStore::build_harmonic(&graph, stract::gen_temp_path());++ c.bench_function("Inbound similarity creation", |b| {+ b.iter(|| {+ InboundSimilarity::build(&graph, &store.harmonic);+ })+ });+}++criterion_group!(benches, criterion_benchmark);+criterion_main!(benches);diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 252ccd86..26e1ec12 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -122,7 +122,6 @@ fn create_inverted_index() -> Result<()> {warc_paths: vec![warc_path.to_str().unwrap().to_string()],base_path: out_path_tmp.to_str().unwrap().to_string(),host_centrality_threshold: None,- max_num_segments: 1,minimum_clean_words: Some(40),};diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 792b553a..f1c15e37 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -1,4 +1,3 @@-use chrono::Utc;// Stract is an open source web search engine.// Copyright (C) 2023 Stract ApS//@@ -14,9 +13,11 @@ use chrono::Utc;//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use chrono::Utc;use futures::StreamExt;use std::net::SocketAddr;use std::path::Path;+use std::thread;use itertools::Itertools;use serde::{Deserialize, Serialize};@@ -55,7 +56,6 @@ pub struct Job {pub base_path: String,pub host_centrality_threshold: Option<f64>,pub minimum_clean_words: Option<usize>,- pub max_num_segments: u32,}pub struct IndexingWorker {@@ -231,10 +231,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {std::fs::remove_file(path).unwrap();}- index- .inverted_index- .merge_into_max_segments(job.max_num_segments)- .unwrap();+ index.inverted_index.merge_into_max_segments(1).unwrap();info!("{} done", name);@@ -339,7 +336,6 @@ impl Indexer {.index_base_path.clone().unwrap_or_else(|| "data/index".to_string()),- max_num_segments: config.final_num_segments.unwrap_or(20),minimum_clean_words: config.minimum_clean_words,}).collect_vec()@@ -354,7 +350,7 @@ impl Indexer {index.inverted_index- .merge_into_max_segments(config.final_num_segments.unwrap_or(20))+ .merge_into_max_segments(num_cpus::get() as u32).unwrap();});@@ -425,27 +421,55 @@ impl Indexer {.output_path.clone().unwrap_or_else(|| "data/index".to_string()),- max_num_segments: config.final_num_segments.unwrap_or(20),minimum_clean_words: config.minimum_clean_words,}),).unwrap_or_default();- Self::merge(indexes, config.final_num_segments.unwrap_or(20))?;+ Self::merge(indexes)?;Ok(())}- pub fn merge(indexes: Vec<IndexPointer>, num_segments: u32) -> Result<()> {+ pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {+ let num_indexes = indexes.len();let mut it = indexes.into_iter();- let mut index = Index::open(it.next().unwrap().0)?;+ let num_cores = num_cpus::get();++ let mut threads = Vec::new();++ for _ in 0..(num_cores + 1) {+ let indexes = it.by_ref().take(num_indexes / num_cores).collect_vec();++ if indexes.is_empty() {+ break;+ }++ threads.push(thread::spawn(move || {+ let mut it = indexes.into_iter();+ let mut index = Index::open(it.next().unwrap().0).unwrap();++ for other in it {+ let other_path = other.0;+ let other = Index::open(&other_path).unwrap();+ index = index.merge(other);+ std::fs::remove_dir_all(other_path).unwrap();+ index.inverted_index.merge_into_max_segments(1).unwrap();+ }++ index+ }));+ }++ let mut indexes = Vec::new();+ for thread in threads {+ indexes.push(thread.join().unwrap());+ }++ let mut it = indexes.into_iter();+ let mut index = it.next().unwrap();for other in it {- let other_path = other.0;- let other = Index::open(&other_path)?;index = index.merge(other);-- std::fs::remove_dir_all(other_path)?;- index.inverted_index.merge_into_max_segments(num_segments)?;}Ok(())diff --git a/core/src/lib.rs b/core/src/lib.rsindex e1de85c8..1b66f0db 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -81,7 +81,6 @@ mod widgets;pub struct IndexingMasterConfig {limit_warc_files: Option<usize>,skip_num_warc_files: Option<usize>,- final_num_segments: Option<u32>,warc_source: WarcSource,workers: Vec<String>,batch_size: Option<usize>,@@ -96,7 +95,6 @@ pub struct IndexingLocalConfig {crawl_stability_path: Option<String>,limit_warc_files: Option<usize>,skip_num_warc_files: Option<usize>,- final_num_segments: Option<u32>,warc_source: WarcSource,batch_size: Option<usize>,webgraph_path: Option<String>,diff --git a/core/src/main.rs b/core/src/main.rsindex 6c9dce60..39a8e153 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -88,6 +88,7 @@ enum Commands {#[derive(Subcommand)]enum CentralityType {+ All,Harmonic,OnlineHarmonic,Similarity,@@ -132,7 +133,6 @@ enum IndexingOptions {output_path: String,},Merge {- num_segments: u32,indexes: Vec<String>,},}@@ -179,13 +179,9 @@ fn main() -> Result<()> {wikipedia_dump_path,output_path,} => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,- IndexingOptions::Merge {- num_segments,- indexes,- } => entrypoint::Indexer::merge(- indexes.into_iter().map(IndexPointer::from).collect(),- num_segments,- )?,+ IndexingOptions::Merge { indexes } => {+ entrypoint::Indexer::merge(indexes.into_iter().map(IndexPointer::from).collect())?+ }},Commands::Centrality {mode,@@ -201,6 +197,11 @@ fn main() -> Result<()> {CentralityType::Similarity => {entrypoint::Centrality::build_similarity(webgraph_path, output_path)}+ CentralityType::All => {+ entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);+ entrypoint::Centrality::build_online(&webgraph_path, &output_path);+ entrypoint::Centrality::build_similarity(&webgraph_path, &output_path);+ }},Commands::Webgraph { options } => match options {WebgraphOptions::Master { config_path } => {diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rsindex a608c222..db85e20c 100644--- a/core/src/ranking/bitvec_similarity.rs+++ b/core/src/ranking/bitvec_similarity.rs@@ -14,7 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use itertools::Itertools;use serde::{Deserialize, Serialize};#[derive(Serialize, Deserialize, Debug, Clone)]@@ -25,33 +24,15 @@ struct Block {#[derive(Serialize, Deserialize, Debug, Clone)]pub struct BitVec {- blocks: Vec<Block>,+ ranks: Vec<usize>,sqrt_len: f64,}impl BitVec {- pub fn new(data: &[bool]) -> Self {- let mut blocks = Vec::new();- let mut len = 0;-- for (offset, chunk) in data.iter().chunks(64).into_iter().enumerate() {- let mut data = 0u64;- for (i, b) in chunk.into_iter().enumerate() {- if *b {- data |= 1u64 << (63 - i);- len += 1;- }- }-- if data == 0 {- continue;- }-- blocks.push(Block { data, offset })- }-+ pub fn new(ranks: Vec<usize>) -> Self {+ let len = ranks.len();Self {- blocks,+ ranks,sqrt_len: (len as f64).sqrt(),}}@@ -62,12 +43,12 @@ impl BitVec {let mut dot: u64 = 0;- while i < self.blocks.len() && j < other.blocks.len() {- match self.blocks[i].offset.cmp(&other.blocks[j].offset) {+ while i < self.ranks.len() && j < other.ranks.len() {+ match self.ranks[i].cmp(&other.ranks[j]) {std::cmp::Ordering::Less => i += 1,std::cmp::Ordering::Greater => j += 1,std::cmp::Ordering::Equal => {- dot += (self.blocks[i].data & other.blocks[j].data).count_ones() as u64;+ dot += 1;i += 1;j += 1;}@@ -85,8 +66,17 @@ impl BitVec {#[cfg(test)]mod tests {use super::*;+ use itertools::Itertools;use std::iter::repeat;+ fn into_ranks(a: &[bool]) -> Vec<usize> {+ a.iter()+ .enumerate()+ .filter(|(_, b)| **b)+ .map(|(i, _)| i)+ .collect()+ }+fn naive_sim(a: &[bool], b: &[bool]) -> f64 {let dot = a.iter()@@ -117,8 +107,8 @@ mod tests {assert!(expected > 0.894);- let a = BitVec::new(&a);- let b = BitVec::new(&b);+ let a = BitVec::new(into_ranks(&a));+ let b = BitVec::new(into_ranks(&b));assert!((expected - a.sim(&b)).abs() < 0.00001);}@@ -129,8 +119,8 @@ mod tests {let b: Vec<_> = repeat(true).take(1000).collect();- let a = BitVec::new(&a);- let b = BitVec::new(&b);+ let a = BitVec::new(into_ranks(&a));+ let b = BitVec::new(into_ranks(&b));assert_eq!(a.sim(&b), 0.0);}@@ -140,8 +130,8 @@ mod tests {let a: Vec<_> = Vec::new();let b: Vec<_> = Vec::new();- let a = BitVec::new(&a);- let b = BitVec::new(&b);+ let a = BitVec::new(into_ranks(&a));+ let b = BitVec::new(into_ranks(&b));assert_eq!(a.sim(&b), 0.0);}@@ -163,8 +153,8 @@ mod tests {assert!(expected < 0.01);- let a = BitVec::new(&a);- let b = BitVec::new(&b);+ let a = BitVec::new(into_ranks(&a));+ let b = BitVec::new(into_ranks(&b));assert!((expected - a.sim(&b)).abs() < 0.00001);}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex d4a1e3b7..81e3cd5f 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -121,7 +121,6 @@ impl InboundSimilarity {num_top_nodes: usize,) -> Self {let mut vectors = IntMap::new();- let nodes: Vec<_> = graph.nodes().collect();let mut top_nodes: BinaryHeap<Reverse<ScoredNode>> =BinaryHeap::with_capacity(num_top_nodes);@@ -143,19 +142,14 @@ impl InboundSimilarity {}}- if let Some(max_node) = nodes.iter().max().copied() {- let mut buf = vec![false; max_node.0 as usize + 1];+ for node_id in top_nodes.into_iter().map(|n| n.0.node) {+ let mut ranks = Vec::new();- for node_id in top_nodes.into_iter().map(|n| n.0.node) {- buf.clear();- buf.resize(max_node.0 as usize + 1, false);-- for edge in graph.raw_ingoing_edges(&node_id) {- buf[edge.from.0 as usize] = true;- }-- vectors.insert(node_id.0, bitvec_similarity::BitVec::new(&buf));+ for edge in graph.raw_ingoing_edges(&node_id) {+ ranks.push(edge.from.0 as usize);}++ vectors.insert(node_id.0, bitvec_similarity::BitVec::new(ranks));}Self {