web search done right
Derived centrality. New centrality metric that is derived from the host harmonic centrality. This can hopefully be calculated on the enormous page webgraph.
Mikkel Denker4c49e2deb8c2
diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex e076dd66..2700a2e2 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -3,15 +3,9 @@ limit_warc_files = 20# minimum_clean_words = 40[warc_source]-# type = "Local"-# folder = "./data"-# names = ["sample.warc.gz"]-# type = "HTTP"-# base_url = "https://data.commoncrawl.org/"-# warc_paths_file = "data/warc.paths"-access_key = "crawl_nOf7tywztpjg2"-bucket = "crawl"-endpoint = "http://s3.trystract.com"-folder = "400m_try11"-secret_key = "4Kzo0CPdQYnlFHRgwKpUA7mACzSLsA6OpG6qDFl4"-type = "S3"+base_url = "https://data.commoncrawl.org/"+folder = "./data"+names = ["sample.warc.gz"]+type = "Local"+type = "HTTP"+warc_paths_file = "data/warc.paths"diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rsindex 64e46668..80f0d4d7 100644--- a/core/benches/search-preindexed.rs+++ b/core/benches/search-preindexed.rs@@ -1,8 +1,10 @@+use std::path::Path;+use criterion::{criterion_group, criterion_main, Criterion};use optics::SiteRankings;use stract::{index::Index,- ranking::centrality_store::CentralityStore,+ ranking::inbound_similarity::InboundSimilarity,searcher::{LocalSearcher, SearchQuery},};@@ -39,7 +41,9 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();let mut searcher = LocalSearcher::new(index);- searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH).into());+ searcher.set_inbound_similarity(+ InboundSimilarity::open(Path::new(CENTRALITY_PATH).join("inbound_similarity")).unwrap(),+ );for _ in 0..1000 {bench!("the", searcher, c);diff --git a/core/examples/print_similar_sites.rs b/core/examples/print_similar_sites.rsindex dc05faeb..fac809c6 100644--- a/core/examples/print_similar_sites.rs+++ b/core/examples/print_similar_sites.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use stract::{- ranking::centrality_store::CentralityStore,+ ranking::inbound_similarity::InboundSimilarity,similar_sites::{ScoredNode, SimilarSitesFinder},webgraph::WebgraphBuilder,};@@ -40,11 +40,11 @@ fn print_top_nodes(liked_sites: &[&str], top_n: usize, similarity_finder: &Similpub fn main() {const TOP_N: usize = 50;let graph = WebgraphBuilder::new("data/webgraph").open();- let store = CentralityStore::open("data/centrality");+ let inbound_similarity = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();let similarity_finder = SimilarSitesFinder::new(graph.into(),- store.inbound_similarity,+ inbound_similarity,stract::config::defaults::WebgraphServer::max_similar_sites(),);diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 10f1b162..8b25b03e 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -243,7 +243,7 @@ pub struct SearchServerConfig {pub shard_id: ShardId,pub index_path: String,pub entity_index_path: Option<String>,- pub host_centrality_store_path: Option<String>,+ pub centrality_store_path: Option<String>,pub linear_model_path: Option<String>,pub lambda_model_path: Option<String>,pub host: SocketAddr,diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex 530fc566..9fab3b48 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -14,29 +14,138 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::path::Path;+use anyhow::Result;+use std::{cmp::Reverse, collections::BinaryHeap, fs::File, path::Path};use crate::{- ranking::centrality_store::CentralityStore,- webgraph::{Compression, WebgraphBuilder},+ kv::{rocksdb_store::RocksDbStore, Kv},+ ranking::inbound_similarity::InboundSimilarity,+ webgraph::{+ centrality::{derived_harmonic::DerivedCentrality, harmonic::HarmonicCentrality},+ Node, WebgraphBuilder,+ },};-pub struct Centrality {}+fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {+ let csv_file = File::options()+ .write(true)+ .create(true)+ .truncate(true)+ .open(output)+ .unwrap();++ let mut data = data;+ data.sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));+ let mut wtr = csv::Writer::from_writer(csv_file);+ for (node, centrality) in data {+ wtr.write_record(&[node.name, centrality.to_string()])+ .unwrap();+ }+ wtr.flush().unwrap();+}++struct SortableFloat(f64);++impl PartialEq for SortableFloat {+ fn eq(&self, other: &Self) -> bool {+ self.0 == other.0+ }+}++impl Eq for SortableFloat {}++impl PartialOrd for SortableFloat {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}++impl Ord for SortableFloat {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.0.total_cmp(&other.0)+ }+}++pub struct Centrality;impl Centrality {- pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, output_path: P) {+ pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) {tracing::info!("Building harmonic centrality");- let graph = WebgraphBuilder::new(webgraph_path)- .compression(Compression::Lz4)- .open();- CentralityStore::build_harmonic(&graph, output_path);+ let graph = WebgraphBuilder::new(webgraph_path).open();+ let harmonic_centrality = HarmonicCentrality::calculate(&graph);+ let store = RocksDbStore::open(base_output.as_ref().join("harmonic"));++ for (node_id, centrality) in harmonic_centrality.iter() {+ store.insert(*node_id, centrality);+ }+ store.flush();++ let mut top_harmonics = BinaryHeap::new();++ for (node_id, centrality) in harmonic_centrality.iter() {+ if top_harmonics.len() < 1_000_000 {+ top_harmonics.push((Reverse(SortableFloat(centrality)), *node_id));+ } else {+ let mut min = top_harmonics.peek_mut().unwrap();+ if min.0 .0 < SortableFloat(centrality) {+ *min = (Reverse(SortableFloat(centrality)), *node_id);+ }+ }+ }++ let harmonics: Vec<_> = top_harmonics+ .into_iter()+ .map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))+ .collect();++ store_csv(harmonics, base_output.as_ref().join("harmonic.csv"));}- pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {+ pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, base_output: P) {tracing::info!("Building inbound similarity");- let graph = WebgraphBuilder::new(webgraph_path)- .compression(Compression::Lz4)- .open();- CentralityStore::build_similarity(&graph, output_path);+ let graph = WebgraphBuilder::new(webgraph_path).open();++ let sim = InboundSimilarity::build(&graph);++ sim.save(base_output.as_ref().join("inbound_similarity"))+ .unwrap();+ }++ pub fn build_derived_harmonic<P: AsRef<Path>>(+ webgraph_path: P,+ host_centrality_path: P,+ base_output: P,+ ) -> Result<()> {+ tracing::info!("Building derived harmonic centrality");+ let graph = WebgraphBuilder::new(webgraph_path).open();+ let host_centrality = RocksDbStore::open(host_centrality_path.as_ref().join("harmonic"));++ let derived = DerivedCentrality::build(+ &host_centrality,+ &graph,+ base_output.as_ref().join("derived_harmonic"),+ )?;++ let mut top_nodes = BinaryHeap::new();++ for (node_id, centrality) in derived.iter() {+ if top_nodes.len() < 1_000_000 {+ top_nodes.push((Reverse(SortableFloat(centrality)), node_id));+ } else {+ let mut min = top_nodes.peek_mut().unwrap();+ if min.0 .0 < SortableFloat(centrality) {+ *min = (Reverse(SortableFloat(centrality)), node_id);+ }+ }+ }++ let derived: Vec<_> = top_nodes+ .into_iter()+ .map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))+ .collect();++ store_csv(derived, base_output.as_ref().join("derived_centrality.csv"));++ Ok(())}}diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 96947c29..2714ce28 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -101,8 +101,17 @@ fn calculate_centrality() {let webgraph_path = Path::new(DATA_PATH).join("webgraph_host");let out_path = Path::new(DATA_PATH).join("centrality");- Centrality::build_harmonic(&webgraph_path, &out_path);- Centrality::build_similarity(&webgraph_path, &out_path);+ if !out_path.exists() {+ Centrality::build_harmonic(&webgraph_path, &out_path);+ Centrality::build_similarity(&webgraph_path, &out_path);+ }++ let webgraph_page = Path::new(DATA_PATH).join("webgraph_page");+ let out_path_page = Path::new(DATA_PATH).join("centrality_page");++ if !out_path_page.exists() {+ Centrality::build_derived_harmonic(webgraph_page, out_path, out_path_page).unwrap();+ }}fn create_inverted_index() -> Result<()> {@@ -132,10 +141,11 @@ fn create_inverted_index() -> Result<()> {let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");let centrality_path = Path::new(DATA_PATH).join("centrality");+ let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");let worker = indexer::IndexingWorker::new(centrality_path.to_str().unwrap().to_string(),- None,+ Some(page_centrality_path.to_str().unwrap().to_string()),Some(webgraph_path.to_str().unwrap().to_string()),Some(Path::new(DATA_PATH)diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 2243177c..897a5e39 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -26,10 +26,11 @@ use tracing::{debug, info, trace, warn};use crate::config;use crate::entrypoint::download_all_warc_files;use crate::index::{FrozenIndex, Index};+use crate::kv::rocksdb_store::RocksDbStore;+use crate::kv::Kv;use crate::mapreduce::{Map, Reduce, Worker};-use crate::ranking::centrality_store::IndexerCentralityStore;use crate::ranking::SignalAggregator;-use crate::webgraph::{Node, Webgraph, WebgraphBuilder};+use crate::webgraph::{Node, NodeID, Webgraph, WebgraphBuilder};use crate::webpage::{safety_classifier, Html, Webpage};use crate::{human_website_annotations, Result};@@ -72,8 +73,8 @@ pub struct Job {}pub struct IndexingWorker {- host_centrality_store: IndexerCentralityStore,- page_centrality_store: Option<IndexerCentralityStore>,+ host_centrality_store: RocksDbStore<NodeID, f64>,+ page_centrality_store: Option<RocksDbStore<NodeID, f64>>,webgraph: Option<Webgraph>,topics: Option<human_website_annotations::Mapper>,safety_classifier: Option<safety_classifier::Model>,@@ -88,8 +89,11 @@ impl IndexingWorker {safety_classifier_path: Option<String>,) -> Self {Self {- host_centrality_store: IndexerCentralityStore::open(host_centrality_store_path),- page_centrality_store: page_centrality_store_path.map(IndexerCentralityStore::open),+ host_centrality_store: RocksDbStore::open(+ Path::new(&host_centrality_store_path).join("harmonic"),+ ),+ page_centrality_store: page_centrality_store_path+ .map(|p| RocksDbStore::open(Path::new(&p).join("derived_harmonic"))),webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).single_threaded().open()),topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),safety_classifier: safety_classifier_path@@ -148,7 +152,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let mut host_centrality = worker.host_centrality_store- .harmonic.get(&host_node_id).unwrap_or_default();@@ -194,7 +197,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {if let Some(store) = worker.page_centrality_store.as_ref() {let node_id = node.id();- page_centrality = store.harmonic.get(&node_id).unwrap_or_default();+ page_centrality = store.get(&node_id).unwrap_or_default();}if host_centrality > 0.0 {diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex a3efc046..aa388cdb 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::collections::HashMap;+use std::{collections::HashMap, path::Path};use serde::{Deserialize, Serialize};use tracing::info;@@ -32,7 +32,7 @@ use crate::{index::Index,inverted_index::{self, RetrievedWebpage},ranking::{- centrality_store::SearchCentralityStore,+ inbound_similarity::InboundSimilarity,models::{lambdamart::LambdaMART, linear::LinearRegression},},searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},@@ -63,8 +63,8 @@ impl SearchService {.entity_index_path.map(|path| EntityIndex::open(path).unwrap());let centrality_store = config- .host_centrality_store_path- .map(SearchCentralityStore::open);+ .centrality_store_path+ .map(|p| InboundSimilarity::open(Path::new(&p).join("inbound_similarity")).unwrap());let search_index = Index::open(config.index_path)?;let mut local_searcher = LocalSearcher::new(search_index);@@ -74,7 +74,7 @@ impl SearchService {}if let Some(centrality_store) = centrality_store {- local_searcher.set_centrality_store(centrality_store);+ local_searcher.set_inbound_similarity(centrality_store);}if let Some(model_path) = config.linear_model_path {diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rsindex 90f26af4..a825799c 100644--- a/core/src/kv/rocksdb_store.rs+++ b/core/src/kv/rocksdb_store.rs@@ -24,12 +24,21 @@ use serde::{de::DeserializeOwned, Serialize};use crate::kv::Kv;-pub struct RocksDbStore {+pub struct RocksDbStore<K, V>+where+ K: Serialize + DeserializeOwned,+ V: Serialize + DeserializeOwned,+{db: DB,_cache: rocksdb::Cache,+ _phantom: PhantomData<(K, V)>,}-impl RocksDbStore {+impl<K, V> RocksDbStore<K, V>+where+ K: Serialize + DeserializeOwned,+ V: Serialize + DeserializeOwned,+{pub fn open<P>(path: P) -> SelfwhereP: AsRef<std::path::Path>,@@ -66,21 +75,30 @@ impl RocksDbStore {let db = DB::open(&options, path).expect("unable to open rocks db");- Self { db, _cache: cache }+ Self {+ db,+ _cache: cache,+ _phantom: PhantomData,+ }}}-impl<K, V> Kv<K, V> for RocksDbStore+impl<K, V> Kv<K, V> for RocksDbStore<K, V>where- K: Serialize + DeserializeOwned + 'static,- V: Serialize + DeserializeOwned + 'static,+ K: Serialize + DeserializeOwned + 'static + Send + Sync,+ V: Serialize + DeserializeOwned + 'static + Send + Sync,{fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {self.db.get(key).expect("failed to retrieve key")}fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {- self.db.put(key, value).expect("failed to insert value");+ let mut opt = rocksdb::WriteOptions::default();+ opt.disable_wal(true);++ self.db+ .put_opt(key, value, &opt)+ .expect("failed to insert value");}fn flush(&self) {diff --git a/core/src/main.rs b/core/src/main.rsindex 223ed936..6259d45c 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -55,9 +55,7 @@ enum Commands {/// Calculate centrality metrics that estimates a websites importance. These metrics are used to rank search results.Centrality {#[clap(subcommand)]- mode: CentralityType,- webgraph_path: String,- output_path: String,+ mode: CentralityMode,},/// Parse the DMOZ dataset. DMOZ contains a list of websites and their categories.@@ -140,15 +138,18 @@ enum SafetyClassifierOptions {}#[derive(Subcommand)]-enum CentralityType {- /// Calculate all the supported centrality metrics.- All,- /// Calculate harmonic centrality. Harmonic centrality is the sum of the inverse distance from all- /// other nodes to a given node. It is a measure of how central a node (webpage) is in a graph.- Harmonic,- /// Create an index that can be used to quickly calculate the inbound link similarity between two nodes.- /// This is used to calculate the similarity between two webpages.- Similarity,+enum CentralityMode {+ /// Calculate metrics for the host webgraph.+ Host {+ webgraph_path: String,+ output_path: String,+ },+ /// Calculate metrics for the page webgraph.+ Page {+ webgraph_path: String,+ host_centrality_path: String,+ output_path: String,+ },}#[derive(Subcommand)]@@ -217,22 +218,24 @@ fn main() -> Result<()> {entrypoint::Indexer::merge(indexes.into_iter().map(IndexPointer::from).collect())?}},- Commands::Centrality {- mode,- webgraph_path,- output_path,- } => {+ Commands::Centrality { mode } => {match mode {- CentralityType::Harmonic => {- entrypoint::Centrality::build_harmonic(webgraph_path, output_path)- }- CentralityType::Similarity => {- entrypoint::Centrality::build_similarity(webgraph_path, output_path)- }- CentralityType::All => {+ CentralityMode::Host {+ webgraph_path,+ output_path,+ } => {entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);entrypoint::Centrality::build_similarity(&webgraph_path, &output_path);}+ CentralityMode::Page {+ webgraph_path,+ host_centrality_path,+ output_path,+ } => entrypoint::Centrality::build_derived_harmonic(+ webgraph_path,+ host_centrality_path,+ output_path,+ )?,}tracing::info!("Done");}diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 948b8906..4433ebae 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -238,7 +238,7 @@ mod tests {use crate::{gen_temp_path,index::Index,- ranking::centrality_store::CentralityStore,+ ranking::inbound_similarity::InboundSimilarity,searcher::{LocalSearcher, SearchQuery},webgraph::{Node, WebgraphWriter},webpage::{Html, Webpage},@@ -658,8 +658,6 @@ mod tests {let graph = writer.finalize();- let centrality_store = CentralityStore::build(&graph, gen_temp_path());-index.insert(Webpage {html: Html::parse(@@ -755,7 +753,7 @@ mod tests {index.commit().expect("failed to commit index");let mut searcher = LocalSearcher::from(index);- searcher.set_centrality_store(centrality_store.into());+ searcher.set_inbound_similarity(InboundSimilarity::build(&graph));let res = searcher.search(&SearchQuery {diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsdeleted file mode 100644index 11b96a1e..00000000--- a/core/src/ranking/centrality_store.rs+++ /dev/null@@ -1,159 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{fs::File, path::Path};--use tracing::debug;-use tracing::log::trace;--use crate::{- kv::{rocksdb_store::RocksDbStore, Kv},- webgraph::{centrality::harmonic::HarmonicCentrality, NodeID, Webgraph},-};--use super::inbound_similarity::InboundSimilarity;--pub type HarmonicCentralityStore = Box<dyn Kv<NodeID, f64> + Send + Sync>;--pub struct IndexerCentralityStore {- pub harmonic: HarmonicCentralityStore,-}--impl IndexerCentralityStore {- pub fn open<P: AsRef<Path>>(path: P) -> Self {- Self {- harmonic: Box::new(RocksDbStore::open(path.as_ref().join("harmonic"))),- }- }-}--impl From<CentralityStore> for IndexerCentralityStore {- fn from(store: CentralityStore) -> Self {- Self {- harmonic: store.harmonic,- }- }-}--pub struct SearchCentralityStore {- pub inbound_similarity: InboundSimilarity,-}--impl SearchCentralityStore {- pub fn open<P: AsRef<Path>>(path: P) -> Self {- Self {- inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))- .unwrap(),- }- }-}--impl From<CentralityStore> for SearchCentralityStore {- fn from(store: CentralityStore) -> Self {- Self {- inbound_similarity: store.inbound_similarity,- }- }-}--pub struct CentralityStore {- pub harmonic: HarmonicCentralityStore,- pub inbound_similarity: InboundSimilarity,- pub base_path: String,-}--impl CentralityStore {- pub fn open<P: AsRef<Path>>(path: P) -> Self {- Self {- harmonic: Box::new(RocksDbStore::open(path.as_ref().join("harmonic"))),- inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))- .ok()- .unwrap_or_default(),- base_path: path.as_ref().to_str().unwrap().to_string(),- }- }-- fn store_harmonic<P: AsRef<Path>>(- output_path: P,- store: &CentralityStore,- harmonic_centrality: HarmonicCentrality,- graph: &Webgraph,- ) {- let csv_file = File::options()- .write(true)- .create(true)- .truncate(true)- .open(output_path.as_ref().join("harmonic.csv"))- .unwrap();-- for (node_id, centrality) in harmonic_centrality.iter() {- store.harmonic.insert(*node_id, centrality);- }- store.harmonic.flush();-- let mut harmonic: Vec<_> = harmonic_centrality- .iter()- .map(|(node, centrality)| (*node, centrality))- .take(1_000_000)- .collect();-- harmonic.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));- let mut wtr = csv::Writer::from_writer(csv_file);- for (node_id, centrality) in harmonic {- let node = graph.id2node(&node_id).unwrap();-- wtr.write_record(&[node.name, centrality.to_string()])- .unwrap();- }- wtr.flush().unwrap();- }-- pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {- Self::build_harmonic(graph, &output_path);- Self::build_similarity(graph, &output_path)- }-- pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {- let store = CentralityStore::open(output_path.as_ref());-- let harmonic_centrality =- HarmonicCentrality::calculate(graph, output_path.as_ref().join("calculations"));- Self::store_harmonic(&output_path, &store, harmonic_centrality, graph);-- store.flush();- store- }-- pub fn build_similarity<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {- let mut store = CentralityStore::open(output_path.as_ref());-- debug!("Begin inbound similarity index construction");- store.inbound_similarity = InboundSimilarity::build(graph);-- store.flush();- store- }-- pub fn flush(&self) {- trace!("flushing");- self.harmonic.flush();-- trace!("saving inbound similarity");- self.inbound_similarity- .save(Path::new(&self.base_path).join("inbound_similarity"))- .unwrap();- }-}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 7f09233d..6d070a49 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -212,7 +212,6 @@ mod tests {gen_temp_path,index::Index,rand_words,- ranking::centrality_store::CentralityStore,searcher::{LocalSearcher, SearchQuery},webgraph::{Node, WebgraphWriter},webpage::{Html, Webpage},@@ -266,9 +265,6 @@ mod tests {let inbound = InboundSimilarity::build(&graph);- let mut centrality_store = CentralityStore::build(&graph, gen_temp_path());- centrality_store.inbound_similarity = inbound;-let mut index = Index::temporary().expect("Unable to open index");index@@ -335,7 +331,7 @@ mod tests {index.commit().unwrap();let mut searcher = LocalSearcher::new(index);- searcher.set_centrality_store(centrality_store.into());+ searcher.set_inbound_similarity(inbound);let res = searcher.search(&SearchQuery {diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex e7f39fec..809a2cbc 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -16,7 +16,6 @@mod bitvec_similarity;pub mod bm25;-pub mod centrality_store;pub mod inbound_similarity;pub mod initial;pub mod models;diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 029e7962..e9f18f76 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -21,7 +21,7 @@ mod tests {use crate::{gen_temp_path,index::Index,- ranking::centrality_store::CentralityStore,+ ranking::inbound_similarity::InboundSimilarity,searcher::{LocalSearcher, SearchQuery},webgraph::{Node, WebgraphWriter},webpage::{Html, Webpage},@@ -174,7 +174,7 @@ mod tests {index.commit().expect("failed to commit index");let mut searcher = LocalSearcher::new(index);- searcher.set_centrality_store(CentralityStore::build(&graph, gen_temp_path()).into());+ searcher.set_inbound_similarity(InboundSimilarity::build(&graph));let result = searcher.search(&SearchQuery {diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex e7cebb03..481d603f 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -27,7 +27,7 @@ use crate::image_store::Image;use crate::index::Index;use crate::inverted_index::RetrievedWebpage;use crate::query::Query;-use crate::ranking::centrality_store::SearchCentralityStore;+use crate::ranking::inbound_similarity::InboundSimilarity;#[cfg(not(feature = "libtorch"))]use crate::ranking::models::cross_encoder::DummyCrossEncoder;use crate::ranking::models::lambdamart::LambdaMART;@@ -49,7 +49,7 @@ pub struct LocalSearcher {index: Index,spell: Option<Spell>,entity_index: Option<EntityIndex>,- centrality_store: Option<SearchCentralityStore>,+ inbound_similarity: Option<InboundSimilarity>,linear_regression: Option<Arc<LinearRegression>>,lambda_model: Option<Arc<LambdaMART>>,collector_config: CollectorConfig,@@ -76,7 +76,7 @@ impl LocalSearcher {index,spell: None,entity_index: None,- centrality_store: None,+ inbound_similarity: None,linear_regression: None,lambda_model: None,collector_config: CollectorConfig::default(),@@ -91,8 +91,8 @@ impl LocalSearcher {self.entity_index = Some(entity_index);}- pub fn set_centrality_store(&mut self, centrality_store: SearchCentralityStore) {- self.centrality_store = Some(centrality_store);+ pub fn set_inbound_similarity(&mut self, inbound: InboundSimilarity) {+ self.inbound_similarity = Some(inbound);}pub fn set_linear_model(&mut self, model: LinearRegression) {@@ -145,7 +145,7 @@ impl LocalSearcher {ranker.de_rank_similar(de_rank_similar);if query_centrality_coeff > 0.0 {- if let Some(centrality_store) = self.centrality_store.as_ref() {+ if let Some(inbound_sim) = self.inbound_similarity.as_ref() {ranker = ranker.with_max_docs(1_000, self.index.num_segments()).with_num_results(100);@@ -155,10 +155,7 @@ impl LocalSearcher {.top_nodes(query, ctx, ranker.collector(ctx.clone()))?;if !top_host_nodes.is_empty() {- let inbound =- centrality_store- .inbound_similarity- .scorer(&top_host_nodes, &[], false);+ let inbound = inbound_sim.scorer(&top_host_nodes, &[], false);let query_centrality = query_centrality::Scorer::new(inbound);@@ -192,7 +189,7 @@ impl LocalSearcher {let mut aggregator = SignalAggregator::new(Some(&parsed_query));- if let Some(store) = &self.centrality_store {+ if let Some(inbound_sim) = &self.inbound_similarity {let liked_sites: Vec<_> = parsed_query.site_rankings().liked@@ -209,9 +206,7 @@ impl LocalSearcher {.map(|node| node.id()).collect();- let scorer = store- .inbound_similarity- .scorer(&liked_sites, &disliked_sites, false);+ let scorer = inbound_sim.scorer(&liked_sites, &disliked_sites, false);aggregator.set_inbound_similarity(scorer);}diff --git a/core/src/webgraph/centrality/derived_harmonic.rs b/core/src/webgraph/centrality/derived_harmonic.rsnew file mode 100644index 00000000..56bdcb1b--- /dev/null+++ b/core/src/webgraph/centrality/derived_harmonic.rs@@ -0,0 +1,96 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! This is a centrality measure that is based on the harmonic centrality.+//! The idea is to use the harmonic centrality from the domain graph to+//! derive a centrality measure for the page graph.++use anyhow::Result;+use std::{collections::BTreeMap, path::Path};++use crate::{+ kv::{rocksdb_store::RocksDbStore, Kv},+ webgraph::{Node, NodeID, Webgraph},+};+pub struct DerivedCentrality {+ inner: RocksDbStore<NodeID, f64>,+}++impl DerivedCentrality {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ let inner = RocksDbStore::open(path);+ Self { inner }+ }++ pub fn build<P: AsRef<Path>>(+ host_harmonic: &RocksDbStore<NodeID, f64>,+ page_graph: &Webgraph,+ output: P,+ ) -> Result<Self> {+ if output.as_ref().exists() {+ return Err(anyhow::anyhow!("output path already exists"));+ }++ let non_normalized = RocksDbStore::open(output.as_ref().join("non_normalized"));++ let mut norms: BTreeMap<Node, usize> = BTreeMap::new();++ for (node, id) in page_graph.node_ids() {+ let host_node = node.clone().into_host();++ if let Some(harmonic) = host_harmonic.get(&host_node.id()) {+ let mut ingoing: Vec<_> = page_graph+ .ingoing_edges(node.clone())+ .into_iter()+ .map(|e| e.from.into_host())+ .collect();+ ingoing.sort();+ ingoing.dedup();++ let count = ingoing.len();+ let page_score = harmonic * count as f64;++ non_normalized.insert(id, page_score);++ let norm = norms.entry(host_node.clone()).or_insert(0);+ *norm = (*norm).max(count);+ }+ }++ let db = RocksDbStore::open(output.as_ref());+ for (id, score) in non_normalized.iter() {+ let node = page_graph.id2node(&id).unwrap();+ let host_node = node.clone().into_host();+ let norm = norms.get(&host_node).unwrap();+ let normalized = score / (*norm as f64);+ db.insert(id, normalized);+ }+ db.flush();++ drop(non_normalized);+ std::fs::remove_dir_all(output.as_ref().join("non_normalized"))?;++ Ok(Self { inner: db })+ }++ pub fn get(&self, node: &NodeID) -> Option<f64> {+ self.inner.get(node)+ }++ pub fn iter(&self) -> impl Iterator<Item = (NodeID, f64)> + '_ {+ self.inner.iter()+ }+}diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex f88aa9f5..bbedfe07 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -16,12 +16,9 @@use std::{collections::{BTreeMap, BTreeSet},- fs,- path::{Path, PathBuf},- sync::{atomic::AtomicBool, Mutex},+ sync::atomic::AtomicBool,};-use anyhow::Result;use std::sync::atomic::Ordering;use bitvec::vec::BitVec;@@ -85,106 +82,10 @@ impl JankyBloomFilter {}}-struct Db<K, V>-where- K: serde::Serialize + serde::de::DeserializeOwned,- V: serde::Serialize + serde::de::DeserializeOwned,-{- db: rocksdb::DB,- path: PathBuf,- _cache: rocksdb::Cache,- _phantom: std::marker::PhantomData<(K, V)>,-}--impl<K, V> Db<K, V>-where- K: serde::Serialize + serde::de::DeserializeOwned,- V: serde::Serialize + serde::de::DeserializeOwned,-{- fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- if !path.as_ref().exists() {- fs::create_dir_all(path.as_ref())?;- }-- let mut options = rocksdb::Options::default();- options.create_if_missing(true);-- options.set_max_background_jobs(8);- options.increase_parallelism(8);- options.set_max_subcompactions(8);-- let cache = rocksdb::Cache::new_lru_cache(20 * 1024 * 1024 * 1024); // 20 gb-- // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)- options.set_level_compaction_dynamic_level_bytes(true);- options.set_bytes_per_sync(1048576);- let mut block_options = rocksdb::BlockBasedOptions::default();- block_options.set_block_size(16 * 1024);- block_options.set_format_version(5);- block_options.set_cache_index_and_filter_blocks(true);- block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);-- block_options.set_ribbon_filter(10.0);- block_options.set_block_cache(&cache);-- options.set_block_based_table_factory(&block_options);- options.set_compression_type(rocksdb::DBCompressionType::None);-- options.optimize_for_point_lookup(512); // 512 MB-- let db = rocksdb::DB::open(&options, &path)?;-- Ok(Self {- db,- path: path.as_ref().to_path_buf(),- _cache: cache,- _phantom: std::marker::PhantomData,- })- }-- fn get(&self, key: &K) -> Option<V> {- let key_bytes = bincode::serialize(key).expect("failed to serialize key");-- self.db- .get(key_bytes)- .expect("failed to retrieve key")- .map(|bytes| bincode::deserialize(&bytes).expect("failed to deserialize stored value"))- }-- fn insert(&self, key: &K, value: &V) {- let key = bincode::serialize(key).expect("failed to serialize key");- let value = bincode::serialize(value).expect("failed to serialize value");-- self.db.put(key, value).expect("failed to insert key");- }-- fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {- self.db.iterator(rocksdb::IteratorMode::Start).map(|s| {- let (key, value) = s.expect("failed to iterate over db");- (- bincode::deserialize(&key).expect("failed to deserialize key"),- bincode::deserialize(&value).expect("failed to deserialize value"),- )- })- }-- fn path(&self) -> &Path {- &self.path- }-}--fn calculate_centrality<P: AsRef<Path>>(- graph: &Webgraph,- calculation_path: P,-) -> BTreeMap<NodeID, f64> {+fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {let mut num_nodes = 0;- let mut counters: Db<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = Db::open(- calculation_path- .as_ref()- .join(uuid::Uuid::new_v4().to_string()),- )- .unwrap();+ let mut counters: BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = BTreeMap::new();let mut centralities: BTreeMap<NodeID, KahanSum> = BTreeMap::new();@@ -192,7 +93,7 @@ fn calculate_centrality<P: AsRef<Path>>(let mut counter = HyperLogLog::default();counter.add(node.bit_64());- counters.insert(&node, &counter);+ counters.insert(node, counter);centralities.insert(node, KahanSum::default());num_nodes += 1;@@ -219,19 +120,10 @@ fn calculate_centrality<P: AsRef<Path>>(break;}- let new_counters: Db<_, _> = Db::open(- calculation_path- .as_ref()- .join(uuid::Uuid::new_v4().to_string()),- )- .unwrap();-- for (node, counter) in counters.iter() {- new_counters.insert(&node, &counter);- }+ let mut new_counters = counters.clone();has_changes.store(false, Ordering::Relaxed);- let new_changed_nodes = Mutex::new(JankyBloomFilter::new(num_nodes as u64, 0.05));+ let mut new_changed_nodes = JankyBloomFilter::new(num_nodes as u64, 0.05);if !exact_changed_nodes.is_empty()&& exact_changed_nodes.len() as u64 <= exact_counting_threshold@@ -240,8 +132,8 @@ fn calculate_centrality<P: AsRef<Path>>(exact_changed_nodes.iter().for_each(|changed_node| {for edge in graph.raw_outgoing_edges(changed_node) {- if let (Some(mut counter_to), Some(counter_from)) =- (new_counters.get(&edge.to), counters.get(&edge.from))+ if let (Some(counter_to), Some(counter_from)) =+ (new_counters.get_mut(&edge.to), counters.get(&edge.from)){if counter_to.registers()@@ -249,9 +141,8 @@ fn calculate_centrality<P: AsRef<Path>>(.zip(counter_from.registers().iter()).any(|(to, from)| *from > *to){- counter_to.merge(&counter_from);- new_counters.insert(&edge.to, &counter_to);- new_changed_nodes.lock().unwrap().insert(edge.to.bit_64());+ counter_to.merge(counter_from);+ new_changed_nodes.insert(edge.to.bit_64());new_exact_changed_nodes.insert(edge.to);@@ -266,8 +157,8 @@ fn calculate_centrality<P: AsRef<Path>>(exact_changed_nodes = BTreeSet::default();graph.edges().for_each(|edge| {if changed_nodes.contains(&edge.from.bit_64()) {- if let (Some(mut counter_to), Some(counter_from)) =- (new_counters.get(&edge.to), counters.get(&edge.from))+ if let (Some(counter_to), Some(counter_from)) =+ (new_counters.get_mut(&edge.to), counters.get(&edge.from)){if counter_to.registers()@@ -275,9 +166,8 @@ fn calculate_centrality<P: AsRef<Path>>(.zip(counter_from.registers().iter()).any(|(to, from)| *from > *to){- counter_to.merge(&counter_from);- new_counters.insert(&edge.to, &counter_to);- new_changed_nodes.lock().unwrap().insert(edge.to.bit_64());+ counter_to.merge(counter_from);+ new_changed_nodes.insert(edge.to.bit_64());if exact_counting {exact_changed_nodes.insert(edge.to);@@ -306,9 +196,7 @@ fn calculate_centrality<P: AsRef<Path>>(});counters = new_counters;- let old_path = counters.path().to_path_buf();- changed_nodes = new_changed_nodes.into_inner().unwrap();- std::fs::remove_dir_all(old_path).unwrap();+ changed_nodes = new_changed_nodes;t += 1;if changed_nodes.estimate_card() <= exact_counting_threshold {@@ -331,8 +219,8 @@ fn calculate_centrality<P: AsRef<Path>>(pub struct HarmonicCentrality(BTreeMap<NodeID, f64>);impl HarmonicCentrality {- pub fn calculate<P: AsRef<Path>>(graph: &Webgraph, path: P) -> Self {- Self(calculate_centrality(graph, path))+ pub fn calculate(graph: &Webgraph) -> Self {+ Self(calculate_centrality(graph))}pub fn get(&self, node: &NodeID) -> Option<f64> {@@ -464,7 +352,7 @@ mod tests {let graph = writer.finalize();- let centrality = HarmonicCentrality::calculate(&graph, crate::gen_temp_path());+ let centrality = HarmonicCentrality::calculate(&graph);assert!(centrality.get(&Node::from("B.com").id()).unwrap()@@ -475,7 +363,7 @@ mod tests {#[test]fn harmonic_centrality() {let graph = test_graph();- let centrality = HarmonicCentrality::calculate(&graph, crate::gen_temp_path());+ let centrality = HarmonicCentrality::calculate(&graph);assert!(centrality.get(&Node::from("C").id()).unwrap()@@ -491,7 +379,7 @@ mod tests {#[test]fn additional_edges_ignored() {let graph = test_graph();- let centrality = HarmonicCentrality::calculate(&graph, crate::gen_temp_path());+ let centrality = HarmonicCentrality::calculate(&graph);let mut other = WebgraphWriter::new(crate::gen_temp_path(),@@ -520,7 +408,7 @@ mod tests {let graph = other.finalize();- let centrality_extra = HarmonicCentrality::calculate(&graph, crate::gen_temp_path());+ let centrality_extra = HarmonicCentrality::calculate(&graph);assert_eq!(centrality.0, centrality_extra.0);}diff --git a/core/src/webgraph/centrality/mod.rs b/core/src/webgraph/centrality/mod.rsindex df9aa96a..9fe59ff1 100644--- a/core/src/webgraph/centrality/mod.rs+++ b/core/src/webgraph/centrality/mod.rs@@ -15,4 +15,5 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.pub mod betweenness;+pub mod derived_harmonic;pub mod harmonic;