Skip to content

web search done right

Derived centrality. New centrality metric that is derived from the host harmonic centrality. This can hopefully be calculated on the enormous page webgraph.

Mikkel Denker4c49e2deb8c2
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index e076dd66..2700a2e2 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -3,15 +3,9 @@ limit_warc_files = 20
# minimum_clean_words = 40
[warc_source]
-# type = "Local"
-# folder = "./data"
-# names = ["sample.warc.gz"]
-# type = "HTTP"
-# base_url = "https://data.commoncrawl.org/"
-# warc_paths_file = "data/warc.paths"
-access_key = "crawl_nOf7tywztpjg2"
-bucket = "crawl"
-endpoint = "http://s3.trystract.com"
-folder = "400m_try11"
-secret_key = "4Kzo0CPdQYnlFHRgwKpUA7mACzSLsA6OpG6qDFl4"
-type = "S3"
+base_url = "https://data.commoncrawl.org/"
+folder = "./data"
+names = ["sample.warc.gz"]
+type = "Local"
+type = "HTTP"
+warc_paths_file = "data/warc.paths"
diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rs
index 64e46668..80f0d4d7 100644
--- a/core/benches/search-preindexed.rs
+++ b/core/benches/search-preindexed.rs
@@ -1,8 +1,10 @@
+use std::path::Path;
+
use criterion::{criterion_group, criterion_main, Criterion};
use optics::SiteRankings;
use stract::{
index::Index,
- ranking::centrality_store::CentralityStore,
+ ranking::inbound_similarity::InboundSimilarity,
searcher::{LocalSearcher, SearchQuery},
};
@@ -39,7 +41,9 @@ macro_rules! bench {
pub fn criterion_benchmark(c: &mut Criterion) {
let index = Index::open(INDEX_PATH).unwrap();
let mut searcher = LocalSearcher::new(index);
- searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH).into());
+ searcher.set_inbound_similarity(
+ InboundSimilarity::open(Path::new(CENTRALITY_PATH).join("inbound_similarity")).unwrap(),
+ );
for _ in 0..1000 {
bench!("the", searcher, c);
diff --git a/core/examples/print_similar_sites.rs b/core/examples/print_similar_sites.rs
index dc05faeb..fac809c6 100644
--- a/core/examples/print_similar_sites.rs
+++ b/core/examples/print_similar_sites.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use stract::{
- ranking::centrality_store::CentralityStore,
+ ranking::inbound_similarity::InboundSimilarity,
similar_sites::{ScoredNode, SimilarSitesFinder},
webgraph::WebgraphBuilder,
};
@@ -40,11 +40,11 @@ fn print_top_nodes(liked_sites: &[&str], top_n: usize, similarity_finder: &Simil
pub fn main() {
const TOP_N: usize = 50;
let graph = WebgraphBuilder::new("data/webgraph").open();
- let store = CentralityStore::open("data/centrality");
+ let inbound_similarity = InboundSimilarity::open("data/centrality/inbound_similarity").unwrap();
let similarity_finder = SimilarSitesFinder::new(
graph.into(),
- store.inbound_similarity,
+ inbound_similarity,
stract::config::defaults::WebgraphServer::max_similar_sites(),
);
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 10f1b162..8b25b03e 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -243,7 +243,7 @@ pub struct SearchServerConfig {
pub shard_id: ShardId,
pub index_path: String,
pub entity_index_path: Option<String>,
- pub host_centrality_store_path: Option<String>,
+ pub centrality_store_path: Option<String>,
pub linear_model_path: Option<String>,
pub lambda_model_path: Option<String>,
pub host: SocketAddr,
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index 530fc566..9fab3b48 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -14,29 +14,138 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::path::Path;
+use anyhow::Result;
+use std::{cmp::Reverse, collections::BinaryHeap, fs::File, path::Path};
use crate::{
- ranking::centrality_store::CentralityStore,
- webgraph::{Compression, WebgraphBuilder},
+ kv::{rocksdb_store::RocksDbStore, Kv},
+ ranking::inbound_similarity::InboundSimilarity,
+ webgraph::{
+ centrality::{derived_harmonic::DerivedCentrality, harmonic::HarmonicCentrality},
+ Node, WebgraphBuilder,
+ },
};
-pub struct Centrality {}
+fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {
+ let csv_file = File::options()
+ .write(true)
+ .create(true)
+ .truncate(true)
+ .open(output)
+ .unwrap();
+
+ let mut data = data;
+ data.sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));
+ let mut wtr = csv::Writer::from_writer(csv_file);
+ for (node, centrality) in data {
+ wtr.write_record(&[node.name, centrality.to_string()])
+ .unwrap();
+ }
+ wtr.flush().unwrap();
+}
+
+struct SortableFloat(f64);
+
+impl PartialEq for SortableFloat {
+ fn eq(&self, other: &Self) -> bool {
+ self.0 == other.0
+ }
+}
+
+impl Eq for SortableFloat {}
+
+impl PartialOrd for SortableFloat {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl Ord for SortableFloat {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.0.total_cmp(&other.0)
+ }
+}
+
+pub struct Centrality;
impl Centrality {
- pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
+ pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) {
tracing::info!("Building harmonic centrality");
- let graph = WebgraphBuilder::new(webgraph_path)
- .compression(Compression::Lz4)
- .open();
- CentralityStore::build_harmonic(&graph, output_path);
+ let graph = WebgraphBuilder::new(webgraph_path).open();
+ let harmonic_centrality = HarmonicCentrality::calculate(&graph);
+ let store = RocksDbStore::open(base_output.as_ref().join("harmonic"));
+
+ for (node_id, centrality) in harmonic_centrality.iter() {
+ store.insert(*node_id, centrality);
+ }
+ store.flush();
+
+ let mut top_harmonics = BinaryHeap::new();
+
+ for (node_id, centrality) in harmonic_centrality.iter() {
+ if top_harmonics.len() < 1_000_000 {
+ top_harmonics.push((Reverse(SortableFloat(centrality)), *node_id));
+ } else {
+ let mut min = top_harmonics.peek_mut().unwrap();
+ if min.0 .0 < SortableFloat(centrality) {
+ *min = (Reverse(SortableFloat(centrality)), *node_id);
+ }
+ }
+ }
+
+ let harmonics: Vec<_> = top_harmonics
+ .into_iter()
+ .map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))
+ .collect();
+
+ store_csv(harmonics, base_output.as_ref().join("harmonic.csv"));
}
- pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, output_path: P) {
+ pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, base_output: P) {
tracing::info!("Building inbound similarity");
- let graph = WebgraphBuilder::new(webgraph_path)
- .compression(Compression::Lz4)
- .open();
- CentralityStore::build_similarity(&graph, output_path);
+ let graph = WebgraphBuilder::new(webgraph_path).open();
+
+ let sim = InboundSimilarity::build(&graph);
+
+ sim.save(base_output.as_ref().join("inbound_similarity"))
+ .unwrap();
+ }
+
+ pub fn build_derived_harmonic<P: AsRef<Path>>(
+ webgraph_path: P,
+ host_centrality_path: P,
+ base_output: P,
+ ) -> Result<()> {
+ tracing::info!("Building derived harmonic centrality");
+ let graph = WebgraphBuilder::new(webgraph_path).open();
+ let host_centrality = RocksDbStore::open(host_centrality_path.as_ref().join("harmonic"));
+
+ let derived = DerivedCentrality::build(
+ &host_centrality,
+ &graph,
+ base_output.as_ref().join("derived_harmonic"),
+ )?;
+
+ let mut top_nodes = BinaryHeap::new();
+
+ for (node_id, centrality) in derived.iter() {
+ if top_nodes.len() < 1_000_000 {
+ top_nodes.push((Reverse(SortableFloat(centrality)), node_id));
+ } else {
+ let mut min = top_nodes.peek_mut().unwrap();
+ if min.0 .0 < SortableFloat(centrality) {
+ *min = (Reverse(SortableFloat(centrality)), node_id);
+ }
+ }
+ }
+
+ let derived: Vec<_> = top_nodes
+ .into_iter()
+ .map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))
+ .collect();
+
+ store_csv(derived, base_output.as_ref().join("derived_centrality.csv"));
+
+ Ok(())
}
}
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 96947c29..2714ce28 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -101,8 +101,17 @@ fn calculate_centrality() {
let webgraph_path = Path::new(DATA_PATH).join("webgraph_host");
let out_path = Path::new(DATA_PATH).join("centrality");
- Centrality::build_harmonic(&webgraph_path, &out_path);
- Centrality::build_similarity(&webgraph_path, &out_path);
+ if !out_path.exists() {
+ Centrality::build_harmonic(&webgraph_path, &out_path);
+ Centrality::build_similarity(&webgraph_path, &out_path);
+ }
+
+ let webgraph_page = Path::new(DATA_PATH).join("webgraph_page");
+ let out_path_page = Path::new(DATA_PATH).join("centrality_page");
+
+ if !out_path_page.exists() {
+ Centrality::build_derived_harmonic(webgraph_page, out_path, out_path_page).unwrap();
+ }
}
fn create_inverted_index() -> Result<()> {
@@ -132,10 +141,11 @@ fn create_inverted_index() -> Result<()> {
let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");
let centrality_path = Path::new(DATA_PATH).join("centrality");
+ let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");
let worker = indexer::IndexingWorker::new(
centrality_path.to_str().unwrap().to_string(),
- None,
+ Some(page_centrality_path.to_str().unwrap().to_string()),
Some(webgraph_path.to_str().unwrap().to_string()),
Some(
Path::new(DATA_PATH)
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 2243177c..897a5e39 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -26,10 +26,11 @@ use tracing::{debug, info, trace, warn};
use crate::config;
use crate::entrypoint::download_all_warc_files;
use crate::index::{FrozenIndex, Index};
+use crate::kv::rocksdb_store::RocksDbStore;
+use crate::kv::Kv;
use crate::mapreduce::{Map, Reduce, Worker};
-use crate::ranking::centrality_store::IndexerCentralityStore;
use crate::ranking::SignalAggregator;
-use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
+use crate::webgraph::{Node, NodeID, Webgraph, WebgraphBuilder};
use crate::webpage::{safety_classifier, Html, Webpage};
use crate::{human_website_annotations, Result};
@@ -72,8 +73,8 @@ pub struct Job {
}
pub struct IndexingWorker {
- host_centrality_store: IndexerCentralityStore,
- page_centrality_store: Option<IndexerCentralityStore>,
+ host_centrality_store: RocksDbStore<NodeID, f64>,
+ page_centrality_store: Option<RocksDbStore<NodeID, f64>>,
webgraph: Option<Webgraph>,
topics: Option<human_website_annotations::Mapper>,
safety_classifier: Option<safety_classifier::Model>,
@@ -88,8 +89,11 @@ impl IndexingWorker {
safety_classifier_path: Option<String>,
) -> Self {
Self {
- host_centrality_store: IndexerCentralityStore::open(host_centrality_store_path),
- page_centrality_store: page_centrality_store_path.map(IndexerCentralityStore::open),
+ host_centrality_store: RocksDbStore::open(
+ Path::new(&host_centrality_store_path).join("harmonic"),
+ ),
+ page_centrality_store: page_centrality_store_path
+ .map(|p| RocksDbStore::open(Path::new(&p).join("derived_harmonic"))),
webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).single_threaded().open()),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
safety_classifier: safety_classifier_path
@@ -148,7 +152,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let mut host_centrality = worker
.host_centrality_store
- .harmonic
.get(&host_node_id)
.unwrap_or_default();
@@ -194,7 +197,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
if let Some(store) = worker.page_centrality_store.as_ref() {
let node_id = node.id();
- page_centrality = store.harmonic.get(&node_id).unwrap_or_default();
+ page_centrality = store.get(&node_id).unwrap_or_default();
}
if host_centrality > 0.0 {
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index a3efc046..aa388cdb 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::collections::HashMap;
+use std::{collections::HashMap, path::Path};
use serde::{Deserialize, Serialize};
use tracing::info;
@@ -32,7 +32,7 @@ use crate::{
index::Index,
inverted_index::{self, RetrievedWebpage},
ranking::{
- centrality_store::SearchCentralityStore,
+ inbound_similarity::InboundSimilarity,
models::{lambdamart::LambdaMART, linear::LinearRegression},
},
searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},
@@ -63,8 +63,8 @@ impl SearchService {
.entity_index_path
.map(|path| EntityIndex::open(path).unwrap());
let centrality_store = config
- .host_centrality_store_path
- .map(SearchCentralityStore::open);
+ .centrality_store_path
+ .map(|p| InboundSimilarity::open(Path::new(&p).join("inbound_similarity")).unwrap());
let search_index = Index::open(config.index_path)?;
let mut local_searcher = LocalSearcher::new(search_index);
@@ -74,7 +74,7 @@ impl SearchService {
}
if let Some(centrality_store) = centrality_store {
- local_searcher.set_centrality_store(centrality_store);
+ local_searcher.set_inbound_similarity(centrality_store);
}
if let Some(model_path) = config.linear_model_path {
diff --git a/core/src/kv/rocksdb_store.rs b/core/src/kv/rocksdb_store.rs
index 90f26af4..a825799c 100644
--- a/core/src/kv/rocksdb_store.rs
+++ b/core/src/kv/rocksdb_store.rs
@@ -24,12 +24,21 @@ use serde::{de::DeserializeOwned, Serialize};
use crate::kv::Kv;
-pub struct RocksDbStore {
+pub struct RocksDbStore<K, V>
+where
+ K: Serialize + DeserializeOwned,
+ V: Serialize + DeserializeOwned,
+{
db: DB,
_cache: rocksdb::Cache,
+ _phantom: PhantomData<(K, V)>,
}
-impl RocksDbStore {
+impl<K, V> RocksDbStore<K, V>
+where
+ K: Serialize + DeserializeOwned,
+ V: Serialize + DeserializeOwned,
+{
pub fn open<P>(path: P) -> Self
where
P: AsRef<std::path::Path>,
@@ -66,21 +75,30 @@ impl RocksDbStore {
let db = DB::open(&options, path).expect("unable to open rocks db");
- Self { db, _cache: cache }
+ Self {
+ db,
+ _cache: cache,
+ _phantom: PhantomData,
+ }
}
}
-impl<K, V> Kv<K, V> for RocksDbStore
+impl<K, V> Kv<K, V> for RocksDbStore<K, V>
where
- K: Serialize + DeserializeOwned + 'static,
- V: Serialize + DeserializeOwned + 'static,
+ K: Serialize + DeserializeOwned + 'static + Send + Sync,
+ V: Serialize + DeserializeOwned + 'static + Send + Sync,
{
fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {
self.db.get(key).expect("failed to retrieve key")
}
fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {
- self.db.put(key, value).expect("failed to insert value");
+ let mut opt = rocksdb::WriteOptions::default();
+ opt.disable_wal(true);
+
+ self.db
+ .put_opt(key, value, &opt)
+ .expect("failed to insert value");
}
fn flush(&self) {
diff --git a/core/src/main.rs b/core/src/main.rs
index 223ed936..6259d45c 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -55,9 +55,7 @@ enum Commands {
/// Calculate centrality metrics that estimates a websites importance. These metrics are used to rank search results.
Centrality {
#[clap(subcommand)]
- mode: CentralityType,
- webgraph_path: String,
- output_path: String,
+ mode: CentralityMode,
},
/// Parse the DMOZ dataset. DMOZ contains a list of websites and their categories.
@@ -140,15 +138,18 @@ enum SafetyClassifierOptions {
}
#[derive(Subcommand)]
-enum CentralityType {
- /// Calculate all the supported centrality metrics.
- All,
- /// Calculate harmonic centrality. Harmonic centrality is the sum of the inverse distance from all
- /// other nodes to a given node. It is a measure of how central a node (webpage) is in a graph.
- Harmonic,
- /// Create an index that can be used to quickly calculate the inbound link similarity between two nodes.
- /// This is used to calculate the similarity between two webpages.
- Similarity,
+enum CentralityMode {
+ /// Calculate metrics for the host webgraph.
+ Host {
+ webgraph_path: String,
+ output_path: String,
+ },
+ /// Calculate metrics for the page webgraph.
+ Page {
+ webgraph_path: String,
+ host_centrality_path: String,
+ output_path: String,
+ },
}
#[derive(Subcommand)]
@@ -217,22 +218,24 @@ fn main() -> Result<()> {
entrypoint::Indexer::merge(indexes.into_iter().map(IndexPointer::from).collect())?
}
},
- Commands::Centrality {
- mode,
- webgraph_path,
- output_path,
- } => {
+ Commands::Centrality { mode } => {
match mode {
- CentralityType::Harmonic => {
- entrypoint::Centrality::build_harmonic(webgraph_path, output_path)
- }
- CentralityType::Similarity => {
- entrypoint::Centrality::build_similarity(webgraph_path, output_path)
- }
- CentralityType::All => {
+ CentralityMode::Host {
+ webgraph_path,
+ output_path,
+ } => {
entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);
entrypoint::Centrality::build_similarity(&webgraph_path, &output_path);
}
+ CentralityMode::Page {
+ webgraph_path,
+ host_centrality_path,
+ output_path,
+ } => entrypoint::Centrality::build_derived_harmonic(
+ webgraph_path,
+ host_centrality_path,
+ output_path,
+ )?,
}
tracing::info!("Done");
}
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 948b8906..4433ebae 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -238,7 +238,7 @@ mod tests {
use crate::{
gen_temp_path,
index::Index,
- ranking::centrality_store::CentralityStore,
+ ranking::inbound_similarity::InboundSimilarity,
searcher::{LocalSearcher, SearchQuery},
webgraph::{Node, WebgraphWriter},
webpage::{Html, Webpage},
@@ -658,8 +658,6 @@ mod tests {
let graph = writer.finalize();
- let centrality_store = CentralityStore::build(&graph, gen_temp_path());
-
index
.insert(Webpage {
html: Html::parse(
@@ -755,7 +753,7 @@ mod tests {
index.commit().expect("failed to commit index");
let mut searcher = LocalSearcher::from(index);
- searcher.set_centrality_store(centrality_store.into());
+ searcher.set_inbound_similarity(InboundSimilarity::build(&graph));
let res = searcher
.search(&SearchQuery {
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
deleted file mode 100644
index 11b96a1e..00000000
--- a/core/src/ranking/centrality_store.rs
+++ /dev/null
@@ -1,159 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{fs::File, path::Path};
-
-use tracing::debug;
-use tracing::log::trace;
-
-use crate::{
- kv::{rocksdb_store::RocksDbStore, Kv},
- webgraph::{centrality::harmonic::HarmonicCentrality, NodeID, Webgraph},
-};
-
-use super::inbound_similarity::InboundSimilarity;
-
-pub type HarmonicCentralityStore = Box<dyn Kv<NodeID, f64> + Send + Sync>;
-
-pub struct IndexerCentralityStore {
- pub harmonic: HarmonicCentralityStore,
-}
-
-impl IndexerCentralityStore {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- Self {
- harmonic: Box::new(RocksDbStore::open(path.as_ref().join("harmonic"))),
- }
- }
-}
-
-impl From<CentralityStore> for IndexerCentralityStore {
- fn from(store: CentralityStore) -> Self {
- Self {
- harmonic: store.harmonic,
- }
- }
-}
-
-pub struct SearchCentralityStore {
- pub inbound_similarity: InboundSimilarity,
-}
-
-impl SearchCentralityStore {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- Self {
- inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))
- .unwrap(),
- }
- }
-}
-
-impl From<CentralityStore> for SearchCentralityStore {
- fn from(store: CentralityStore) -> Self {
- Self {
- inbound_similarity: store.inbound_similarity,
- }
- }
-}
-
-pub struct CentralityStore {
- pub harmonic: HarmonicCentralityStore,
- pub inbound_similarity: InboundSimilarity,
- pub base_path: String,
-}
-
-impl CentralityStore {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- Self {
- harmonic: Box::new(RocksDbStore::open(path.as_ref().join("harmonic"))),
- inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))
- .ok()
- .unwrap_or_default(),
- base_path: path.as_ref().to_str().unwrap().to_string(),
- }
- }
-
- fn store_harmonic<P: AsRef<Path>>(
- output_path: P,
- store: &CentralityStore,
- harmonic_centrality: HarmonicCentrality,
- graph: &Webgraph,
- ) {
- let csv_file = File::options()
- .write(true)
- .create(true)
- .truncate(true)
- .open(output_path.as_ref().join("harmonic.csv"))
- .unwrap();
-
- for (node_id, centrality) in harmonic_centrality.iter() {
- store.harmonic.insert(*node_id, centrality);
- }
- store.harmonic.flush();
-
- let mut harmonic: Vec<_> = harmonic_centrality
- .iter()
- .map(|(node, centrality)| (*node, centrality))
- .take(1_000_000)
- .collect();
-
- harmonic.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
- let mut wtr = csv::Writer::from_writer(csv_file);
- for (node_id, centrality) in harmonic {
- let node = graph.id2node(&node_id).unwrap();
-
- wtr.write_record(&[node.name, centrality.to_string()])
- .unwrap();
- }
- wtr.flush().unwrap();
- }
-
- pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
- Self::build_harmonic(graph, &output_path);
- Self::build_similarity(graph, &output_path)
- }
-
- pub fn build_harmonic<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
- let store = CentralityStore::open(output_path.as_ref());
-
- let harmonic_centrality =
- HarmonicCentrality::calculate(graph, output_path.as_ref().join("calculations"));
- Self::store_harmonic(&output_path, &store, harmonic_centrality, graph);
-
- store.flush();
- store
- }
-
- pub fn build_similarity<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
- let mut store = CentralityStore::open(output_path.as_ref());
-
- debug!("Begin inbound similarity index construction");
- store.inbound_similarity = InboundSimilarity::build(graph);
-
- store.flush();
- store
- }
-
- pub fn flush(&self) {
- trace!("flushing");
- self.harmonic.flush();
-
- trace!("saving inbound similarity");
- self.inbound_similarity
- .save(Path::new(&self.base_path).join("inbound_similarity"))
- .unwrap();
- }
-}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 7f09233d..6d070a49 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -212,7 +212,6 @@ mod tests {
gen_temp_path,
index::Index,
rand_words,
- ranking::centrality_store::CentralityStore,
searcher::{LocalSearcher, SearchQuery},
webgraph::{Node, WebgraphWriter},
webpage::{Html, Webpage},
@@ -266,9 +265,6 @@ mod tests {
let inbound = InboundSimilarity::build(&graph);
- let mut centrality_store = CentralityStore::build(&graph, gen_temp_path());
- centrality_store.inbound_similarity = inbound;
-
let mut index = Index::temporary().expect("Unable to open index");
index
@@ -335,7 +331,7 @@ mod tests {
index.commit().unwrap();
let mut searcher = LocalSearcher::new(index);
- searcher.set_centrality_store(centrality_store.into());
+ searcher.set_inbound_similarity(inbound);
let res = searcher
.search(&SearchQuery {
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index e7f39fec..809a2cbc 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -16,7 +16,6 @@
mod bitvec_similarity;
pub mod bm25;
-pub mod centrality_store;
pub mod inbound_similarity;
pub mod initial;
pub mod models;
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 029e7962..e9f18f76 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -21,7 +21,7 @@ mod tests {
use crate::{
gen_temp_path,
index::Index,
- ranking::centrality_store::CentralityStore,
+ ranking::inbound_similarity::InboundSimilarity,
searcher::{LocalSearcher, SearchQuery},
webgraph::{Node, WebgraphWriter},
webpage::{Html, Webpage},
@@ -174,7 +174,7 @@ mod tests {
index.commit().expect("failed to commit index");
let mut searcher = LocalSearcher::new(index);
- searcher.set_centrality_store(CentralityStore::build(&graph, gen_temp_path()).into());
+ searcher.set_inbound_similarity(InboundSimilarity::build(&graph));
let result = searcher
.search(&SearchQuery {
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index e7cebb03..481d603f 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -27,7 +27,7 @@ use crate::image_store::Image;
use crate::index::Index;
use crate::inverted_index::RetrievedWebpage;
use crate::query::Query;
-use crate::ranking::centrality_store::SearchCentralityStore;
+use crate::ranking::inbound_similarity::InboundSimilarity;
#[cfg(not(feature = "libtorch"))]
use crate::ranking::models::cross_encoder::DummyCrossEncoder;
use crate::ranking::models::lambdamart::LambdaMART;
@@ -49,7 +49,7 @@ pub struct LocalSearcher {
index: Index,
spell: Option<Spell>,
entity_index: Option<EntityIndex>,
- centrality_store: Option<SearchCentralityStore>,
+ inbound_similarity: Option<InboundSimilarity>,
linear_regression: Option<Arc<LinearRegression>>,
lambda_model: Option<Arc<LambdaMART>>,
collector_config: CollectorConfig,
@@ -76,7 +76,7 @@ impl LocalSearcher {
index,
spell: None,
entity_index: None,
- centrality_store: None,
+ inbound_similarity: None,
linear_regression: None,
lambda_model: None,
collector_config: CollectorConfig::default(),
@@ -91,8 +91,8 @@ impl LocalSearcher {
self.entity_index = Some(entity_index);
}
- pub fn set_centrality_store(&mut self, centrality_store: SearchCentralityStore) {
- self.centrality_store = Some(centrality_store);
+ pub fn set_inbound_similarity(&mut self, inbound: InboundSimilarity) {
+ self.inbound_similarity = Some(inbound);
}
pub fn set_linear_model(&mut self, model: LinearRegression) {
@@ -145,7 +145,7 @@ impl LocalSearcher {
ranker.de_rank_similar(de_rank_similar);
if query_centrality_coeff > 0.0 {
- if let Some(centrality_store) = self.centrality_store.as_ref() {
+ if let Some(inbound_sim) = self.inbound_similarity.as_ref() {
ranker = ranker
.with_max_docs(1_000, self.index.num_segments())
.with_num_results(100);
@@ -155,10 +155,7 @@ impl LocalSearcher {
.top_nodes(query, ctx, ranker.collector(ctx.clone()))?;
if !top_host_nodes.is_empty() {
- let inbound =
- centrality_store
- .inbound_similarity
- .scorer(&top_host_nodes, &[], false);
+ let inbound = inbound_sim.scorer(&top_host_nodes, &[], false);
let query_centrality = query_centrality::Scorer::new(inbound);
@@ -192,7 +189,7 @@ impl LocalSearcher {
let mut aggregator = SignalAggregator::new(Some(&parsed_query));
- if let Some(store) = &self.centrality_store {
+ if let Some(inbound_sim) = &self.inbound_similarity {
let liked_sites: Vec<_> = parsed_query
.site_rankings()
.liked
@@ -209,9 +206,7 @@ impl LocalSearcher {
.map(|node| node.id())
.collect();
- let scorer = store
- .inbound_similarity
- .scorer(&liked_sites, &disliked_sites, false);
+ let scorer = inbound_sim.scorer(&liked_sites, &disliked_sites, false);
aggregator.set_inbound_similarity(scorer);
}
diff --git a/core/src/webgraph/centrality/derived_harmonic.rs b/core/src/webgraph/centrality/derived_harmonic.rs
new file mode 100644
index 00000000..56bdcb1b
--- /dev/null
+++ b/core/src/webgraph/centrality/derived_harmonic.rs
@@ -0,0 +1,96 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! This is a centrality measure that is based on the harmonic centrality.
+//! The idea is to use the harmonic centrality from the domain graph to
+//! derive a centrality measure for the page graph.
+
+use anyhow::Result;
+use std::{collections::BTreeMap, path::Path};
+
+use crate::{
+ kv::{rocksdb_store::RocksDbStore, Kv},
+ webgraph::{Node, NodeID, Webgraph},
+};
+pub struct DerivedCentrality {
+ inner: RocksDbStore<NodeID, f64>,
+}
+
+impl DerivedCentrality {
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ let inner = RocksDbStore::open(path);
+ Self { inner }
+ }
+
+ pub fn build<P: AsRef<Path>>(
+ host_harmonic: &RocksDbStore<NodeID, f64>,
+ page_graph: &Webgraph,
+ output: P,
+ ) -> Result<Self> {
+ if output.as_ref().exists() {
+ return Err(anyhow::anyhow!("output path already exists"));
+ }
+
+ let non_normalized = RocksDbStore::open(output.as_ref().join("non_normalized"));
+
+ let mut norms: BTreeMap<Node, usize> = BTreeMap::new();
+
+ for (node, id) in page_graph.node_ids() {
+ let host_node = node.clone().into_host();
+
+ if let Some(harmonic) = host_harmonic.get(&host_node.id()) {
+ let mut ingoing: Vec<_> = page_graph
+ .ingoing_edges(node.clone())
+ .into_iter()
+ .map(|e| e.from.into_host())
+ .collect();
+ ingoing.sort();
+ ingoing.dedup();
+
+ let count = ingoing.len();
+ let page_score = harmonic * count as f64;
+
+ non_normalized.insert(id, page_score);
+
+ let norm = norms.entry(host_node.clone()).or_insert(0);
+ *norm = (*norm).max(count);
+ }
+ }
+
+ let db = RocksDbStore::open(output.as_ref());
+ for (id, score) in non_normalized.iter() {
+ let node = page_graph.id2node(&id).unwrap();
+ let host_node = node.clone().into_host();
+ let norm = norms.get(&host_node).unwrap();
+ let normalized = score / (*norm as f64);
+ db.insert(id, normalized);
+ }
+ db.flush();
+
+ drop(non_normalized);
+ std::fs::remove_dir_all(output.as_ref().join("non_normalized"))?;
+
+ Ok(Self { inner: db })
+ }
+
+ pub fn get(&self, node: &NodeID) -> Option<f64> {
+ self.inner.get(node)
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = (NodeID, f64)> + '_ {
+ self.inner.iter()
+ }
+}
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index f88aa9f5..bbedfe07 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -16,12 +16,9 @@
use std::{
collections::{BTreeMap, BTreeSet},
- fs,
- path::{Path, PathBuf},
- sync::{atomic::AtomicBool, Mutex},
+ sync::atomic::AtomicBool,
};
-use anyhow::Result;
use std::sync::atomic::Ordering;
use bitvec::vec::BitVec;
@@ -85,106 +82,10 @@ impl JankyBloomFilter {
}
}
-struct Db<K, V>
-where
- K: serde::Serialize + serde::de::DeserializeOwned,
- V: serde::Serialize + serde::de::DeserializeOwned,
-{
- db: rocksdb::DB,
- path: PathBuf,
- _cache: rocksdb::Cache,
- _phantom: std::marker::PhantomData<(K, V)>,
-}
-
-impl<K, V> Db<K, V>
-where
- K: serde::Serialize + serde::de::DeserializeOwned,
- V: serde::Serialize + serde::de::DeserializeOwned,
-{
- fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- if !path.as_ref().exists() {
- fs::create_dir_all(path.as_ref())?;
- }
-
- let mut options = rocksdb::Options::default();
- options.create_if_missing(true);
-
- options.set_max_background_jobs(8);
- options.increase_parallelism(8);
- options.set_max_subcompactions(8);
-
- let cache = rocksdb::Cache::new_lru_cache(20 * 1024 * 1024 * 1024); // 20 gb
-
- // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
- options.set_level_compaction_dynamic_level_bytes(true);
- options.set_bytes_per_sync(1048576);
- let mut block_options = rocksdb::BlockBasedOptions::default();
- block_options.set_block_size(16 * 1024);
- block_options.set_format_version(5);
- block_options.set_cache_index_and_filter_blocks(true);
- block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
-
- block_options.set_ribbon_filter(10.0);
- block_options.set_block_cache(&cache);
-
- options.set_block_based_table_factory(&block_options);
- options.set_compression_type(rocksdb::DBCompressionType::None);
-
- options.optimize_for_point_lookup(512); // 512 MB
-
- let db = rocksdb::DB::open(&options, &path)?;
-
- Ok(Self {
- db,
- path: path.as_ref().to_path_buf(),
- _cache: cache,
- _phantom: std::marker::PhantomData,
- })
- }
-
- fn get(&self, key: &K) -> Option<V> {
- let key_bytes = bincode::serialize(key).expect("failed to serialize key");
-
- self.db
- .get(key_bytes)
- .expect("failed to retrieve key")
- .map(|bytes| bincode::deserialize(&bytes).expect("failed to deserialize stored value"))
- }
-
- fn insert(&self, key: &K, value: &V) {
- let key = bincode::serialize(key).expect("failed to serialize key");
- let value = bincode::serialize(value).expect("failed to serialize value");
-
- self.db.put(key, value).expect("failed to insert key");
- }
-
- fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
- self.db.iterator(rocksdb::IteratorMode::Start).map(|s| {
- let (key, value) = s.expect("failed to iterate over db");
- (
- bincode::deserialize(&key).expect("failed to deserialize key"),
- bincode::deserialize(&value).expect("failed to deserialize value"),
- )
- })
- }
-
- fn path(&self) -> &Path {
- &self.path
- }
-}
-
-fn calculate_centrality<P: AsRef<Path>>(
- graph: &Webgraph,
- calculation_path: P,
-) -> BTreeMap<NodeID, f64> {
+fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let mut num_nodes = 0;
- let mut counters: Db<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = Db::open(
- calculation_path
- .as_ref()
- .join(uuid::Uuid::new_v4().to_string()),
- )
- .unwrap();
+ let mut counters: BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = BTreeMap::new();
let mut centralities: BTreeMap<NodeID, KahanSum> = BTreeMap::new();
@@ -192,7 +93,7 @@ fn calculate_centrality<P: AsRef<Path>>(
let mut counter = HyperLogLog::default();
counter.add(node.bit_64());
- counters.insert(&node, &counter);
+ counters.insert(node, counter);
centralities.insert(node, KahanSum::default());
num_nodes += 1;
@@ -219,19 +120,10 @@ fn calculate_centrality<P: AsRef<Path>>(
break;
}
- let new_counters: Db<_, _> = Db::open(
- calculation_path
- .as_ref()
- .join(uuid::Uuid::new_v4().to_string()),
- )
- .unwrap();
-
- for (node, counter) in counters.iter() {
- new_counters.insert(&node, &counter);
- }
+ let mut new_counters = counters.clone();
has_changes.store(false, Ordering::Relaxed);
- let new_changed_nodes = Mutex::new(JankyBloomFilter::new(num_nodes as u64, 0.05));
+ let mut new_changed_nodes = JankyBloomFilter::new(num_nodes as u64, 0.05);
if !exact_changed_nodes.is_empty()
&& exact_changed_nodes.len() as u64 <= exact_counting_threshold
@@ -240,8 +132,8 @@ fn calculate_centrality<P: AsRef<Path>>(
exact_changed_nodes.iter().for_each(|changed_node| {
for edge in graph.raw_outgoing_edges(changed_node) {
- if let (Some(mut counter_to), Some(counter_from)) =
- (new_counters.get(&edge.to), counters.get(&edge.from))
+ if let (Some(counter_to), Some(counter_from)) =
+ (new_counters.get_mut(&edge.to), counters.get(&edge.from))
{
if counter_to
.registers()
@@ -249,9 +141,8 @@ fn calculate_centrality<P: AsRef<Path>>(
.zip(counter_from.registers().iter())
.any(|(to, from)| *from > *to)
{
- counter_to.merge(&counter_from);
- new_counters.insert(&edge.to, &counter_to);
- new_changed_nodes.lock().unwrap().insert(edge.to.bit_64());
+ counter_to.merge(counter_from);
+ new_changed_nodes.insert(edge.to.bit_64());
new_exact_changed_nodes.insert(edge.to);
@@ -266,8 +157,8 @@ fn calculate_centrality<P: AsRef<Path>>(
exact_changed_nodes = BTreeSet::default();
graph.edges().for_each(|edge| {
if changed_nodes.contains(&edge.from.bit_64()) {
- if let (Some(mut counter_to), Some(counter_from)) =
- (new_counters.get(&edge.to), counters.get(&edge.from))
+ if let (Some(counter_to), Some(counter_from)) =
+ (new_counters.get_mut(&edge.to), counters.get(&edge.from))
{
if counter_to
.registers()
@@ -275,9 +166,8 @@ fn calculate_centrality<P: AsRef<Path>>(
.zip(counter_from.registers().iter())
.any(|(to, from)| *from > *to)
{
- counter_to.merge(&counter_from);
- new_counters.insert(&edge.to, &counter_to);
- new_changed_nodes.lock().unwrap().insert(edge.to.bit_64());
+ counter_to.merge(counter_from);
+ new_changed_nodes.insert(edge.to.bit_64());
if exact_counting {
exact_changed_nodes.insert(edge.to);
@@ -306,9 +196,7 @@ fn calculate_centrality<P: AsRef<Path>>(
});
counters = new_counters;
- let old_path = counters.path().to_path_buf();
- changed_nodes = new_changed_nodes.into_inner().unwrap();
- std::fs::remove_dir_all(old_path).unwrap();
+ changed_nodes = new_changed_nodes;
t += 1;
if changed_nodes.estimate_card() <= exact_counting_threshold {
@@ -331,8 +219,8 @@ fn calculate_centrality<P: AsRef<Path>>(
pub struct HarmonicCentrality(BTreeMap<NodeID, f64>);
impl HarmonicCentrality {
- pub fn calculate<P: AsRef<Path>>(graph: &Webgraph, path: P) -> Self {
- Self(calculate_centrality(graph, path))
+ pub fn calculate(graph: &Webgraph) -> Self {
+ Self(calculate_centrality(graph))
}
pub fn get(&self, node: &NodeID) -> Option<f64> {
@@ -464,7 +352,7 @@ mod tests {
let graph = writer.finalize();
- let centrality = HarmonicCentrality::calculate(&graph, crate::gen_temp_path());
+ let centrality = HarmonicCentrality::calculate(&graph);
assert!(
centrality.get(&Node::from("B.com").id()).unwrap()
@@ -475,7 +363,7 @@ mod tests {
#[test]
fn harmonic_centrality() {
let graph = test_graph();
- let centrality = HarmonicCentrality::calculate(&graph, crate::gen_temp_path());
+ let centrality = HarmonicCentrality::calculate(&graph);
assert!(
centrality.get(&Node::from("C").id()).unwrap()
@@ -491,7 +379,7 @@ mod tests {
#[test]
fn additional_edges_ignored() {
let graph = test_graph();
- let centrality = HarmonicCentrality::calculate(&graph, crate::gen_temp_path());
+ let centrality = HarmonicCentrality::calculate(&graph);
let mut other = WebgraphWriter::new(
crate::gen_temp_path(),
@@ -520,7 +408,7 @@ mod tests {
let graph = other.finalize();
- let centrality_extra = HarmonicCentrality::calculate(&graph, crate::gen_temp_path());
+ let centrality_extra = HarmonicCentrality::calculate(&graph);
assert_eq!(centrality.0, centrality_extra.0);
}
diff --git a/core/src/webgraph/centrality/mod.rs b/core/src/webgraph/centrality/mod.rs
index df9aa96a..9fe59ff1 100644
--- a/core/src/webgraph/centrality/mod.rs
+++ b/core/src/webgraph/centrality/mod.rs
@@ -15,4 +15,5 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
pub mod betweenness;
+pub mod derived_harmonic;
pub mod harmonic;