Skip to content

web search done right

centrality store in webgraph creation is now raw centrality instead of rank

Mikkel Denkerdda40bd4e5a1
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 184d04d2..b45c6b9d 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -99,7 +99,7 @@ pub struct IndexerDualEncoderConfig {
pub struct WebgraphConstructConfig {
pub graph_base_path: String,
pub shard: ShardId,
- pub host_centrality_rank_store_path: String,
+ pub host_centrality_store_path: String,
pub warc_source: WarcSource,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 1cf99a6b..d714447f 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -13,6 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
use crate::{
canon_index::CanonicalIndex,
config::{self, WarcSource, WebgraphConstructConfig},
@@ -176,8 +177,8 @@ impl Webgraph {
None
};
- let host_centrality_rank_store = Arc::new(speedy_kv::Db::open_or_create(
- &config.host_centrality_rank_store_path,
+ let host_centrality_store = Arc::new(speedy_kv::Db::open_or_create(
+ &config.host_centrality_store_path,
)?);
let num_workers = usize::from(std::thread::available_parallelism()?);
@@ -198,7 +199,7 @@ impl Webgraph {
let mut worker = WebgraphWorker {
graph: webgraph::Webgraph::open(graph_path, config.shard)?,
- host_centrality_store: Some(host_centrality_rank_store.clone()),
+ host_centrality_store: Some(host_centrality_store.clone()),
canonical_index: canonical_index.clone(),
};