Skip to content

web search done right

Ability to open search or indexer specific centrality store. This reduces the required memory for each part

Mikkel Denker191652ccd57f
diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rs
index af256ebc..f6db97ed 100644
--- a/core/benches/search-preindexed.rs
+++ b/core/benches/search-preindexed.rs
@@ -29,7 +29,7 @@ macro_rules! bench {
pub fn criterion_benchmark(c: &mut Criterion) {
let index = Index::open(INDEX_PATH).unwrap();
let mut searcher = LocalSearcher::new(index);
- searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH));
+ searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH).into());
for _ in 0..100 {
bench!("the", searcher, c);
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index d8fd2b7e..62fc190a 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -26,7 +26,7 @@ use tracing::{debug, info, trace};
use crate::entrypoint::async_download_all_warc_files;
use crate::index::{FrozenIndex, Index};
use crate::mapreduce::{Manager, Map, Reduce, Worker};
-use crate::ranking::centrality_store::CentralityStore;
+use crate::ranking::centrality_store::IndexerCentralityStore;
use crate::ranking::SignalAggregator;
use crate::warc::WarcFile;
use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
@@ -57,7 +57,7 @@ pub struct Job {
}
pub struct IndexingWorker {
- centrality_store: CentralityStore,
+ centrality_store: IndexerCentralityStore,
webgraph: Option<Webgraph>,
crawl_stabilty: Option<CrawlStability>,
topics: Option<human_website_annotations::Mapper>,
@@ -71,7 +71,7 @@ impl IndexingWorker {
topics_path: Option<String>,
) -> Self {
Self {
- centrality_store: CentralityStore::open(centrality_store_path),
+ centrality_store: IndexerCentralityStore::open(centrality_store_path),
webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).read_only(true).open()),
crawl_stabilty: crawl_stability_path.map(CrawlStability::open),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
@@ -170,7 +170,6 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
let node_id = worker
.centrality_store
- .online_harmonic
.node2id
.get(&Node::from_url(html.url()).into_host())
.cloned();
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 4f0939ad..12c71049 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -20,7 +20,7 @@ use crate::{
entity_index::EntityIndex,
index::Index,
inverted_index,
- ranking::centrality_store::CentralityStore,
+ ranking::centrality_store::SearchCentralityStore,
searcher::{self, LocalSearcher},
sonic, Result, SearchServerConfig,
};
@@ -33,7 +33,9 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
let entity_index = config
.entity_index_path
.map(|path| EntityIndex::open(path).unwrap());
- let centrality_store = config.centrality_store_path.map(CentralityStore::open);
+ let centrality_store = config
+ .centrality_store_path
+ .map(SearchCentralityStore::open);
let search_index = Index::open(config.index_path)?;
let mut local_searcher = LocalSearcher::new(search_index);
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 2e2334d2..40851663 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -706,7 +706,7 @@ mod tests {
index.commit().expect("failed to commit index");
let mut searcher = LocalSearcher::from(index);
- searcher.set_centrality_store(centrality_store);
+ searcher.set_centrality_store(centrality_store.into());
let res = searcher
.search(&SearchQuery {
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index c1413b19..a0f254fb 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -47,6 +47,57 @@ impl HarmonicCentralityStore {
}
}
+pub struct IndexerCentralityStore {
+ pub harmonic: HarmonicCentralityStore,
+ pub node2id: BTreeMap<Node, NodeID>,
+}
+
+impl IndexerCentralityStore {
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ Self {
+ harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),
+ node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),
+ }
+ }
+}
+
+impl From<CentralityStore> for IndexerCentralityStore {
+ fn from(store: CentralityStore) -> Self {
+ Self {
+ node2id: store.node2id,
+ harmonic: store.harmonic,
+ }
+ }
+}
+
+pub struct SearchCentralityStore {
+ pub online_harmonic: OnlineHarmonicCentrality,
+ pub inbound_similarity: InboundSimilarity,
+ pub node2id: BTreeMap<Node, NodeID>,
+}
+
+impl SearchCentralityStore {
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ Self {
+ online_harmonic: OnlineHarmonicCentrality::open(path.as_ref().join("online_harmonic"))
+ .unwrap(),
+ inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))
+ .unwrap(),
+ node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),
+ }
+ }
+}
+
+impl From<CentralityStore> for SearchCentralityStore {
+ fn from(store: CentralityStore) -> Self {
+ Self {
+ online_harmonic: store.online_harmonic,
+ inbound_similarity: store.inbound_similarity,
+ node2id: store.node2id,
+ }
+ }
+}
+
pub struct CentralityStore {
pub harmonic: HarmonicCentralityStore,
pub online_harmonic: OnlineHarmonicCentrality,
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 330a0769..27dbc3f1 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -260,7 +260,7 @@ mod tests {
index.commit().unwrap();
let mut searcher = LocalSearcher::new(index);
- searcher.set_centrality_store(centrality_store);
+ searcher.set_centrality_store(centrality_store.into());
let res = searcher
.search(&SearchQuery {
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 8e65982b..8012d0a6 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -189,7 +189,7 @@ mod tests {
index.commit().expect("failed to commit index");
let mut searcher = LocalSearcher::new(index);
- searcher.set_centrality_store(CentralityStore::build(&graph, gen_temp_path()));
+ searcher.set_centrality_store(CentralityStore::build(&graph, gen_temp_path()).into());
let result = searcher
.search(&SearchQuery {
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 338fa72b..40a06f4a 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -24,7 +24,7 @@ use crate::image_store::Image;
use crate::index::Index;
use crate::query::parser::Term;
use crate::query::{self, Query};
-use crate::ranking::centrality_store::CentralityStore;
+use crate::ranking::centrality_store::SearchCentralityStore;
use crate::ranking::optics::CreateAggregator;
use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
@@ -42,7 +42,7 @@ use super::{InitialWebsiteResult, SearchQuery};
pub struct LocalSearcher {
index: Index,
entity_index: Option<EntityIndex>,
- centrality_store: Option<CentralityStore>,
+ centrality_store: Option<SearchCentralityStore>,
topic_centrality: Option<TopicCentrality>,
}
@@ -66,7 +66,7 @@ impl LocalSearcher {
self.entity_index = Some(entity_index);
}
- pub fn set_centrality_store(&mut self, centrality_store: CentralityStore) {
+ pub fn set_centrality_store(&mut self, centrality_store: SearchCentralityStore) {
self.centrality_store = Some(centrality_store);
}