web search done right
Ability to open search or indexer specific centrality store. This reduces the required memory for each part
Mikkel Denker191652ccd57f
diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rsindex af256ebc..f6db97ed 100644--- a/core/benches/search-preindexed.rs+++ b/core/benches/search-preindexed.rs@@ -29,7 +29,7 @@ macro_rules! bench {pub fn criterion_benchmark(c: &mut Criterion) {let index = Index::open(INDEX_PATH).unwrap();let mut searcher = LocalSearcher::new(index);- searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH));+ searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH).into());for _ in 0..100 {bench!("the", searcher, c);diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex d8fd2b7e..62fc190a 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -26,7 +26,7 @@ use tracing::{debug, info, trace};use crate::entrypoint::async_download_all_warc_files;use crate::index::{FrozenIndex, Index};use crate::mapreduce::{Manager, Map, Reduce, Worker};-use crate::ranking::centrality_store::CentralityStore;+use crate::ranking::centrality_store::IndexerCentralityStore;use crate::ranking::SignalAggregator;use crate::warc::WarcFile;use crate::webgraph::{Node, Webgraph, WebgraphBuilder};@@ -57,7 +57,7 @@ pub struct Job {}pub struct IndexingWorker {- centrality_store: CentralityStore,+ centrality_store: IndexerCentralityStore,webgraph: Option<Webgraph>,crawl_stabilty: Option<CrawlStability>,topics: Option<human_website_annotations::Mapper>,@@ -71,7 +71,7 @@ impl IndexingWorker {topics_path: Option<String>,) -> Self {Self {- centrality_store: CentralityStore::open(centrality_store_path),+ centrality_store: IndexerCentralityStore::open(centrality_store_path),webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).read_only(true).open()),crawl_stabilty: crawl_stability_path.map(CrawlStability::open),topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),@@ -170,7 +170,6 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {let node_id = worker.centrality_store- .online_harmonic.node2id.get(&Node::from_url(html.url()).into_host()).cloned();diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 4f0939ad..12c71049 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -20,7 +20,7 @@ use crate::{entity_index::EntityIndex,index::Index,inverted_index,- ranking::centrality_store::CentralityStore,+ ranking::centrality_store::SearchCentralityStore,searcher::{self, LocalSearcher},sonic, Result, SearchServerConfig,};@@ -33,7 +33,9 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {let entity_index = config.entity_index_path.map(|path| EntityIndex::open(path).unwrap());- let centrality_store = config.centrality_store_path.map(CentralityStore::open);+ let centrality_store = config+ .centrality_store_path+ .map(SearchCentralityStore::open);let search_index = Index::open(config.index_path)?;let mut local_searcher = LocalSearcher::new(search_index);diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 2e2334d2..40851663 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -706,7 +706,7 @@ mod tests {index.commit().expect("failed to commit index");let mut searcher = LocalSearcher::from(index);- searcher.set_centrality_store(centrality_store);+ searcher.set_centrality_store(centrality_store.into());let res = searcher.search(&SearchQuery {diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex c1413b19..a0f254fb 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -47,6 +47,57 @@ impl HarmonicCentralityStore {}}+pub struct IndexerCentralityStore {+ pub harmonic: HarmonicCentralityStore,+ pub node2id: BTreeMap<Node, NodeID>,+}++impl IndexerCentralityStore {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ Self {+ harmonic: HarmonicCentralityStore::open(path.as_ref().join("harmonic")),+ node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),+ }+ }+}++impl From<CentralityStore> for IndexerCentralityStore {+ fn from(store: CentralityStore) -> Self {+ Self {+ node2id: store.node2id,+ harmonic: store.harmonic,+ }+ }+}++pub struct SearchCentralityStore {+ pub online_harmonic: OnlineHarmonicCentrality,+ pub inbound_similarity: InboundSimilarity,+ pub node2id: BTreeMap<Node, NodeID>,+}++impl SearchCentralityStore {+ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ Self {+ online_harmonic: OnlineHarmonicCentrality::open(path.as_ref().join("online_harmonic"))+ .unwrap(),+ inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))+ .unwrap(),+ node2id: open_node2id(path.as_ref().join("node2id")).unwrap(),+ }+ }+}++impl From<CentralityStore> for SearchCentralityStore {+ fn from(store: CentralityStore) -> Self {+ Self {+ online_harmonic: store.online_harmonic,+ inbound_similarity: store.inbound_similarity,+ node2id: store.node2id,+ }+ }+}+pub struct CentralityStore {pub harmonic: HarmonicCentralityStore,pub online_harmonic: OnlineHarmonicCentrality,diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 330a0769..27dbc3f1 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -260,7 +260,7 @@ mod tests {index.commit().unwrap();let mut searcher = LocalSearcher::new(index);- searcher.set_centrality_store(centrality_store);+ searcher.set_centrality_store(centrality_store.into());let res = searcher.search(&SearchQuery {diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 8e65982b..8012d0a6 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -189,7 +189,7 @@ mod tests {index.commit().expect("failed to commit index");let mut searcher = LocalSearcher::new(index);- searcher.set_centrality_store(CentralityStore::build(&graph, gen_temp_path()));+ searcher.set_centrality_store(CentralityStore::build(&graph, gen_temp_path()).into());let result = searcher.search(&SearchQuery {diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 338fa72b..40a06f4a 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -24,7 +24,7 @@ use crate::image_store::Image;use crate::index::Index;use crate::query::parser::Term;use crate::query::{self, Query};-use crate::ranking::centrality_store::CentralityStore;+use crate::ranking::centrality_store::SearchCentralityStore;use crate::ranking::optics::CreateAggregator;use crate::ranking::pipeline::RankingWebsite;use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};@@ -42,7 +42,7 @@ use super::{InitialWebsiteResult, SearchQuery};pub struct LocalSearcher {index: Index,entity_index: Option<EntityIndex>,- centrality_store: Option<CentralityStore>,+ centrality_store: Option<SearchCentralityStore>,topic_centrality: Option<TopicCentrality>,}@@ -66,7 +66,7 @@ impl LocalSearcher {self.entity_index = Some(entity_index);}- pub fn set_centrality_store(&mut self, centrality_store: CentralityStore) {+ pub fn set_centrality_store(&mut self, centrality_store: SearchCentralityStore) {self.centrality_store = Some(centrality_store);}