web search done right
calculate query specific centrality during ranking
Mikkel Denker0d2fb2444286
diff --git a/configs/search_server.toml b/configs/search_server.tomlindex 089bf194..c35d3057 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -1,4 +1,5 @@host = "0.0.0.0:3001"index_path = "data/index"entity_index_path = "data/entity"-bangs_path = "data/bangs.json"\ No newline at end of file+bangs_path = "data/bangs.json"+centrality_store_path = "data/centrality"\ No newline at end of filediff --git a/src/human_website_annotations.rs b/src/human_website_annotations.rsindex 64eca81f..cb3ab7a7 100644--- a/src/human_website_annotations.rs+++ b/src/human_website_annotations.rs@@ -104,13 +104,13 @@ impl Mapper {}pub fn all_topics(&self) -> HashSet<Topic> {- self.0.iter().map(|(_, info)| info.topic.clone()).collect()+ self.0.values().map(|info| info.topic.clone()).collect()}pub fn top_topics(&self, top_n: usize) -> Vec<Topic> {let mut topics: HashMap<Topic, usize> = HashMap::new();- for topic in self.0.iter().map(|(_, info)| info.topic.clone()) {+ for topic in self.0.values().map(|info| info.topic.clone()) {*topics.entry(topic).or_default() += 1;}diff --git a/src/index.rs b/src/index.rsindex 844a45b4..f7a06b8b 100644--- a/src/index.rs+++ b/src/index.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::cmp::Ordering;+use std::collections::HashSet;use std::fs;use std::path::Path;use std::sync::Arc;@@ -35,6 +36,7 @@ use crate::spell::{Correction, CorrectionTerm, Dictionary, LogarithmicEdit, SpellChecker, TermSplitter,};use crate::subdomain_count::SubdomainCounter;+use crate::webgraph::NodeID;use crate::webpage::region::{Region, RegionCount};use crate::webpage::{Url, Webpage};use crate::Result;@@ -134,6 +136,24 @@ impl Index {self.inverted_index.search_initial(query, collector)}+ pub fn top_nodes<C>(&self, query: &Query, collector: C) -> Result<Vec<NodeID>>+ where+ C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,+ {+ let pointers = self+ .inverted_index+ .search_initial(query, collector)?+ .top_websites;++ let mut hosts = HashSet::with_capacity(pointers.len());+ for pointer in &pointers {+ if let Some(id) = self.inverted_index.website_host_node(pointer)? {+ hosts.insert(id);+ }+ }+ Ok(hosts.into_iter().collect())+ }+pub fn retrieve_websites(&self,websites: &[inverted_index::WebsitePointer],diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex d93b020e..b17b0725 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -31,6 +31,7 @@ use crate::query::Query;use crate::schema::{FastField, Field, TextField, ALL_FIELDS};use crate::snippet;use crate::tokenizer::Identity;+use crate::webgraph::NodeID;use crate::webpage::region::Region;use crate::webpage::{StoredPrimaryImage, Webpage};use crate::Result;@@ -191,6 +192,22 @@ impl InvertedIndex {})}+ pub fn website_host_node(&self, website: &WebsitePointer) -> Result<Option<NodeID>> {+ let searcher = self.reader.searcher();+ let doc = searcher.doc(website.address.into())?;+ let field = self+ .schema()+ .get_field(Field::Fast(FastField::HostNodeID).name())+ .unwrap();++ let id = doc.get_first(field).unwrap().as_u64().unwrap();+ if id == u64::MAX {+ Ok(None)+ } else {+ Ok(Some(id))+ }+ }+pub fn retrieve_websites(&self,websites: &[WebsitePointer],diff --git a/src/main.rs b/src/main.rsindex 36c679c9..445fbb41 100644--- a/src/main.rs+++ b/src/main.rs@@ -131,7 +131,7 @@ fn main() -> Result<()> {match args.command {Commands::Indexer { options } => match options {IndexingOptions::Master { config_path } => {- let config = load_toml_config(&config_path);+ let config = load_toml_config(config_path);entrypoint::Indexer::run_master(&config)?;}IndexingOptions::Worker {@@ -150,7 +150,7 @@ fn main() -> Result<()> {)?;}IndexingOptions::Local { config_path } => {- let config = load_toml_config(&config_path);+ let config = load_toml_config(config_path);entrypoint::Indexer::run_locally(&config)?;}IndexingOptions::Entity {@@ -183,7 +183,7 @@ fn main() -> Result<()> {}},Commands::Frontend { config_path } => {- let config: FrontendConfig = load_toml_config(&config_path);+ let config: FrontendConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all()@@ -195,7 +195,7 @@ fn main() -> Result<()> {))?}Commands::SearchServer { config_path } => {- let config: SearchServerConfig = load_toml_config(&config_path);+ let config: SearchServerConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all()diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex d4e22c5e..f29b1b9b 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -20,7 +20,7 @@ pub mod ast;mod const_query;mod pattern_query;-use std::{collections::HashMap, convert::TryFrom, sync::Arc};+use std::{collections::HashMap, convert::TryFrom};use crate::{query::union::UnionQuery,@@ -257,9 +257,7 @@ impl Goggle {);if let Some(approx) = approx {- aggregator- .personal_harmonic- .push(Arc::new(self.site_rankings.centrality_scorer(approx)));+ aggregator.add_personal_harmonic(self.site_rankings.centrality_scorer(approx));}aggregatordiff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 2d205d36..e51f653a 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -31,7 +31,7 @@ use crate::{fastfield_cache::FastFieldCache,inverted_index,searcher::NUM_RESULTS_PER_PAGE,- webgraph::centrality::topic,+ webgraph::centrality::{approximate_harmonic, topic},webpage::region::{Region, RegionCount},};@@ -46,6 +46,7 @@ pub struct Ranker {fastfield_cache: Arc<FastFieldCache>,de_rank_similar: bool,topic_scorer: Option<topic::Scorer>,+ num_results: Option<usize>,}impl Ranker {@@ -63,6 +64,7 @@ impl Ranker {de_rank_similar: true,fastfield_cache,topic_scorer: None,+ num_results: None,}}@@ -84,6 +86,11 @@ impl Ranker {self}+ pub fn with_num_results(mut self, num_results: usize) -> Self {+ self.num_results = Some(num_results);+ self+ }+pub fn de_rank_similar(&mut self, de_rank_similar: bool) {self.de_rank_similar = de_rank_similar;}@@ -92,7 +99,7 @@ impl Ranker {let mut aggregator = self.aggregator.clone();if let Some(topic_scorer) = self.topic_scorer.clone() {- aggregator.register_topic_scorer(topic_scorer);+ aggregator.set_topic_scorer(topic_scorer);}let score_tweaker = InitialScoreTweaker::new(@@ -102,8 +109,10 @@ impl Ranker {Arc::clone(&self.fastfield_cache),);- let mut collector =- TopDocs::with_limit(NUM_RESULTS_PER_PAGE, Arc::clone(&self.fastfield_cache));+ let mut collector = TopDocs::with_limit(+ self.num_results.unwrap_or(NUM_RESULTS_PER_PAGE),+ Arc::clone(&self.fastfield_cache),+ );if self.de_rank_similar {collector = collector.and_de_rank_similar()@@ -120,9 +129,13 @@ impl Ranker {collector.tweak_score(score_tweaker)}- pub(crate) fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {+ pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {self.topic_scorer = Some(topic_scorer);}++ pub fn set_query_centrality(&mut self, approx: approximate_harmonic::Scorer) {+ self.aggregator.set_query_centrality(approx);+ }}#[cfg(test)]diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex 8dfa2041..4ed1561c 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -17,7 +17,10 @@use crate::{fastfield_cache,schema::{FastField, TextField},- webgraph::centrality::{approximate_harmonic, topic},+ webgraph::centrality::{+ approximate_harmonic::{self, SHIFT},+ topic,+ },webpage::Webpage,Result,};@@ -46,9 +49,10 @@ pub enum Signal {PersonalCentrality,CrawlStability,TopicCentrality,+ QueryCentrality,}-pub const ALL_SIGNALS: [Signal; 11] = [+pub const ALL_SIGNALS: [Signal; 12] = [Signal::Bm25,Signal::HostCentrality,Signal::PageCentrality,@@ -60,6 +64,7 @@ pub const ALL_SIGNALS: [Signal; 11] = [Signal::PersonalCentrality,Signal::CrawlStability,Signal::TopicCentrality,+ Signal::QueryCentrality,];struct SignalValue {@@ -69,6 +74,7 @@ struct SignalValue {selected_region: Option<Region>,personal_centrality: f64,topic_score: Option<f64>,+ query_centrality: Option<f64>,}impl Signal {@@ -133,6 +139,7 @@ impl Signal {}Signal::CrawlStability => value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64,Signal::TopicCentrality => value.topic_score.unwrap_or_default(),+ Signal::QueryCentrality => value.query_centrality.unwrap_or_default(),}}@@ -142,12 +149,13 @@ impl Signal {Signal::HostCentrality => 2500.0,Signal::PageCentrality => 4500.0,Signal::TopicCentrality => 2500.0,+ Signal::PersonalCentrality => 4500.0,+ Signal::QueryCentrality => 2500.0,Signal::IsHomepage => 0.1,Signal::FetchTimeMs => 0.01,Signal::UpdateTimestamp => 80.0,Signal::NumTrackers => 20.0,Signal::Region => 60.0,- Signal::PersonalCentrality => 2500.0,Signal::CrawlStability => 20.0,}}@@ -164,6 +172,7 @@ impl Signal {"region" => Some(Signal::Region),"personal_centrality" => Some(Signal::PersonalCentrality),"topic_centrality" => Some(Signal::TopicCentrality),+ "query_centrality" => Some(Signal::QueryCentrality),_ => None,}}@@ -181,6 +190,7 @@ impl Signal {Signal::CrawlStability => Some(FastField::CrawlStability),Signal::PersonalCentrality => None,Signal::TopicCentrality => None,+ Signal::QueryCentrality => None,}}}@@ -248,11 +258,12 @@ impl FieldBoost {pub struct SignalAggregator {fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,signal_coefficients: SignalCoefficient,- pub personal_harmonic: Vec<Arc<approximate_harmonic::Scorer>>,+ personal_centrality: Vec<Arc<approximate_harmonic::Scorer>>,field_boost: FieldBoost,fetch_time_ms_cache: [f64; 1000],update_time_cache: Vec<f64>,topic_scorer: Option<topic::Scorer>,+ query_centrality: Option<Arc<approximate_harmonic::Scorer>>,}impl std::fmt::Debug for SignalAggregator {@@ -286,12 +297,13 @@ impl SignalAggregator {Self {fastfield_cache: None,- personal_harmonic: Vec::new(),+ personal_centrality: Vec::new(),signal_coefficients,field_boost,fetch_time_ms_cache,update_time_cache,topic_scorer: None,+ query_centrality: None,}}@@ -299,10 +311,18 @@ impl SignalAggregator {self.fastfield_cache = Some(cache);}- pub fn register_topic_scorer(&mut self, topic_scorer: topic::Scorer) {+ pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {self.topic_scorer = Some(topic_scorer);}+ pub fn set_query_centrality(&mut self, query_centrality: approximate_harmonic::Scorer) {+ self.query_centrality = Some(Arc::new(query_centrality));+ }++ pub fn add_personal_harmonic(&mut self, personal_centrality: approximate_harmonic::Scorer) {+ self.personal_centrality.push(Arc::new(personal_centrality))+ }+pub fn score(&self,doc: DocId,@@ -311,20 +331,20 @@ impl SignalAggregator {current_timestamp: usize,selected_region: Option<Region>,) -> f64 {- let topic_score = self.topic_scorer.as_ref().and_then(|scorer| {- self.fastfield_cache.as_ref().and_then(|cache| {- cache- .get_doc_cache(&FastField::HostNodeID)- .get_u64(&doc)- .and_then(|host| {- if host == u64::MAX {- None- } else {- Some(scorer.score(host))- }- })- })- });+ let host_id = self+ .fastfield_cache+ .as_ref()+ .and_then(|cache| cache.get_doc_cache(&FastField::HostNodeID).get_u64(&doc));++ let topic_score = self+ .topic_scorer+ .as_ref()+ .and_then(|scorer| host_id.map(|host| scorer.score(host)));++ let query_centrality = self+ .query_centrality+ .as_ref()+ .and_then(|scorer| host_id.map(|host| scorer.score(host) - SHIFT));ALL_SIGNALS.into_iter()@@ -336,7 +356,7 @@ impl SignalAggregator {});let personal_centrality = self- .personal_harmonic+ .personal_centrality.iter().map(|scorer| scorer.score(doc as u64)).sum();@@ -352,6 +372,7 @@ impl SignalAggregator {selected_region,personal_centrality,topic_score,+ query_centrality,},)})@@ -382,7 +403,10 @@ impl SignalAggregator {Signal::CrawlStability => {(webpage.crawl_stability * (FLOAT_SCALING as f64)) as u64}- Signal::Bm25 | Signal::PersonalCentrality | Signal::TopicCentrality => {+ Signal::Bm25+ | Signal::PersonalCentrality+ | Signal::TopicCentrality+ | Signal::QueryCentrality => {panic!("signal cannot be determined from webpage")}};@@ -400,6 +424,7 @@ impl SignalAggregator {selected_region: None,personal_centrality: 0.0,topic_score: None,+ query_centrality: None,},)})diff --git a/src/searcher/local.rs b/src/searcher/local.rsindex 74e2f439..0e468f2c 100644--- a/src/searcher/local.rs+++ b/src/searcher/local.rs@@ -142,10 +142,6 @@ impl LocalSearcher {self.index.inverted_index.fastfield_cache(),);- if let Some(skip_pages) = query.skip_pages {- ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);- }-if let Some(region) = query.selected_region {if region != Region::All {ranker = ranker.with_region(region);@@ -157,9 +153,30 @@ impl LocalSearcher {ranker.set_topic_scorer(topic_scorer);}- ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());ranker.de_rank_similar(de_rank_similar);+ if let Some(centrality_store) = self.centrality_store.as_ref() {+ ranker = ranker+ .with_max_docs(10_000, self.index.num_segments())+ .with_num_results(100);++ let top_host_nodes = self.index.top_nodes(&parsed_query, ranker.collector())?;+ if !top_host_nodes.is_empty() {+ let approx = centrality_store+ .approx_harmonic+ .scorer_without_fixed_from_ids(&top_host_nodes, &[]);+ ranker.set_query_centrality(approx);+ }+ }++ ranker = ranker+ .with_max_docs(10_000_000, self.index.num_segments())+ .with_num_results(NUM_RESULTS_PER_PAGE);++ if let Some(skip_pages) = query.skip_pages {+ ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);+ }+let webpages = self.index.search_initial(&parsed_query, ranker.collector())?;diff --git a/src/spell/distance.rs b/src/spell/distance.rsindex 5317f439..f294f218 100644--- a/src/spell/distance.rs+++ b/src/spell/distance.rs@@ -77,7 +77,7 @@ impl LevenshteinDistance {// insertioncur[j] + 1,// match or substitution- pre + if ca == cb { 0 } else { 1 },+ pre + usize::from(ca != cb),),);pre = tmp;diff --git a/src/spell/spell_checker.rs b/src/spell/spell_checker.rsindex a826a613..ef413f0d 100644--- a/src/spell/spell_checker.rs+++ b/src/spell/spell_checker.rs@@ -62,15 +62,10 @@ impl<'a, T: EditStrategy, const DICT_N: usize> SpellChecker<'a, T, DICT_N> {}pub fn correct(&self, term: &str) -> Option<String> {- if let Some(correction) = self.correct_top(term, 1).into_iter().next() {- if correction.to_ascii_lowercase() == term.to_ascii_lowercase() {- None- } else {- Some(correction)- }- } else {- None- }+ self.correct_top(term, 1)+ .into_iter()+ .next()+ .filter(|correction| correction.to_ascii_lowercase() != term.to_ascii_lowercase())}}diff --git a/src/webgraph/centrality/approximate_harmonic.rs b/src/webgraph/centrality/approximate_harmonic.rsindex c727d1bf..2aba833e 100644--- a/src/webgraph/centrality/approximate_harmonic.rs+++ b/src/webgraph/centrality/approximate_harmonic.rs@@ -340,10 +340,18 @@ impl ApproximatedHarmonicCentrality {.filter_map(|node| self.node2id.get(&node).copied()).collect_vec();+ self.scorer_without_fixed_from_ids(&liked_nodes, &disliked_nodes)+ }++ pub fn scorer_without_fixed_from_ids(+ &self,+ liked_nodes: &[NodeID],+ disliked_nodes: &[NodeID],+ ) -> Scorer {Scorer::new(&self.proxy_nodes,- &liked_nodes,- &disliked_nodes,+ liked_nodes,+ disliked_nodes,HashMap::new(),&self.betweenness,)diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rsindex a484f5e3..4b1cd609 100644--- a/src/webgraph/centrality/harmonic.rs+++ b/src/webgraph/centrality/harmonic.rs@@ -51,11 +51,7 @@ where.map(|(other_node, dist)| (other_node, 1f64 / dist as f64)).collect();- let centrality = centrality_values- .into_iter()- .map(|(_, val)| val)- .sum::<f64>()- / norm_factor;+ let centrality = centrality_values.into_values().sum::<f64>() / norm_factor;(node, centrality)})diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex 6ee4ab42..d9c55320 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -31,7 +31,7 @@ use crate::kv::rocksdb_store::RocksDbStore;pub mod centrality;-type NodeID = u64;+pub type NodeID = u64;#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]pub(crate) struct StoredEdge {diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 420c383d..99d0850b 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -183,14 +183,14 @@ impl Webpage {schema.get_field(Field::Fast(FastField::Region).name()).expect("Failed to get region field"),- region.id() as u64,+ region.id(),);} else {doc.add_u64(schema.get_field(Field::Fast(FastField::Region).name()).expect("Failed to get region field"),- Region::All.id() as u64,+ Region::All.id(),);}diff --git a/src/webpage/region.rs b/src/webpage/region.rsindex 7ecdbfdd..edae215d 100644--- a/src/webpage/region.rs+++ b/src/webpage/region.rs@@ -156,7 +156,7 @@ impl RegionCount {}RegionCount {- total_counts: map.iter().map(|(_, count)| count).sum(),+ total_counts: map.values().sum(),map,fast_count,path: path.as_ref().to_str().unwrap().to_string(),@@ -173,7 +173,7 @@ impl RegionCount {let json = serde_json::to_string(&self.map).unwrap();let mut file = File::options().write(true).open(&self.path).unwrap();file.write_all(json.as_bytes()).unwrap();- self.total_counts = self.map.iter().map(|(_, count)| count).sum();+ self.total_counts = self.map.values().sum();let mut fast_count = Vec::new();