Skip to content

web search done right

calculate query specific centrality during ranking

Mikkel Denker0d2fb2444286
diff --git a/configs/search_server.toml b/configs/search_server.toml
index 089bf194..c35d3057 100644
--- a/configs/search_server.toml
+++ b/configs/search_server.toml
@@ -1,4 +1,5 @@
host = "0.0.0.0:3001"
index_path = "data/index"
entity_index_path = "data/entity"
-bangs_path = "data/bangs.json"
\ No newline at end of file
+bangs_path = "data/bangs.json"
+centrality_store_path = "data/centrality"
\ No newline at end of file
diff --git a/src/human_website_annotations.rs b/src/human_website_annotations.rs
index 64eca81f..cb3ab7a7 100644
--- a/src/human_website_annotations.rs
+++ b/src/human_website_annotations.rs
@@ -104,13 +104,13 @@ impl Mapper {
}
pub fn all_topics(&self) -> HashSet<Topic> {
- self.0.iter().map(|(_, info)| info.topic.clone()).collect()
+ self.0.values().map(|info| info.topic.clone()).collect()
}
pub fn top_topics(&self, top_n: usize) -> Vec<Topic> {
let mut topics: HashMap<Topic, usize> = HashMap::new();
- for topic in self.0.iter().map(|(_, info)| info.topic.clone()) {
+ for topic in self.0.values().map(|info| info.topic.clone()) {
*topics.entry(topic).or_default() += 1;
}
diff --git a/src/index.rs b/src/index.rs
index 844a45b4..f7a06b8b 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::cmp::Ordering;
+use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::sync::Arc;
@@ -35,6 +36,7 @@ use crate::spell::{
Correction, CorrectionTerm, Dictionary, LogarithmicEdit, SpellChecker, TermSplitter,
};
use crate::subdomain_count::SubdomainCounter;
+use crate::webgraph::NodeID;
use crate::webpage::region::{Region, RegionCount};
use crate::webpage::{Url, Webpage};
use crate::Result;
@@ -134,6 +136,24 @@ impl Index {
self.inverted_index.search_initial(query, collector)
}
+ pub fn top_nodes<C>(&self, query: &Query, collector: C) -> Result<Vec<NodeID>>
+ where
+ C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,
+ {
+ let pointers = self
+ .inverted_index
+ .search_initial(query, collector)?
+ .top_websites;
+
+ let mut hosts = HashSet::with_capacity(pointers.len());
+ for pointer in &pointers {
+ if let Some(id) = self.inverted_index.website_host_node(pointer)? {
+ hosts.insert(id);
+ }
+ }
+ Ok(hosts.into_iter().collect())
+ }
+
pub fn retrieve_websites(
&self,
websites: &[inverted_index::WebsitePointer],
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index d93b020e..b17b0725 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -31,6 +31,7 @@ use crate::query::Query;
use crate::schema::{FastField, Field, TextField, ALL_FIELDS};
use crate::snippet;
use crate::tokenizer::Identity;
+use crate::webgraph::NodeID;
use crate::webpage::region::Region;
use crate::webpage::{StoredPrimaryImage, Webpage};
use crate::Result;
@@ -191,6 +192,22 @@ impl InvertedIndex {
})
}
+ pub fn website_host_node(&self, website: &WebsitePointer) -> Result<Option<NodeID>> {
+ let searcher = self.reader.searcher();
+ let doc = searcher.doc(website.address.into())?;
+ let field = self
+ .schema()
+ .get_field(Field::Fast(FastField::HostNodeID).name())
+ .unwrap();
+
+ let id = doc.get_first(field).unwrap().as_u64().unwrap();
+ if id == u64::MAX {
+ Ok(None)
+ } else {
+ Ok(Some(id))
+ }
+ }
+
pub fn retrieve_websites(
&self,
websites: &[WebsitePointer],
diff --git a/src/main.rs b/src/main.rs
index 36c679c9..445fbb41 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -131,7 +131,7 @@ fn main() -> Result<()> {
match args.command {
Commands::Indexer { options } => match options {
IndexingOptions::Master { config_path } => {
- let config = load_toml_config(&config_path);
+ let config = load_toml_config(config_path);
entrypoint::Indexer::run_master(&config)?;
}
IndexingOptions::Worker {
@@ -150,7 +150,7 @@ fn main() -> Result<()> {
)?;
}
IndexingOptions::Local { config_path } => {
- let config = load_toml_config(&config_path);
+ let config = load_toml_config(config_path);
entrypoint::Indexer::run_locally(&config)?;
}
IndexingOptions::Entity {
@@ -183,7 +183,7 @@ fn main() -> Result<()> {
}
},
Commands::Frontend { config_path } => {
- let config: FrontendConfig = load_toml_config(&config_path);
+ let config: FrontendConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -195,7 +195,7 @@ fn main() -> Result<()> {
))?
}
Commands::SearchServer { config_path } => {
- let config: SearchServerConfig = load_toml_config(&config_path);
+ let config: SearchServerConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index d4e22c5e..f29b1b9b 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -20,7 +20,7 @@ pub mod ast;
mod const_query;
mod pattern_query;
-use std::{collections::HashMap, convert::TryFrom, sync::Arc};
+use std::{collections::HashMap, convert::TryFrom};
use crate::{
query::union::UnionQuery,
@@ -257,9 +257,7 @@ impl Goggle {
);
if let Some(approx) = approx {
- aggregator
- .personal_harmonic
- .push(Arc::new(self.site_rankings.centrality_scorer(approx)));
+ aggregator.add_personal_harmonic(self.site_rankings.centrality_scorer(approx));
}
aggregator
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 2d205d36..e51f653a 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -31,7 +31,7 @@ use crate::{
fastfield_cache::FastFieldCache,
inverted_index,
searcher::NUM_RESULTS_PER_PAGE,
- webgraph::centrality::topic,
+ webgraph::centrality::{approximate_harmonic, topic},
webpage::region::{Region, RegionCount},
};
@@ -46,6 +46,7 @@ pub struct Ranker {
fastfield_cache: Arc<FastFieldCache>,
de_rank_similar: bool,
topic_scorer: Option<topic::Scorer>,
+ num_results: Option<usize>,
}
impl Ranker {
@@ -63,6 +64,7 @@ impl Ranker {
de_rank_similar: true,
fastfield_cache,
topic_scorer: None,
+ num_results: None,
}
}
@@ -84,6 +86,11 @@ impl Ranker {
self
}
+ pub fn with_num_results(mut self, num_results: usize) -> Self {
+ self.num_results = Some(num_results);
+ self
+ }
+
pub fn de_rank_similar(&mut self, de_rank_similar: bool) {
self.de_rank_similar = de_rank_similar;
}
@@ -92,7 +99,7 @@ impl Ranker {
let mut aggregator = self.aggregator.clone();
if let Some(topic_scorer) = self.topic_scorer.clone() {
- aggregator.register_topic_scorer(topic_scorer);
+ aggregator.set_topic_scorer(topic_scorer);
}
let score_tweaker = InitialScoreTweaker::new(
@@ -102,8 +109,10 @@ impl Ranker {
Arc::clone(&self.fastfield_cache),
);
- let mut collector =
- TopDocs::with_limit(NUM_RESULTS_PER_PAGE, Arc::clone(&self.fastfield_cache));
+ let mut collector = TopDocs::with_limit(
+ self.num_results.unwrap_or(NUM_RESULTS_PER_PAGE),
+ Arc::clone(&self.fastfield_cache),
+ );
if self.de_rank_similar {
collector = collector.and_de_rank_similar()
@@ -120,9 +129,13 @@ impl Ranker {
collector.tweak_score(score_tweaker)
}
- pub(crate) fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
+ pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
self.topic_scorer = Some(topic_scorer);
}
+
+ pub fn set_query_centrality(&mut self, approx: approximate_harmonic::Scorer) {
+ self.aggregator.set_query_centrality(approx);
+ }
}
#[cfg(test)]
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index 8dfa2041..4ed1561c 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -17,7 +17,10 @@
use crate::{
fastfield_cache,
schema::{FastField, TextField},
- webgraph::centrality::{approximate_harmonic, topic},
+ webgraph::centrality::{
+ approximate_harmonic::{self, SHIFT},
+ topic,
+ },
webpage::Webpage,
Result,
};
@@ -46,9 +49,10 @@ pub enum Signal {
PersonalCentrality,
CrawlStability,
TopicCentrality,
+ QueryCentrality,
}
-pub const ALL_SIGNALS: [Signal; 11] = [
+pub const ALL_SIGNALS: [Signal; 12] = [
Signal::Bm25,
Signal::HostCentrality,
Signal::PageCentrality,
@@ -60,6 +64,7 @@ pub const ALL_SIGNALS: [Signal; 11] = [
Signal::PersonalCentrality,
Signal::CrawlStability,
Signal::TopicCentrality,
+ Signal::QueryCentrality,
];
struct SignalValue {
@@ -69,6 +74,7 @@ struct SignalValue {
selected_region: Option<Region>,
personal_centrality: f64,
topic_score: Option<f64>,
+ query_centrality: Option<f64>,
}
impl Signal {
@@ -133,6 +139,7 @@ impl Signal {
}
Signal::CrawlStability => value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64,
Signal::TopicCentrality => value.topic_score.unwrap_or_default(),
+ Signal::QueryCentrality => value.query_centrality.unwrap_or_default(),
}
}
@@ -142,12 +149,13 @@ impl Signal {
Signal::HostCentrality => 2500.0,
Signal::PageCentrality => 4500.0,
Signal::TopicCentrality => 2500.0,
+ Signal::PersonalCentrality => 4500.0,
+ Signal::QueryCentrality => 2500.0,
Signal::IsHomepage => 0.1,
Signal::FetchTimeMs => 0.01,
Signal::UpdateTimestamp => 80.0,
Signal::NumTrackers => 20.0,
Signal::Region => 60.0,
- Signal::PersonalCentrality => 2500.0,
Signal::CrawlStability => 20.0,
}
}
@@ -164,6 +172,7 @@ impl Signal {
"region" => Some(Signal::Region),
"personal_centrality" => Some(Signal::PersonalCentrality),
"topic_centrality" => Some(Signal::TopicCentrality),
+ "query_centrality" => Some(Signal::QueryCentrality),
_ => None,
}
}
@@ -181,6 +190,7 @@ impl Signal {
Signal::CrawlStability => Some(FastField::CrawlStability),
Signal::PersonalCentrality => None,
Signal::TopicCentrality => None,
+ Signal::QueryCentrality => None,
}
}
}
@@ -248,11 +258,12 @@ impl FieldBoost {
pub struct SignalAggregator {
fastfield_cache: Option<Arc<fastfield_cache::SegmentCache>>,
signal_coefficients: SignalCoefficient,
- pub personal_harmonic: Vec<Arc<approximate_harmonic::Scorer>>,
+ personal_centrality: Vec<Arc<approximate_harmonic::Scorer>>,
field_boost: FieldBoost,
fetch_time_ms_cache: [f64; 1000],
update_time_cache: Vec<f64>,
topic_scorer: Option<topic::Scorer>,
+ query_centrality: Option<Arc<approximate_harmonic::Scorer>>,
}
impl std::fmt::Debug for SignalAggregator {
@@ -286,12 +297,13 @@ impl SignalAggregator {
Self {
fastfield_cache: None,
- personal_harmonic: Vec::new(),
+ personal_centrality: Vec::new(),
signal_coefficients,
field_boost,
fetch_time_ms_cache,
update_time_cache,
topic_scorer: None,
+ query_centrality: None,
}
}
@@ -299,10 +311,18 @@ impl SignalAggregator {
self.fastfield_cache = Some(cache);
}
- pub fn register_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
+ pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
self.topic_scorer = Some(topic_scorer);
}
+ pub fn set_query_centrality(&mut self, query_centrality: approximate_harmonic::Scorer) {
+ self.query_centrality = Some(Arc::new(query_centrality));
+ }
+
+ pub fn add_personal_harmonic(&mut self, personal_centrality: approximate_harmonic::Scorer) {
+ self.personal_centrality.push(Arc::new(personal_centrality))
+ }
+
pub fn score(
&self,
doc: DocId,
@@ -311,20 +331,20 @@ impl SignalAggregator {
current_timestamp: usize,
selected_region: Option<Region>,
) -> f64 {
- let topic_score = self.topic_scorer.as_ref().and_then(|scorer| {
- self.fastfield_cache.as_ref().and_then(|cache| {
- cache
- .get_doc_cache(&FastField::HostNodeID)
- .get_u64(&doc)
- .and_then(|host| {
- if host == u64::MAX {
- None
- } else {
- Some(scorer.score(host))
- }
- })
- })
- });
+ let host_id = self
+ .fastfield_cache
+ .as_ref()
+ .and_then(|cache| cache.get_doc_cache(&FastField::HostNodeID).get_u64(&doc));
+
+ let topic_score = self
+ .topic_scorer
+ .as_ref()
+ .and_then(|scorer| host_id.map(|host| scorer.score(host)));
+
+ let query_centrality = self
+ .query_centrality
+ .as_ref()
+ .and_then(|scorer| host_id.map(|host| scorer.score(host) - SHIFT));
ALL_SIGNALS
.into_iter()
@@ -336,7 +356,7 @@ impl SignalAggregator {
});
let personal_centrality = self
- .personal_harmonic
+ .personal_centrality
.iter()
.map(|scorer| scorer.score(doc as u64))
.sum();
@@ -352,6 +372,7 @@ impl SignalAggregator {
selected_region,
personal_centrality,
topic_score,
+ query_centrality,
},
)
})
@@ -382,7 +403,10 @@ impl SignalAggregator {
Signal::CrawlStability => {
(webpage.crawl_stability * (FLOAT_SCALING as f64)) as u64
}
- Signal::Bm25 | Signal::PersonalCentrality | Signal::TopicCentrality => {
+ Signal::Bm25
+ | Signal::PersonalCentrality
+ | Signal::TopicCentrality
+ | Signal::QueryCentrality => {
panic!("signal cannot be determined from webpage")
}
};
@@ -400,6 +424,7 @@ impl SignalAggregator {
selected_region: None,
personal_centrality: 0.0,
topic_score: None,
+ query_centrality: None,
},
)
})
diff --git a/src/searcher/local.rs b/src/searcher/local.rs
index 74e2f439..0e468f2c 100644
--- a/src/searcher/local.rs
+++ b/src/searcher/local.rs
@@ -142,10 +142,6 @@ impl LocalSearcher {
self.index.inverted_index.fastfield_cache(),
);
- if let Some(skip_pages) = query.skip_pages {
- ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);
- }
-
if let Some(region) = query.selected_region {
if region != Region::All {
ranker = ranker.with_region(region);
@@ -157,9 +153,30 @@ impl LocalSearcher {
ranker.set_topic_scorer(topic_scorer);
}
- ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());
ranker.de_rank_similar(de_rank_similar);
+ if let Some(centrality_store) = self.centrality_store.as_ref() {
+ ranker = ranker
+ .with_max_docs(10_000, self.index.num_segments())
+ .with_num_results(100);
+
+ let top_host_nodes = self.index.top_nodes(&parsed_query, ranker.collector())?;
+ if !top_host_nodes.is_empty() {
+ let approx = centrality_store
+ .approx_harmonic
+ .scorer_without_fixed_from_ids(&top_host_nodes, &[]);
+ ranker.set_query_centrality(approx);
+ }
+ }
+
+ ranker = ranker
+ .with_max_docs(10_000_000, self.index.num_segments())
+ .with_num_results(NUM_RESULTS_PER_PAGE);
+
+ if let Some(skip_pages) = query.skip_pages {
+ ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);
+ }
+
let webpages = self
.index
.search_initial(&parsed_query, ranker.collector())?;
diff --git a/src/spell/distance.rs b/src/spell/distance.rs
index 5317f439..f294f218 100644
--- a/src/spell/distance.rs
+++ b/src/spell/distance.rs
@@ -77,7 +77,7 @@ impl LevenshteinDistance {
// insertion
cur[j] + 1,
// match or substitution
- pre + if ca == cb { 0 } else { 1 },
+ pre + usize::from(ca != cb),
),
);
pre = tmp;
diff --git a/src/spell/spell_checker.rs b/src/spell/spell_checker.rs
index a826a613..ef413f0d 100644
--- a/src/spell/spell_checker.rs
+++ b/src/spell/spell_checker.rs
@@ -62,15 +62,10 @@ impl<'a, T: EditStrategy, const DICT_N: usize> SpellChecker<'a, T, DICT_N> {
}
pub fn correct(&self, term: &str) -> Option<String> {
- if let Some(correction) = self.correct_top(term, 1).into_iter().next() {
- if correction.to_ascii_lowercase() == term.to_ascii_lowercase() {
- None
- } else {
- Some(correction)
- }
- } else {
- None
- }
+ self.correct_top(term, 1)
+ .into_iter()
+ .next()
+ .filter(|correction| correction.to_ascii_lowercase() != term.to_ascii_lowercase())
}
}
diff --git a/src/webgraph/centrality/approximate_harmonic.rs b/src/webgraph/centrality/approximate_harmonic.rs
index c727d1bf..2aba833e 100644
--- a/src/webgraph/centrality/approximate_harmonic.rs
+++ b/src/webgraph/centrality/approximate_harmonic.rs
@@ -340,10 +340,18 @@ impl ApproximatedHarmonicCentrality {
.filter_map(|node| self.node2id.get(&node).copied())
.collect_vec();
+ self.scorer_without_fixed_from_ids(&liked_nodes, &disliked_nodes)
+ }
+
+ pub fn scorer_without_fixed_from_ids(
+ &self,
+ liked_nodes: &[NodeID],
+ disliked_nodes: &[NodeID],
+ ) -> Scorer {
Scorer::new(
&self.proxy_nodes,
- &liked_nodes,
- &disliked_nodes,
+ liked_nodes,
+ disliked_nodes,
HashMap::new(),
&self.betweenness,
)
diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rs
index a484f5e3..4b1cd609 100644
--- a/src/webgraph/centrality/harmonic.rs
+++ b/src/webgraph/centrality/harmonic.rs
@@ -51,11 +51,7 @@ where
.map(|(other_node, dist)| (other_node, 1f64 / dist as f64))
.collect();
- let centrality = centrality_values
- .into_iter()
- .map(|(_, val)| val)
- .sum::<f64>()
- / norm_factor;
+ let centrality = centrality_values.into_values().sum::<f64>() / norm_factor;
(node, centrality)
})
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
index 6ee4ab42..d9c55320 100644
--- a/src/webgraph/mod.rs
+++ b/src/webgraph/mod.rs
@@ -31,7 +31,7 @@ use crate::kv::rocksdb_store::RocksDbStore;
pub mod centrality;
-type NodeID = u64;
+pub type NodeID = u64;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub(crate) struct StoredEdge {
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 420c383d..99d0850b 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -183,14 +183,14 @@ impl Webpage {
schema
.get_field(Field::Fast(FastField::Region).name())
.expect("Failed to get region field"),
- region.id() as u64,
+ region.id(),
);
} else {
doc.add_u64(
schema
.get_field(Field::Fast(FastField::Region).name())
.expect("Failed to get region field"),
- Region::All.id() as u64,
+ Region::All.id(),
);
}
diff --git a/src/webpage/region.rs b/src/webpage/region.rs
index 7ecdbfdd..edae215d 100644
--- a/src/webpage/region.rs
+++ b/src/webpage/region.rs
@@ -156,7 +156,7 @@ impl RegionCount {
}
RegionCount {
- total_counts: map.iter().map(|(_, count)| count).sum(),
+ total_counts: map.values().sum(),
map,
fast_count,
path: path.as_ref().to_str().unwrap().to_string(),
@@ -173,7 +173,7 @@ impl RegionCount {
let json = serde_json::to_string(&self.map).unwrap();
let mut file = File::options().write(true).open(&self.path).unwrap();
file.write_all(json.as_bytes()).unwrap();
- self.total_counts = self.map.iter().map(|(_, count)| count).sum();
+ self.total_counts = self.map.values().sum();
let mut fast_count = Vec::new();