Skip to content

web search done right

also use inbound similarity during query centrality

Mikkel Denker00768ca3c4ad
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 3f749a40..d90b9c2e 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -22,6 +22,7 @@ pub mod initial;
pub mod models;
pub mod optics;
pub mod pipeline;
+pub mod query_centrality;
pub mod signal;
use std::collections::BTreeMap;
@@ -38,7 +39,7 @@ use crate::{
searcher::NUM_RESULTS_PER_PAGE,
webgraph::{
centrality::{
- online_harmonic::{self, OnlineHarmonicCentrality, Scorer},
+ online_harmonic::{OnlineHarmonicCentrality, Scorer},
topic,
},
Node, NodeID,
@@ -134,8 +135,8 @@ impl Ranker {
self.aggregator.set_topic_scorer(topic_scorer);
}
- pub fn set_query_centrality(&mut self, harmonic: online_harmonic::Scorer) {
- self.aggregator.set_query_centrality(harmonic);
+ pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {
+ self.aggregator.set_query_centrality(query_centrality);
}
}
diff --git a/core/src/ranking/query_centrality.rs b/core/src/ranking/query_centrality.rs
new file mode 100644
index 00000000..615fd920
--- /dev/null
+++ b/core/src/ranking/query_centrality.rs
@@ -0,0 +1,40 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::webgraph::{centrality::online_harmonic, NodeID};
+
+use super::inbound_similarity;
+
+pub struct Scorer {
+ online_harmonic: online_harmonic::Scorer,
+ inbound_centrality: inbound_similarity::Scorer,
+}
+
+impl Scorer {
+ pub fn new(
+ online_harmonic: online_harmonic::Scorer,
+ inbound_centrality: inbound_similarity::Scorer,
+ ) -> Self {
+ Self {
+ online_harmonic,
+ inbound_centrality,
+ }
+ }
+
+ pub fn score(&self, node: NodeID) -> f64 {
+ self.online_harmonic.score(node) + self.inbound_centrality.score(&node)
+ }
+}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 2eb4773c..8ef73088 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -47,8 +47,8 @@ use crate::{
};
use super::bm25::Bm25Weight;
-use super::inbound_similarity;
use super::models::linear::LinearRegression;
+use super::{inbound_similarity, query_centrality};
#[derive(Debug, Error)]
pub enum Error {
@@ -690,7 +690,7 @@ pub struct SignalAggregator {
fetch_time_ms_cache: Vec<f64>,
update_time_cache: Vec<f64>,
topic_scorer: Option<topic::Scorer>,
- query_centrality: Option<Arc<online_harmonic::Scorer>>,
+ query_centrality: Option<Arc<query_centrality::Scorer>>,
region_count: Option<Arc<RegionCount>>,
selected_region: Option<Region>,
current_timestamp: Option<usize>,
@@ -945,7 +945,7 @@ impl SignalAggregator {
self.topic_scorer = Some(topic_scorer);
}
- pub fn set_query_centrality(&mut self, query_centrality: online_harmonic::Scorer) {
+ pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {
self.query_centrality = Some(Arc::new(query_centrality));
}
@@ -1044,7 +1044,7 @@ impl SignalAggregator {
.sum()
}
- fn coefficient(&self, signal: &Signal) -> f64 {
+ pub fn coefficient(&self, signal: &Signal) -> f64 {
self.query_signal_coefficients
.as_ref()
.and_then(|coefficients| coefficients.get(signal))
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 2e1599c4..65d0a75f 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -29,7 +29,9 @@ use crate::ranking::centrality_store::SearchCentralityStore;
use crate::ranking::models::lambdamart::LambdaMART;
use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
-use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator, ALL_SIGNALS};
+use crate::ranking::{
+ online_centrality_scorer, query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS,
+};
use crate::schema::TextField;
use crate::search_ctx::Ctx;
use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};
@@ -116,6 +118,8 @@ impl LocalSearcher {
de_rank_similar: bool,
aggregator: SignalAggregator,
) -> Result<Ranker> {
+ let query_centrality_coeff = aggregator.coefficient(&Signal::QueryCentrality);
+
let mut ranker = Ranker::new(
aggregator,
self.index.inverted_index.fastfield_reader(&ctx.tv_searcher),
@@ -134,19 +138,28 @@ impl LocalSearcher {
ranker.de_rank_similar(de_rank_similar);
- if let Some(centrality_store) = self.centrality_store.as_ref() {
- ranker = ranker
- .with_max_docs(10_000, self.index.num_segments())
- .with_num_results(100);
-
- let top_host_nodes = self
- .index
- .top_nodes(query, ctx, ranker.collector(ctx.clone()))?;
- if !top_host_nodes.is_empty() {
- let harmonic = centrality_store
- .online_harmonic
- .scorer(&top_host_nodes, &[]);
- ranker.set_query_centrality(harmonic);
+ if query_centrality_coeff > 0.0 {
+ if let Some(centrality_store) = self.centrality_store.as_ref() {
+ ranker = ranker
+ .with_max_docs(10_000, self.index.num_segments())
+ .with_num_results(100);
+
+ let top_host_nodes =
+ self.index
+ .top_nodes(query, ctx, ranker.collector(ctx.clone()))?;
+ if !top_host_nodes.is_empty() {
+ let harmonic = centrality_store
+ .online_harmonic
+ .scorer(&top_host_nodes, &[]);
+
+ let inbound = centrality_store
+ .inbound_similarity
+ .scorer(&top_host_nodes, &[]);
+
+ let query_centrality = query_centrality::Scorer::new(harmonic, inbound);
+
+ ranker.set_query_centrality(query_centrality);
+ }
}
}