web search done right
also use inbound similarity during query centrality
Mikkel Denker00768ca3c4ad
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 3f749a40..d90b9c2e 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -22,6 +22,7 @@ pub mod initial;pub mod models;pub mod optics;pub mod pipeline;+pub mod query_centrality;pub mod signal;use std::collections::BTreeMap;@@ -38,7 +39,7 @@ use crate::{searcher::NUM_RESULTS_PER_PAGE,webgraph::{centrality::{- online_harmonic::{self, OnlineHarmonicCentrality, Scorer},+ online_harmonic::{OnlineHarmonicCentrality, Scorer},topic,},Node, NodeID,@@ -134,8 +135,8 @@ impl Ranker {self.aggregator.set_topic_scorer(topic_scorer);}- pub fn set_query_centrality(&mut self, harmonic: online_harmonic::Scorer) {- self.aggregator.set_query_centrality(harmonic);+ pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {+ self.aggregator.set_query_centrality(query_centrality);}}diff --git a/core/src/ranking/query_centrality.rs b/core/src/ranking/query_centrality.rsnew file mode 100644index 00000000..615fd920--- /dev/null+++ b/core/src/ranking/query_centrality.rs@@ -0,0 +1,40 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::webgraph::{centrality::online_harmonic, NodeID};++use super::inbound_similarity;++pub struct Scorer {+ online_harmonic: online_harmonic::Scorer,+ inbound_centrality: inbound_similarity::Scorer,+}++impl Scorer {+ pub fn new(+ online_harmonic: online_harmonic::Scorer,+ inbound_centrality: inbound_similarity::Scorer,+ ) -> Self {+ Self {+ online_harmonic,+ inbound_centrality,+ }+ }++ pub fn score(&self, node: NodeID) -> f64 {+ self.online_harmonic.score(node) + self.inbound_centrality.score(&node)+ }+}diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 2eb4773c..8ef73088 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -47,8 +47,8 @@ use crate::{};use super::bm25::Bm25Weight;-use super::inbound_similarity;use super::models::linear::LinearRegression;+use super::{inbound_similarity, query_centrality};#[derive(Debug, Error)]pub enum Error {@@ -690,7 +690,7 @@ pub struct SignalAggregator {fetch_time_ms_cache: Vec<f64>,update_time_cache: Vec<f64>,topic_scorer: Option<topic::Scorer>,- query_centrality: Option<Arc<online_harmonic::Scorer>>,+ query_centrality: Option<Arc<query_centrality::Scorer>>,region_count: Option<Arc<RegionCount>>,selected_region: Option<Region>,current_timestamp: Option<usize>,@@ -945,7 +945,7 @@ impl SignalAggregator {self.topic_scorer = Some(topic_scorer);}- pub fn set_query_centrality(&mut self, query_centrality: online_harmonic::Scorer) {+ pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {self.query_centrality = Some(Arc::new(query_centrality));}@@ -1044,7 +1044,7 @@ impl SignalAggregator {.sum()}- fn coefficient(&self, signal: &Signal) -> f64 {+ pub fn coefficient(&self, signal: &Signal) -> f64 {self.query_signal_coefficients.as_ref().and_then(|coefficients| coefficients.get(signal))diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 2e1599c4..65d0a75f 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -29,7 +29,9 @@ use crate::ranking::centrality_store::SearchCentralityStore;use crate::ranking::models::lambdamart::LambdaMART;use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};-use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator, ALL_SIGNALS};+use crate::ranking::{+ online_centrality_scorer, query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS,+};use crate::schema::TextField;use crate::search_ctx::Ctx;use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};@@ -116,6 +118,8 @@ impl LocalSearcher {de_rank_similar: bool,aggregator: SignalAggregator,) -> Result<Ranker> {+ let query_centrality_coeff = aggregator.coefficient(&Signal::QueryCentrality);+let mut ranker = Ranker::new(aggregator,self.index.inverted_index.fastfield_reader(&ctx.tv_searcher),@@ -134,19 +138,28 @@ impl LocalSearcher {ranker.de_rank_similar(de_rank_similar);- if let Some(centrality_store) = self.centrality_store.as_ref() {- ranker = ranker- .with_max_docs(10_000, self.index.num_segments())- .with_num_results(100);-- let top_host_nodes = self- .index- .top_nodes(query, ctx, ranker.collector(ctx.clone()))?;- if !top_host_nodes.is_empty() {- let harmonic = centrality_store- .online_harmonic- .scorer(&top_host_nodes, &[]);- ranker.set_query_centrality(harmonic);+ if query_centrality_coeff > 0.0 {+ if let Some(centrality_store) = self.centrality_store.as_ref() {+ ranker = ranker+ .with_max_docs(10_000, self.index.num_segments())+ .with_num_results(100);++ let top_host_nodes =+ self.index+ .top_nodes(query, ctx, ranker.collector(ctx.clone()))?;+ if !top_host_nodes.is_empty() {+ let harmonic = centrality_store+ .online_harmonic+ .scorer(&top_host_nodes, &[]);++ let inbound = centrality_store+ .inbound_similarity+ .scorer(&top_host_nodes, &[]);++ let query_centrality = query_centrality::Scorer::new(harmonic, inbound);++ ranker.set_query_centrality(query_centrality);+ }}}