web search done right
boost final score using optics, not just tantivy bm25
Mikkel Denkerba6cdd4da10b
diff --git a/core/src/collector.rs b/core/src/collector.rsindex fec463a8..a9404e6e 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -463,10 +463,7 @@ mod tests {collector.insert(SegmentDoc {hashes: doc.0,id: doc.1,- score: Score {- bm25: 0.0,- total: doc.2,- },+ score: Score { total: doc.2 },segment: 0,});}diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 43a5e235..d60acb4b 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -226,7 +226,6 @@ impl InvertedIndex {if update_segment {let segment_reader = ctx.tv_searcher.segment_reader(pointer.address.segment);- let fastfield_reader = fastfield_reader.get_segment(&segment_reader.segment_id());aggregator.register_segment(&ctx.tv_searcher, segment_reader, fastfield_reader)?;}diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex 960d0b76..542ac64f 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -199,6 +199,10 @@ impl Query {&self.terms}+ pub fn optic(&self) -> Option<&Optic> {+ self.optic.as_ref()+ }+pub fn is_empty(&self) -> bool {self.terms.is_empty()}diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex ec26ac34..7fbf474f 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -17,11 +17,11 @@use itertools::Itertools;use optics::{Action, MatchLocation, Matching, Optic, Rule};use tantivy::{- query::{BooleanQuery, BoostQuery, Occur, QueryClone},+ query::{BooleanQuery, Occur, QueryClone},schema::Schema,};-use crate::{fastfield_reader::FastFieldReader, ranking::optics::SCALE, schema::TextField};+use crate::{fastfield_reader::FastFieldReader, schema::TextField};use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};@@ -54,16 +54,10 @@ impl AsMultipleTantivyQuery for Optic {self.rules.iter().chain(self.site_rankings.rules().iter())- .filter_map(|rule| {- let queries = rule.as_multiple_tantivy(schema, fastfields);-- if queries.is_empty() {- None- } else {- Some(queries)- }+ .filter_map(|rule| rule.as_searchable_rule(schema, fastfields))+ .map(|(occur, rule)| {+ BooleanQuery::from(vec![(occur, rule.query)]).box_clone()})- .map(|queries| BooleanQuery::from(queries).box_clone()).collect_vec(),).box_clone(),@@ -72,18 +66,32 @@ impl AsMultipleTantivyQuery for Optic {self.rules.iter().chain(self.site_rankings.rules().iter())- .filter_map(|rule| rule.as_multiple_tantivy(schema, fastfields).pop())+ .filter_map(|rule| rule.as_searchable_rule(schema, fastfields))+ .map(|(occur, rule)| (occur, rule.query)).collect()}}}-impl AsMultipleTantivyQuery for Rule {- fn as_multiple_tantivy(+pub struct SearchableRule {+ pub query: Box<dyn tantivy::query::Query>,+ pub boost: f64,+}++pub trait AsSearchableRule {+ fn as_searchable_rule(&self,schema: &Schema,fastfield_reader: &FastFieldReader,- ) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {+ ) -> Option<(Occur, SearchableRule)>;+}++impl AsSearchableRule for Rule {+ fn as_searchable_rule(+ &self,+ schema: &Schema,+ fastfield_reader: &FastFieldReader,+ ) -> Option<(Occur, SearchableRule)> {let mut subqueries: Vec<_> = self.matches.iter()@@ -91,7 +99,7 @@ impl AsMultipleTantivyQuery for Rule {.collect();if subqueries.is_empty() {- return vec![];+ return None;}let subquery = if subqueries.len() == 1 {@@ -101,23 +109,27 @@ impl AsMultipleTantivyQuery for Rule {};match &self.action {- Action::Boost(boost) => vec![(+ Action::Boost(boost) => Some((Occur::Should,- BoostQuery::new(- ConstQuery::new(subquery, 1.0).box_clone(),- *boost as f32 * SCALE,- )- .box_clone(),- )],- Action::Downrank(boost) => vec![(+ SearchableRule {+ query: ConstQuery::new(subquery, 1.0).box_clone(),+ boost: *boost as f64,+ },+ )),+ Action::Downrank(boost) => Some((Occur::Should,- BoostQuery::new(- ConstQuery::new(subquery, 1.0).box_clone(),- *boost as f32 * -SCALE,- )- .box_clone(),- )],- Action::Discard => vec![(Occur::MustNot, subquery)],+ SearchableRule {+ query: ConstQuery::new(subquery, 1.0).box_clone(),+ boost: *boost as f64 * -1.0,+ },+ )),+ Action::Discard => Some((+ Occur::MustNot,+ SearchableRule {+ query: subquery,+ boost: 0.0,+ },+ )),}}}diff --git a/core/src/ranking/initial.rs b/core/src/ranking/initial.rsindex 6117af2a..d360b339 100644--- a/core/src/ranking/initial.rs+++ b/core/src/ranking/initial.rs@@ -28,6 +28,12 @@ pub(crate) struct InitialScoreTweaker {aggregator: SignalAggregator,}+/// SAFETY:+/// InitialScoreTweaker is thread-safe because it never mutates it's internal state.+/// It only ever spawns InitialSegmentScoreTweakers which are not thread-safe.+unsafe impl Sync for InitialScoreTweaker {}+unsafe impl Send for InitialScoreTweaker {}+impl InitialScoreTweaker {pub fn new(tv_searcher: tantivy::Searcher,@@ -42,46 +48,45 @@ impl InitialScoreTweaker {}}-pub(crate) struct InitialSegmentScoreTweaker {- aggregator: SignalAggregator,-}-impl ScoreTweaker<Score> for InitialScoreTweaker {type Child = InitialSegmentScoreTweaker;fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {let mut aggregator = self.aggregator.clone();- let fastfield_segment_reader = self- .fastfield_reader- .get_segment(&segment_reader.segment_id());+ let current_timestamp = Utc::now().timestamp() as usize;+ aggregator.set_current_timestamp(current_timestamp);aggregator- .register_segment(&self.tv_searcher, segment_reader, fastfield_segment_reader)+ .register_segment(&self.tv_searcher, segment_reader, &self.fastfield_reader).unwrap();- let current_timestamp = Utc::now().timestamp() as usize;- aggregator.set_current_timestamp(current_timestamp);-Ok(InitialSegmentScoreTweaker { aggregator })}}+pub(crate) struct InitialSegmentScoreTweaker {+ aggregator: SignalAggregator,+}+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]pub struct Score {- pub bm25: tantivy::Score,pub total: f64,}impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {- fn score(&mut self, doc: DocId, score: tantivy::Score) -> Score {- let total = self+ fn score(&mut self, doc: DocId, _score: tantivy::Score) -> Score {+ let mut total = self.aggregator.compute_signals(doc).flatten().map(|computed| computed.coefficient * computed.value).sum();- Score { bm25: score, total }+ if let Some(boost) = self.aggregator.boosts(doc) {+ total *= boost;+ }++ Score { total }}}diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 27aaec58..6f4d55cf 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -14,8 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-pub const SCALE: f32 = 10.0;-#[cfg(test)]mod tests {use optics::SiteRankings;diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 92fb92ce..caebf050 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -115,7 +115,12 @@ struct PrioritizeText {}impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {fn score(&self, websites: &mut [T]) {for website in websites {- let bm25 = website.as_ranking().pointer.score.bm25 as f64;+ let bm25 = website+ .as_ranking()+ .signals+ .get(Signal::Bm25)+ .copied()+ .unwrap_or(0.0);website.as_mut_ranking().score += bm25;}}@@ -287,10 +292,7 @@ mod tests {.map(|i| -> RankingWebsite {RankingWebsite {pointer: WebsitePointer {- score: Score {- bm25: 0.0,- total: 0.0,- },+ score: Score { total: 0.0 },hashes: Hashes {site: Prehashed(0),title: Prehashed(0),diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 7e8dc1ef..fb4273ff 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::fastfield_reader::FieldValue;+use crate::query::optic::AsSearchableRule;use crate::query::Query;use crate::Result;use crate::{@@ -31,6 +32,7 @@ use std::str::FromStr;use std::sync::Arc;use tantivy::fieldnorm::FieldNormReader;use tantivy::postings::SegmentPostings;+use tantivy::query::Scorer;use thiserror::Error;use tantivy::DocSet;@@ -172,7 +174,7 @@ impl Signal {Signal::Bm25StemmedCleanBody => 0.5,Signal::Bm25AllBody => 1.0,Signal::Bm25Url => 1.0,- Signal::Bm25Site => 10.0,+ Signal::Bm25Site => 1.0,Signal::Bm25Domain => 1.0,Signal::Bm25SiteNoTokenizer => 1.0,Signal::Bm25DomainNoTokenizer => 1.0,@@ -545,13 +547,17 @@ struct TextFieldData {fieldnorm_reader: FieldNormReader,}-#[derive(Clone)]+struct RuleBoost {+ docset: Box<dyn Scorer>,+ boost: f64,+}+struct SegmentReader {text_fields: EnumMap<TextField, TextFieldData>,+ optic_rule_boosts: Vec<RuleBoost>,fastfield_reader: Arc<fastfield_reader::SegmentReader>,}-#[derive(Clone)]pub struct SignalAggregator {query: Option<Query>,signal_coefficients: SignalCoefficient,@@ -567,6 +573,25 @@ pub struct SignalAggregator {current_timestamp: Option<usize>,}+impl Clone for SignalAggregator {+ fn clone(&self) -> Self {+ Self {+ query: self.query.clone(),+ signal_coefficients: self.signal_coefficients.clone(),+ segment_reader: None,+ personal_centrality: self.personal_centrality.clone(),+ inbound_similariy: self.inbound_similariy.clone(),+ fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),+ update_time_cache: self.update_time_cache.clone(),+ topic_scorer: self.topic_scorer.clone(),+ query_centrality: self.query_centrality.clone(),+ region_count: self.region_count.clone(),+ selected_region: self.selected_region,+ current_timestamp: self.current_timestamp,+ }+ }+}+impl std::fmt::Debug for SignalAggregator {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {f.debug_struct("SignalAggregator")@@ -610,10 +635,12 @@ impl SignalAggregator {&mut self,tv_searcher: &tantivy::Searcher,segment_reader: &tantivy::SegmentReader,- fastfield_reader: Arc<fastfield_reader::SegmentReader>,+ fastfield_reader: &fastfield_reader::FastFieldReader,) -> Result<()> {+ let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());let mut text_fields = EnumMap::new();let schema = tv_searcher.schema();+ let mut optic_rule_boosts = Vec::new();if let Some(query) = &self.query {if !query.simple_terms().is_empty() {@@ -651,11 +678,31 @@ impl SignalAggregator {}}}++ if let Some(optic) = query.optic() {+ optic_rule_boosts = optic+ .rules+ .iter()+ .filter_map(|rule| {+ rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader)+ })+ .map(|(_, rule)| RuleBoost {+ docset: rule+ .query+ .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))+ .unwrap()+ .scorer(segment_reader, 0.0)+ .unwrap(),+ boost: rule.boost,+ })+ .collect();+ }}self.segment_reader = Some(SegmentReader {text_fields,- fastfield_reader,+ fastfield_reader: fastfield_segment_reader,+ optic_rule_boosts,});Ok(())@@ -724,6 +771,27 @@ impl SignalAggregator {.map(move |signal| signal.compute(self, doc))}+ pub fn boosts(&mut self, doc: DocId) -> Option<f64> {+ self.segment_reader.as_mut().and_then(|segment_reader| {+ let mut res = 0.0;+ for rule in &mut segment_reader.optic_rule_boosts {+ if rule.docset.doc() > doc {+ continue;+ }++ if rule.docset.doc() == doc || rule.docset.seek(doc) == doc {+ res += rule.boost;+ }+ }++ if res == 0.0 {+ None+ } else {+ Some(res)+ }+ })+ }+pub fn precompute_score(&self, webpage: &Webpage) -> f64 {ALL_SIGNALS.into_iter()