Skip to content

web search done right

boost final score using optics, not just tantivy bm25

Mikkel Denkerba6cdd4da10b
diff --git a/core/src/collector.rs b/core/src/collector.rs
index fec463a8..a9404e6e 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -463,10 +463,7 @@ mod tests {
collector.insert(SegmentDoc {
hashes: doc.0,
id: doc.1,
- score: Score {
- bm25: 0.0,
- total: doc.2,
- },
+ score: Score { total: doc.2 },
segment: 0,
});
}
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 43a5e235..d60acb4b 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -226,7 +226,6 @@ impl InvertedIndex {
if update_segment {
let segment_reader = ctx.tv_searcher.segment_reader(pointer.address.segment);
- let fastfield_reader = fastfield_reader.get_segment(&segment_reader.segment_id());
aggregator.register_segment(&ctx.tv_searcher, segment_reader, fastfield_reader)?;
}
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 960d0b76..542ac64f 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -199,6 +199,10 @@ impl Query {
&self.terms
}
+ pub fn optic(&self) -> Option<&Optic> {
+ self.optic.as_ref()
+ }
+
pub fn is_empty(&self) -> bool {
self.terms.is_empty()
}
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index ec26ac34..7fbf474f 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -17,11 +17,11 @@
use itertools::Itertools;
use optics::{Action, MatchLocation, Matching, Optic, Rule};
use tantivy::{
- query::{BooleanQuery, BoostQuery, Occur, QueryClone},
+ query::{BooleanQuery, Occur, QueryClone},
schema::Schema,
};
-use crate::{fastfield_reader::FastFieldReader, ranking::optics::SCALE, schema::TextField};
+use crate::{fastfield_reader::FastFieldReader, schema::TextField};
use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};
@@ -54,16 +54,10 @@ impl AsMultipleTantivyQuery for Optic {
self.rules
.iter()
.chain(self.site_rankings.rules().iter())
- .filter_map(|rule| {
- let queries = rule.as_multiple_tantivy(schema, fastfields);
-
- if queries.is_empty() {
- None
- } else {
- Some(queries)
- }
+ .filter_map(|rule| rule.as_searchable_rule(schema, fastfields))
+ .map(|(occur, rule)| {
+ BooleanQuery::from(vec![(occur, rule.query)]).box_clone()
})
- .map(|queries| BooleanQuery::from(queries).box_clone())
.collect_vec(),
)
.box_clone(),
@@ -72,18 +66,32 @@ impl AsMultipleTantivyQuery for Optic {
self.rules
.iter()
.chain(self.site_rankings.rules().iter())
- .filter_map(|rule| rule.as_multiple_tantivy(schema, fastfields).pop())
+ .filter_map(|rule| rule.as_searchable_rule(schema, fastfields))
+ .map(|(occur, rule)| (occur, rule.query))
.collect()
}
}
}
-impl AsMultipleTantivyQuery for Rule {
- fn as_multiple_tantivy(
+pub struct SearchableRule {
+ pub query: Box<dyn tantivy::query::Query>,
+ pub boost: f64,
+}
+
+pub trait AsSearchableRule {
+ fn as_searchable_rule(
&self,
schema: &Schema,
fastfield_reader: &FastFieldReader,
- ) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
+ ) -> Option<(Occur, SearchableRule)>;
+}
+
+impl AsSearchableRule for Rule {
+ fn as_searchable_rule(
+ &self,
+ schema: &Schema,
+ fastfield_reader: &FastFieldReader,
+ ) -> Option<(Occur, SearchableRule)> {
let mut subqueries: Vec<_> = self
.matches
.iter()
@@ -91,7 +99,7 @@ impl AsMultipleTantivyQuery for Rule {
.collect();
if subqueries.is_empty() {
- return vec![];
+ return None;
}
let subquery = if subqueries.len() == 1 {
@@ -101,23 +109,27 @@ impl AsMultipleTantivyQuery for Rule {
};
match &self.action {
- Action::Boost(boost) => vec![(
+ Action::Boost(boost) => Some((
Occur::Should,
- BoostQuery::new(
- ConstQuery::new(subquery, 1.0).box_clone(),
- *boost as f32 * SCALE,
- )
- .box_clone(),
- )],
- Action::Downrank(boost) => vec![(
+ SearchableRule {
+ query: ConstQuery::new(subquery, 1.0).box_clone(),
+ boost: *boost as f64,
+ },
+ )),
+ Action::Downrank(boost) => Some((
Occur::Should,
- BoostQuery::new(
- ConstQuery::new(subquery, 1.0).box_clone(),
- *boost as f32 * -SCALE,
- )
- .box_clone(),
- )],
- Action::Discard => vec![(Occur::MustNot, subquery)],
+ SearchableRule {
+ query: ConstQuery::new(subquery, 1.0).box_clone(),
+ boost: *boost as f64 * -1.0,
+ },
+ )),
+ Action::Discard => Some((
+ Occur::MustNot,
+ SearchableRule {
+ query: subquery,
+ boost: 0.0,
+ },
+ )),
}
}
}
diff --git a/core/src/ranking/initial.rs b/core/src/ranking/initial.rs
index 6117af2a..d360b339 100644
--- a/core/src/ranking/initial.rs
+++ b/core/src/ranking/initial.rs
@@ -28,6 +28,12 @@ pub(crate) struct InitialScoreTweaker {
aggregator: SignalAggregator,
}
+/// SAFETY:
+/// InitialScoreTweaker is thread-safe because it never mutates it's internal state.
+/// It only ever spawns InitialSegmentScoreTweakers which are not thread-safe.
+unsafe impl Sync for InitialScoreTweaker {}
+unsafe impl Send for InitialScoreTweaker {}
+
impl InitialScoreTweaker {
pub fn new(
tv_searcher: tantivy::Searcher,
@@ -42,46 +48,45 @@ impl InitialScoreTweaker {
}
}
-pub(crate) struct InitialSegmentScoreTweaker {
- aggregator: SignalAggregator,
-}
-
impl ScoreTweaker<Score> for InitialScoreTweaker {
type Child = InitialSegmentScoreTweaker;
fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {
let mut aggregator = self.aggregator.clone();
- let fastfield_segment_reader = self
- .fastfield_reader
- .get_segment(&segment_reader.segment_id());
+ let current_timestamp = Utc::now().timestamp() as usize;
+ aggregator.set_current_timestamp(current_timestamp);
aggregator
- .register_segment(&self.tv_searcher, segment_reader, fastfield_segment_reader)
+ .register_segment(&self.tv_searcher, segment_reader, &self.fastfield_reader)
.unwrap();
- let current_timestamp = Utc::now().timestamp() as usize;
- aggregator.set_current_timestamp(current_timestamp);
-
Ok(InitialSegmentScoreTweaker { aggregator })
}
}
+pub(crate) struct InitialSegmentScoreTweaker {
+ aggregator: SignalAggregator,
+}
+
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
pub struct Score {
- pub bm25: tantivy::Score,
pub total: f64,
}
impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {
- fn score(&mut self, doc: DocId, score: tantivy::Score) -> Score {
- let total = self
+ fn score(&mut self, doc: DocId, _score: tantivy::Score) -> Score {
+ let mut total = self
.aggregator
.compute_signals(doc)
.flatten()
.map(|computed| computed.coefficient * computed.value)
.sum();
- Score { bm25: score, total }
+ if let Some(boost) = self.aggregator.boosts(doc) {
+ total *= boost;
+ }
+
+ Score { total }
}
}
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 27aaec58..6f4d55cf 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -14,8 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-pub const SCALE: f32 = 10.0;
-
#[cfg(test)]
mod tests {
use optics::SiteRankings;
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 92fb92ce..caebf050 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -115,7 +115,12 @@ struct PrioritizeText {}
impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {
fn score(&self, websites: &mut [T]) {
for website in websites {
- let bm25 = website.as_ranking().pointer.score.bm25 as f64;
+ let bm25 = website
+ .as_ranking()
+ .signals
+ .get(Signal::Bm25)
+ .copied()
+ .unwrap_or(0.0);
website.as_mut_ranking().score += bm25;
}
}
@@ -287,10 +292,7 @@ mod tests {
.map(|i| -> RankingWebsite {
RankingWebsite {
pointer: WebsitePointer {
- score: Score {
- bm25: 0.0,
- total: 0.0,
- },
+ score: Score { total: 0.0 },
hashes: Hashes {
site: Prehashed(0),
title: Prehashed(0),
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 7e8dc1ef..fb4273ff 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::fastfield_reader::FieldValue;
+use crate::query::optic::AsSearchableRule;
use crate::query::Query;
use crate::Result;
use crate::{
@@ -31,6 +32,7 @@ use std::str::FromStr;
use std::sync::Arc;
use tantivy::fieldnorm::FieldNormReader;
use tantivy::postings::SegmentPostings;
+use tantivy::query::Scorer;
use thiserror::Error;
use tantivy::DocSet;
@@ -172,7 +174,7 @@ impl Signal {
Signal::Bm25StemmedCleanBody => 0.5,
Signal::Bm25AllBody => 1.0,
Signal::Bm25Url => 1.0,
- Signal::Bm25Site => 10.0,
+ Signal::Bm25Site => 1.0,
Signal::Bm25Domain => 1.0,
Signal::Bm25SiteNoTokenizer => 1.0,
Signal::Bm25DomainNoTokenizer => 1.0,
@@ -545,13 +547,17 @@ struct TextFieldData {
fieldnorm_reader: FieldNormReader,
}
-#[derive(Clone)]
+struct RuleBoost {
+ docset: Box<dyn Scorer>,
+ boost: f64,
+}
+
struct SegmentReader {
text_fields: EnumMap<TextField, TextFieldData>,
+ optic_rule_boosts: Vec<RuleBoost>,
fastfield_reader: Arc<fastfield_reader::SegmentReader>,
}
-#[derive(Clone)]
pub struct SignalAggregator {
query: Option<Query>,
signal_coefficients: SignalCoefficient,
@@ -567,6 +573,25 @@ pub struct SignalAggregator {
current_timestamp: Option<usize>,
}
+impl Clone for SignalAggregator {
+ fn clone(&self) -> Self {
+ Self {
+ query: self.query.clone(),
+ signal_coefficients: self.signal_coefficients.clone(),
+ segment_reader: None,
+ personal_centrality: self.personal_centrality.clone(),
+ inbound_similariy: self.inbound_similariy.clone(),
+ fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),
+ update_time_cache: self.update_time_cache.clone(),
+ topic_scorer: self.topic_scorer.clone(),
+ query_centrality: self.query_centrality.clone(),
+ region_count: self.region_count.clone(),
+ selected_region: self.selected_region,
+ current_timestamp: self.current_timestamp,
+ }
+ }
+}
+
impl std::fmt::Debug for SignalAggregator {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SignalAggregator")
@@ -610,10 +635,12 @@ impl SignalAggregator {
&mut self,
tv_searcher: &tantivy::Searcher,
segment_reader: &tantivy::SegmentReader,
- fastfield_reader: Arc<fastfield_reader::SegmentReader>,
+ fastfield_reader: &fastfield_reader::FastFieldReader,
) -> Result<()> {
+ let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());
let mut text_fields = EnumMap::new();
let schema = tv_searcher.schema();
+ let mut optic_rule_boosts = Vec::new();
if let Some(query) = &self.query {
if !query.simple_terms().is_empty() {
@@ -651,11 +678,31 @@ impl SignalAggregator {
}
}
}
+
+ if let Some(optic) = query.optic() {
+ optic_rule_boosts = optic
+ .rules
+ .iter()
+ .filter_map(|rule| {
+ rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader)
+ })
+ .map(|(_, rule)| RuleBoost {
+ docset: rule
+ .query
+ .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))
+ .unwrap()
+ .scorer(segment_reader, 0.0)
+ .unwrap(),
+ boost: rule.boost,
+ })
+ .collect();
+ }
}
self.segment_reader = Some(SegmentReader {
text_fields,
- fastfield_reader,
+ fastfield_reader: fastfield_segment_reader,
+ optic_rule_boosts,
});
Ok(())
@@ -724,6 +771,27 @@ impl SignalAggregator {
.map(move |signal| signal.compute(self, doc))
}
+ pub fn boosts(&mut self, doc: DocId) -> Option<f64> {
+ self.segment_reader.as_mut().and_then(|segment_reader| {
+ let mut res = 0.0;
+ for rule in &mut segment_reader.optic_rule_boosts {
+ if rule.docset.doc() > doc {
+ continue;
+ }
+
+ if rule.docset.doc() == doc || rule.docset.seek(doc) == doc {
+ res += rule.boost;
+ }
+ }
+
+ if res == 0.0 {
+ None
+ } else {
+ Some(res)
+ }
+ })
+ }
+
pub fn precompute_score(&self, webpage: &Webpage) -> f64 {
ALL_SIGNALS
.into_iter()