Skip to content

web search done right

get more detailed score from collector and fixed disalignment between some frontend and backend terminology

Mikkel Denker453aca402e43
diff --git a/core/src/collector.rs b/core/src/collector.rs
index 1595e3a4..9d8fc14f 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -20,13 +20,14 @@ use min_max_heap::MinMaxHeap;
use serde::{Deserialize, Serialize};
use tantivy::{
collector::{Collector, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector},
- DocId, Score, SegmentOrdinal, SegmentReader,
+ DocId, SegmentOrdinal, SegmentReader,
};
use crate::{
fastfield_cache,
inverted_index::{DocAddress, WebsitePointer},
prehashed::{combine_u64s, Prehashed},
+ ranking::initial::Score,
schema::FastField,
simhash,
};
@@ -42,12 +43,12 @@ pub struct MaxDocsConsidered {
pub segments: usize,
}
-#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
pub struct Hashes {
- site: Prehashed,
- title: Prehashed,
- url: Prehashed,
- simhash: simhash::HashType,
+ pub site: Prehashed,
+ pub title: Prehashed,
+ pub url: Prehashed,
+ pub simhash: simhash::HashType,
}
pub trait Doc: Clone {
@@ -97,23 +98,19 @@ impl TopDocs {
score_tweaker: TScoreTweaker,
) -> impl Collector<Fruit = Vec<WebsitePointer>>
where
- TScoreSegmentTweaker: ScoreSegmentTweaker<f64> + 'static,
- TScoreTweaker: ScoreTweaker<f64, Child = TScoreSegmentTweaker> + Send + Sync,
+ TScoreSegmentTweaker: ScoreSegmentTweaker<Score> + 'static,
+ TScoreTweaker: ScoreTweaker<Score, Child = TScoreSegmentTweaker> + Send + Sync,
{
TweakedScoreTopCollector::new(score_tweaker, self)
}
}
-impl Collector for TopDocs {
- type Fruit = Vec<WebsitePointer>;
-
- type Child = TopSegmentCollector;
-
+impl TopDocs {
fn for_segment(
&self,
segment_local_id: tantivy::SegmentOrdinal,
segment: &tantivy::SegmentReader,
- ) -> tantivy::Result<Self::Child> {
+ ) -> tantivy::Result<TopSegmentCollector> {
let max_docs = self
.max_docs
.as_ref()
@@ -127,37 +124,6 @@ impl Collector for TopDocs {
bucket_collector: BucketCollector::new(self.top_n + self.offset),
})
}
-
- fn requires_scoring(&self) -> bool {
- true
- }
-
- fn merge_fruits(
- &self,
- segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
- ) -> tantivy::Result<Self::Fruit> {
- let mut collector = BucketCollector::new(self.top_n + self.offset);
-
- for docs in segment_fruits {
- for doc in docs {
- collector.insert(doc);
- }
- }
-
- Ok(collector
- .into_sorted_vec(self.de_rank_similar)
- .into_iter()
- .skip(self.offset)
- .map(|doc| WebsitePointer {
- score: doc.score,
- hashes: doc.hashes,
- address: DocAddress {
- segment: doc.segment,
- doc_id: doc.id,
- },
- })
- .collect())
- }
}
pub struct TopSegmentCollector {
@@ -183,9 +149,7 @@ impl TopSegmentCollector {
}
}
-impl SegmentCollector for TopSegmentCollector {
- type Fruit = Vec<SegmentDoc>;
-
+impl TopSegmentCollector {
fn collect(&mut self, doc: DocId, score: Score) {
if let Some(max_docs) = &self.max_docs {
if self.num_docs_taken >= *max_docs {
@@ -208,11 +172,11 @@ impl SegmentCollector for TopSegmentCollector {
},
id: doc,
segment: self.segment_ord,
- score: score as f64,
+ score,
});
}
- fn harvest(self) -> Self::Fruit {
+ fn harvest(self) -> Vec<SegmentDoc> {
self.bucket_collector.into_sorted_vec(false)
}
}
@@ -364,12 +328,12 @@ pub struct SegmentDoc {
hashes: Hashes,
id: DocId,
segment: SegmentOrdinal,
- score: f64,
+ score: Score,
}
impl Doc for SegmentDoc {
fn score(&self) -> &f64 {
- &self.score
+ &self.score.total
}
fn id(&self) -> &DocId {
@@ -383,24 +347,24 @@ impl Doc for SegmentDoc {
pub(crate) struct TweakedScoreTopCollector<TScoreTweaker> {
score_tweaker: TScoreTweaker,
- collector: TopDocs,
+ top_docs: TopDocs,
}
impl<TScoreTweaker> TweakedScoreTopCollector<TScoreTweaker> {
pub fn new(
score_tweaker: TScoreTweaker,
- collector: TopDocs,
+ top_docs: TopDocs,
) -> TweakedScoreTopCollector<TScoreTweaker> {
TweakedScoreTopCollector {
score_tweaker,
- collector,
+ top_docs,
}
}
}
impl<TScoreTweaker> Collector for TweakedScoreTopCollector<TScoreTweaker>
where
- TScoreTweaker: ScoreTweaker<f64> + Send + Sync,
+ TScoreTweaker: ScoreTweaker<Score> + Send + Sync,
{
type Fruit = Vec<WebsitePointer>;
@@ -414,7 +378,7 @@ where
let segment_scorer = self.score_tweaker.segment_tweaker(segment_reader)?;
let segment_collector = self
- .collector
+ .top_docs
.for_segment(segment_local_id, segment_reader)?;
Ok(TopTweakedScoreSegmentCollector {
@@ -431,13 +395,33 @@ where
&self,
segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
) -> tantivy::Result<Self::Fruit> {
- self.collector.merge_fruits(segment_fruits)
+ let mut collector = BucketCollector::new(self.top_docs.top_n + self.top_docs.offset);
+
+ for docs in segment_fruits {
+ for doc in docs {
+ collector.insert(doc);
+ }
+ }
+
+ Ok(collector
+ .into_sorted_vec(self.top_docs.de_rank_similar)
+ .into_iter()
+ .skip(self.top_docs.offset)
+ .map(|doc| WebsitePointer {
+ score: doc.score,
+ hashes: doc.hashes,
+ address: DocAddress {
+ segment: doc.segment,
+ doc_id: doc.id,
+ },
+ })
+ .collect())
}
}
pub struct TopTweakedScoreSegmentCollector<TSegmentScoreTweaker>
where
- TSegmentScoreTweaker: ScoreSegmentTweaker<f64>,
+ TSegmentScoreTweaker: ScoreSegmentTweaker<Score>,
{
segment_collector: TopSegmentCollector,
segment_scorer: TSegmentScoreTweaker,
@@ -446,13 +430,13 @@ where
impl<TSegmentScoreTweaker> SegmentCollector
for TopTweakedScoreSegmentCollector<TSegmentScoreTweaker>
where
- TSegmentScoreTweaker: 'static + ScoreSegmentTweaker<f64>,
+ TSegmentScoreTweaker: 'static + ScoreSegmentTweaker<Score>,
{
type Fruit = Vec<SegmentDoc>;
- fn collect(&mut self, doc: DocId, score: Score) {
+ fn collect(&mut self, doc: DocId, score: tantivy::Score) {
let score = self.segment_scorer.score(doc, score);
- self.segment_collector.collect(doc, score as f32);
+ self.segment_collector.collect(doc, score);
}
fn harvest(self) -> Self::Fruit {
@@ -471,7 +455,10 @@ mod tests {
collector.insert(SegmentDoc {
hashes: doc.0,
id: doc.1,
- score: doc.2,
+ score: Score {
+ bm25: 0.0,
+ total: doc.2,
+ },
segment: 0,
});
}
@@ -479,7 +466,7 @@ mod tests {
let res: Vec<(f64, DocId)> = collector
.into_sorted_vec(true)
.into_iter()
- .map(|doc| (doc.score, doc.id))
+ .map(|doc| (doc.score.total, doc.id))
.collect();
assert_eq!(&res, expected);
diff --git a/core/src/index.rs b/core/src/index.rs
index f36be280..8784cc16 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -133,14 +133,14 @@ impl Index {
where
C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,
{
- let pointers = self
+ let websites = self
.inverted_index
.search_initial(query, collector)?
.top_websites;
- let mut hosts = HashSet::with_capacity(pointers.len());
- for pointer in &pointers {
- if let Some(id) = self.inverted_index.website_host_node(pointer)? {
+ let mut hosts = HashSet::with_capacity(websites.len());
+ for website in &websites {
+ if let Some(id) = self.inverted_index.website_host_node(website)? {
hosts.insert(id);
}
}
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 8559128b..5ddbe4c0 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -28,6 +28,9 @@ use crate::fastfield_cache::FastFieldCache;
use crate::human_website_annotations::Topic;
use crate::image_store::Image;
use crate::query::Query;
+use crate::ranking::initial::Score;
+use crate::ranking::pipeline::RankingWebsite;
+use crate::ranking::SignalAggregator;
use crate::schema::{FastField, Field, TextField, ALL_FIELDS};
use crate::snippet;
use crate::tokenizer::Identity;
@@ -47,14 +50,14 @@ pub struct InitialSearchResult {
pub top_websites: Vec<WebsitePointer>,
}
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
pub struct WebsitePointer {
- pub score: f64,
+ pub score: Score,
pub hashes: Hashes,
pub address: DocAddress,
}
-#[derive(Debug, Serialize, Deserialize, Clone, Copy)]
+#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq)]
pub struct DocAddress {
pub segment: u32,
pub doc_id: u32,
@@ -185,13 +188,30 @@ impl InvertedIndex {
{
let searcher = self.reader.searcher();
- let (count, docs) = searcher.search(query, &(Count, collector))?;
+ let (count, pointers) = searcher.search(query, &(Count, collector))?;
+
Ok(InitialSearchResult {
num_websites: count,
- top_websites: docs,
+ top_websites: pointers,
})
}
+ pub fn retrieve_ranking_websites(
+ &self,
+ pointers: Vec<WebsitePointer>,
+ aggregator: &SignalAggregator,
+ ) -> Result<Vec<RankingWebsite>> {
+ let searcher = self.reader.searcher();
+ let mut top_websites = Vec::new();
+
+ for pointer in pointers {
+ let doc = searcher.doc(pointer.address.into())?;
+ top_websites.push(RankingWebsite::new(doc, pointer, aggregator))
+ }
+
+ Ok(top_websites)
+ }
+
pub fn website_host_node(&self, website: &WebsitePointer) -> Result<Option<NodeID>> {
let searcher = self.reader.searcher();
let doc = searcher.doc(website.address.into())?;
@@ -527,7 +547,10 @@ mod tests {
C: Collector<Fruit = Vec<WebsitePointer>>,
{
let initial_result = index.search_initial(query, collector)?;
- let websites = index.retrieve_websites(&initial_result.top_websites, query)?;
+
+ let pointers: Vec<_> = initial_result.top_websites;
+
+ let websites = index.retrieve_websites(&pointers, query)?;
Ok(SearchResult {
num_docs: initial_result.num_websites,
diff --git a/core/src/ranking/initial.rs b/core/src/ranking/initial.rs
index 838e3065..ba5c3d2a 100644
--- a/core/src/ranking/initial.rs
+++ b/core/src/ranking/initial.rs
@@ -19,8 +19,9 @@ use std::sync::Arc;
use crate::fastfield_cache::FastFieldCache;
use crate::webpage::region::{Region, RegionCount};
use chrono::Utc;
+use serde::{Deserialize, Serialize};
use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};
-use tantivy::{DocId, Score, SegmentReader};
+use tantivy::{DocId, SegmentReader};
use super::SignalAggregator;
@@ -54,7 +55,7 @@ pub(crate) struct InitialSegmentScoreTweaker {
selected_region: Option<Region>,
}
-impl ScoreTweaker<f64> for InitialScoreTweaker {
+impl ScoreTweaker<Score> for InitialScoreTweaker {
type Child = InitialSegmentScoreTweaker;
fn segment_tweaker(&self, segment_reader: &SegmentReader) -> tantivy::Result<Self::Child> {
@@ -77,8 +78,14 @@ impl ScoreTweaker<f64> for InitialScoreTweaker {
}
}
-impl ScoreSegmentTweaker<f64> for InitialSegmentScoreTweaker {
- fn score(&mut self, doc: DocId, score: Score) -> f64 {
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+pub struct Score {
+ pub bm25: tantivy::Score,
+ pub total: f64,
+}
+
+impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {
+ fn score(&mut self, doc: DocId, score: tantivy::Score) -> Score {
self.aggregator.score(
doc,
score,
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 44178f20..eeea88e6 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -45,7 +45,6 @@ pub struct Ranker {
aggregator: SignalAggregator,
fastfield_cache: Arc<FastFieldCache>,
de_rank_similar: bool,
- topic_scorer: Option<topic::Scorer>,
num_results: Option<usize>,
}
@@ -63,7 +62,6 @@ impl Ranker {
max_docs: None,
de_rank_similar: true,
fastfield_cache,
- topic_scorer: None,
num_results: None,
}
}
@@ -95,12 +93,12 @@ impl Ranker {
self.de_rank_similar = de_rank_similar;
}
- pub fn collector(&self) -> impl Collector<Fruit = Vec<inverted_index::WebsitePointer>> {
- let mut aggregator = self.aggregator.clone();
+ pub fn aggregator(&self) -> SignalAggregator {
+ self.aggregator.clone()
+ }
- if let Some(topic_scorer) = self.topic_scorer.clone() {
- aggregator.set_topic_scorer(topic_scorer);
- }
+ pub fn collector(&self) -> impl Collector<Fruit = Vec<inverted_index::WebsitePointer>> {
+ let aggregator = self.aggregator();
let score_tweaker = InitialScoreTweaker::new(
Arc::clone(&self.region_count),
@@ -130,7 +128,7 @@ impl Ranker {
}
pub fn set_topic_scorer(&mut self, topic_scorer: topic::Scorer) {
- self.topic_scorer = Some(topic_scorer);
+ self.aggregator.set_topic_scorer(topic_scorer);
}
pub fn set_query_centrality(&mut self, approx: approximate_harmonic::Scorer) {
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index ac64b34d..3bd9b99c 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -16,18 +16,75 @@
use std::cmp::Ordering;
-#[derive(Clone, Debug)]
-struct RankingWebsite {
- address: tantivy::DocAddress,
- bm25: f64,
- host_centrality: f64,
- page_centrality: f64,
- topic_centrality: f64,
- personal_centrality: f64,
- query_centrality: f64,
- title: String,
- clean_body: String,
- score: f64,
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ inverted_index::WebsitePointer,
+ schema::{FastField, Field, TextField, ALL_FIELDS, FLOAT_SCALING},
+};
+
+use super::SignalAggregator;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct RankingWebsite {
+ pub pointer: WebsitePointer,
+ pub host_centrality: f64,
+ pub page_centrality: f64,
+ pub topic_centrality: f64,
+ pub personal_centrality: f64,
+ pub query_centrality: f64,
+ pub title: String,
+ pub clean_body: String,
+ pub score: f64,
+}
+
+impl RankingWebsite {
+ pub fn new(
+ doc: tantivy::Document,
+ pointer: WebsitePointer,
+ aggregator: &SignalAggregator,
+ ) -> Self {
+ let mut res = RankingWebsite {
+ pointer,
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ topic_centrality: 0.0,
+ personal_centrality: 0.0,
+ query_centrality: 0.0,
+ title: String::new(),
+ clean_body: String::new(),
+ score: 0.0,
+ };
+
+ for value in doc.field_values() {
+ match ALL_FIELDS[value.field().field_id() as usize] {
+ Field::Fast(FastField::HostCentrality) => {
+ res.host_centrality =
+ value.value.as_u64().unwrap() as f64 / FLOAT_SCALING as f64
+ }
+ Field::Fast(FastField::PageCentrality) => {
+ res.page_centrality =
+ value.value.as_u64().unwrap() as f64 / FLOAT_SCALING as f64
+ }
+ Field::Fast(FastField::HostNodeID) => {
+ let node = value.value.as_u64().unwrap();
+
+ res.personal_centrality = dbg!(aggregator.personal_centrality(node));
+ res.topic_centrality = aggregator.topic_centrality(node).unwrap_or_default();
+ res.query_centrality = aggregator.query_centrality(node).unwrap_or_default();
+ }
+ Field::Text(TextField::Title) => {
+ res.title = value.value.as_text().unwrap().to_string()
+ }
+ Field::Text(TextField::CleanBody) => {
+ res.clean_body = value.value.as_text().unwrap().to_string()
+ }
+ _ => {}
+ }
+ }
+
+ res
+ }
}
trait Scorer {
@@ -131,17 +188,33 @@ impl Pipeline {
#[cfg(test)]
mod tests {
+ use crate::{
+ collector::Hashes, inverted_index::DocAddress, prehashed::Prehashed,
+ ranking::initial::Score,
+ };
+
use super::*;
fn sample_websites(n: usize) -> Vec<RankingWebsite> {
(0..n)
.map(|i| -> RankingWebsite {
RankingWebsite {
- address: tantivy::DocAddress {
- segment_ord: 0,
- doc_id: i as u32,
+ pointer: WebsitePointer {
+ score: Score {
+ bm25: 0.0,
+ total: 0.0,
+ },
+ hashes: Hashes {
+ site: Prehashed(0),
+ title: Prehashed(0),
+ url: Prehashed(0),
+ simhash: 0,
+ },
+ address: DocAddress {
+ segment: 0,
+ doc_id: i as u32,
+ },
},
- bm25: 0.0,
host_centrality: 0.0,
page_centrality: 0.0,
topic_centrality: 0.0,
@@ -149,7 +222,7 @@ mod tests {
query_centrality: 0.0,
title: String::new(),
clean_body: String::new(),
- score: i as f64,
+ score: 0.0,
}
})
.collect()
@@ -164,14 +237,14 @@ mod tests {
let res: Vec<_> = pipeline
.apply(sample)
.into_iter()
- .map(|w| w.address)
+ .map(|w| w.pointer.address)
.collect();
let expected: Vec<_> = sample_websites(100)
.into_iter()
.rev()
.take(20)
- .map(|w| w.address)
+ .map(|w| w.pointer.address)
.collect();
assert_eq!(res, expected);
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index a4c75a87..c00e2965 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -26,13 +26,15 @@ use crate::{
use std::{array, ops::Deref, sync::Arc};
use chrono::Utc;
-use tantivy::{DocId, Score};
+use tantivy::DocId;
use crate::{
schema::{Field, FLOAT_SCALING},
webpage::region::{Region, RegionCount},
};
+use super::initial::Score;
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Signal {
Bm25,
@@ -65,11 +67,11 @@ pub const ALL_SIGNALS: [Signal; 12] = [
];
struct SignalValue {
- bm25: Score,
+ bm25: tantivy::Score,
fastfield_value: Option<u64>,
current_timestamp: usize,
selected_region: Option<Region>,
- personal_centrality: f64,
+ personal_centrality: Option<f64>,
topic_score: Option<f64>,
query_centrality: Option<f64>,
}
@@ -90,7 +92,7 @@ impl Signal {
Signal::HostCentrality | Signal::PageCentrality => {
value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64
}
- Signal::PersonalCentrality => value.personal_centrality,
+ Signal::PersonalCentrality => value.personal_centrality.unwrap_or_default(),
Signal::IsHomepage => value.fastfield_value.unwrap() as f64,
Signal::FetchTimeMs => {
let fetch_time_ms = value.fastfield_value.unwrap() as usize;
@@ -320,36 +322,49 @@ impl SignalAggregator {
self.personal_centrality.push(Arc::new(personal_centrality))
}
+ pub fn topic_centrality(&self, host_id: u64) -> Option<f64> {
+ self.topic_scorer
+ .as_ref()
+ .map(|scorer| scorer.score(host_id))
+ }
+
+ pub fn query_centrality(&self, host_id: u64) -> Option<f64> {
+ self.query_centrality
+ .as_ref()
+ .map(|scorer| scorer.score(host_id) - SHIFT)
+ }
+
+ pub fn personal_centrality(&self, host_id: u64) -> f64 {
+ self.personal_centrality
+ .iter()
+ .map(|scorer| scorer.score(host_id))
+ .sum()
+ }
+
pub fn score(
&self,
doc: DocId,
- bm25: Score,
+ bm25: tantivy::Score,
region_count: &Arc<RegionCount>,
current_timestamp: usize,
selected_region: Option<Region>,
- ) -> f64 {
+ ) -> Score {
let host_id = self
.fastfield_cache
.as_ref()
.and_then(|cache| cache.get_doc_cache(&FastField::HostNodeID).get_u64(&doc));
- let topic_score = self
- .topic_scorer
- .as_ref()
- .and_then(|scorer| host_id.map(|host| scorer.score(host)));
+ let topic_score = host_id.and_then(|host_id| self.topic_centrality(host_id));
- let query_centrality = self
- .query_centrality
- .as_ref()
- .and_then(|scorer| host_id.map(|host| scorer.score(host) - SHIFT));
+ let query_centrality = host_id.and_then(|host_id| self.query_centrality(host_id));
- let personal_centrality = self
- .personal_centrality
- .iter()
- .map(|scorer| scorer.score(doc as u64))
- .sum();
+ let personal_centrality = host_id.map(|host_id| self.personal_centrality(host_id));
- ALL_SIGNALS
+ if personal_centrality.unwrap() > 1.0 {
+ dbg!(personal_centrality);
+ }
+
+ let score = ALL_SIGNALS
.into_iter()
.map(|signal| {
let fastfield_value = signal.as_fastfield().and_then(|field| {
@@ -373,7 +388,9 @@ impl SignalAggregator {
},
)
})
- .sum()
+ .sum();
+
+ Score { bm25, total: score }
}
pub fn precompute_score(&self, webpage: &Webpage, region_count: &RegionCount) -> f64 {
@@ -419,7 +436,7 @@ impl SignalAggregator {
fastfield_value: Some(fastfield_value),
current_timestamp,
selected_region: None,
- personal_centrality: 0.0,
+ personal_centrality: None,
topic_score: None,
query_centrality: None,
},
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 609a9377..b87998c5 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -266,7 +266,7 @@ struct ScoredWebsitePointer {
impl collector::Doc for ScoredWebsitePointer {
fn score(&self) -> &f64 {
- &self.local_pointer.score
+ &self.local_pointer.score.total
}
fn id(&self) -> &tantivy::DocId {
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 24ee3390..fa06b391 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -221,7 +221,7 @@ impl LocalSearcher {
if res.num_websites > 0 && !res.top_websites.is_empty() {
let top = res.top_websites.remove(0);
- if top.score > self.stackoverflow_sidebar_threshold {
+ if top.score.total > self.stackoverflow_sidebar_threshold {
let mut retrieved = self.retrieve_websites(&[top], &query.original)?;
let retrieved = retrieved.remove(0);
return Ok(Some(Sidebar::StackOverflow {
diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rs
index c6338a99..c81fbfac 100644
--- a/core/src/webgraph/centrality/topic.rs
+++ b/core/src/webgraph/centrality/topic.rs
@@ -32,6 +32,7 @@ use tantivy::{
};
use crate::{
+ collector::TopDocs,
human_website_annotations::Mapper,
index::Index,
ranking::{initial::InitialScoreTweaker, SignalAggregator},
@@ -144,7 +145,8 @@ impl TopicCentrality {
.get_field(Field::Fast(FastField::HostNodeID).name())
.unwrap();
- let collector = tantivy::collector::TopDocs::with_limit(1000).tweak_score(score_tweaker);
+ let collector = TopDocs::with_limit(1000, index.inverted_index.fastfield_cache())
+ .tweak_score(score_tweaker);
let mut nodes: Vec<_> = webgraph.host.as_ref().unwrap().nodes().collect();
nodes.sort();
@@ -160,8 +162,8 @@ impl TopicCentrality {
.search(&query, &collector)
.unwrap()
.into_iter()
- .filter_map(|(_, doc_address)| {
- let doc = searcher.doc(doc_address).unwrap();
+ .filter_map(|pointer| {
+ let doc = searcher.doc(pointer.address.into()).unwrap();
let id = doc.get_first(graph_node_id).unwrap().as_u64().unwrap();
if id == u64::MAX {
None
diff --git a/frontend/src/components/Searchbar.astro b/frontend/src/components/Searchbar.astro
index c2c0f291..5d6bbc12 100644
--- a/frontend/src/components/Searchbar.astro
+++ b/frontend/src/components/Searchbar.astro
@@ -284,14 +284,14 @@ const { autofocus, query } = Astro.props;
) as Record<string, number>;
let rankings = {
- preferred: [] as string[],
+ liked: [] as string[],
disliked: [] as string[],
blocked: [] as string[],
};
for (const [site, ranking] of Object.entries(sites)) {
if (ranking == MORE) {
- rankings.preferred.push(site);
+ rankings.liked.push(site);
} else if (ranking == LESS) {
rankings.disliked.push(site);
} else if (ranking == BLOCKED) {