Skip to content

web search done right

use localsearcher in all tests

Mikkel Denker1b6192179943
diff --git a/core/src/collector.rs b/core/src/collector.rs
index 899bc2a8..1595e3a4 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -412,9 +412,11 @@ where
segment_reader: &SegmentReader,
) -> tantivy::Result<Self::Child> {
let segment_scorer = self.score_tweaker.segment_tweaker(segment_reader)?;
+
let segment_collector = self
.collector
.for_segment(segment_local_id, segment_reader)?;
+
Ok(TopTweakedScoreSegmentCollector {
segment_collector,
segment_scorer,
diff --git a/core/src/index.rs b/core/src/index.rs
index f7a06b8b..f36be280 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -30,7 +30,7 @@ use uuid::Uuid;
use crate::directory::{self, DirEntry};
use crate::image_downloader::{ImageDownloadJob, ImageDownloader};
use crate::image_store::{FaviconStore, Image, ImageStore, PrimaryImageStore};
-use crate::inverted_index::{self, InitialSearchResult, InvertedIndex, SearchResult};
+use crate::inverted_index::{self, InitialSearchResult, InvertedIndex};
use crate::query::Query;
use crate::spell::{
Correction, CorrectionTerm, Dictionary, LogarithmicEdit, SpellChecker, TermSplitter,
@@ -122,13 +122,6 @@ impl Index {
Ok(())
}
- pub fn search<C>(&self, query: &Query, collector: C) -> Result<SearchResult>
- where
- C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,
- {
- self.inverted_index.search(query, collector)
- }
-
pub fn search_initial<C>(&self, query: &Query, collector: C) -> Result<InitialSearchResult>
where
C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,
@@ -352,7 +345,7 @@ impl From<Index> for FrozenIndex {
#[cfg(test)]
mod tests {
- use crate::ranking::{Ranker, SignalAggregator};
+ use crate::searcher::{LocalSearcher, SearchQuery};
use super::*;
@@ -390,30 +383,21 @@ mod tests {
let deserialized_frozen: FrozenIndex = bincode::deserialize(&bytes).unwrap();
let index: Index = deserialized_frozen.into();
- let query = Query::parse(
- "website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.inverted_index.fastfield_cache(),
- );
+ let searcher = LocalSearcher::from(index);
+
+ let result = searcher
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ ..Default::default()
+ })
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
-
- assert_eq!(
- String::from(index.spell_correction(&["thiss".to_string()]).unwrap()),
- "this".to_string()
- );
}
#[test]
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 9677f3bd..8559128b 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -251,19 +251,6 @@ impl InvertedIndex {
Ok(webpages)
}
- pub fn search<C>(&self, query: &Query, collector: C) -> Result<SearchResult>
- where
- C: Collector<Fruit = Vec<WebsitePointer>>,
- {
- let initial_result = self.search_initial(query, collector)?;
- let websites = self.retrieve_websites(&initial_result.top_websites, query)?;
-
- Ok(SearchResult {
- num_docs: initial_result.num_websites,
- documents: websites,
- })
- }
-
pub fn merge_into_segments(&mut self, num_segments: u32) -> Result<()> {
assert!(num_segments > 0);
@@ -403,7 +390,7 @@ pub struct SearchResult {
pub documents: Vec<RetrievedWebpage>,
}
-#[derive(Default, Debug, Serialize, Deserialize)]
+#[derive(Default, Debug, Serialize, Deserialize, PartialEq)]
pub struct RetrievedWebpage {
pub title: String,
pub url: String,
@@ -535,6 +522,19 @@ mod tests {
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
+ fn search<C>(index: &InvertedIndex, query: &Query, collector: C) -> Result<SearchResult>
+ where
+ C: Collector<Fruit = Vec<WebsitePointer>>,
+ {
+ let initial_result = index.search_initial(query, collector)?;
+ let websites = index.retrieve_websites(&initial_result.top_websites, query)?;
+
+ Ok(SearchResult {
+ num_docs: initial_result.num_websites,
+ documents: websites,
+ })
+ }
+
#[test]
fn simple_search() {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
@@ -551,9 +551,7 @@ mod tests {
index.fastfield_cache(),
);
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
@@ -576,9 +574,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -619,9 +615,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
}
@@ -661,9 +655,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
}
@@ -703,9 +695,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
}
@@ -778,9 +768,7 @@ mod tests {
index.commit().expect("failed to commit index");
- let mut result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let mut result = search(&index, &query, ranker.collector()).expect("Search failed");
result
.documents
@@ -830,9 +818,7 @@ mod tests {
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 20);
}
@@ -871,9 +857,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
}
@@ -938,9 +922,7 @@ mod tests {
index.fastfield_cache(),
);
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 2);
assert_eq!(result.documents.len(), 2);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -963,9 +945,7 @@ mod tests {
index.fastfield_cache(),
);
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
@@ -988,9 +968,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -1037,9 +1015,7 @@ mod tests {
)
.expect("Failed to parse query");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
@@ -1061,9 +1037,7 @@ mod tests {
)
.expect("Failed to parse query");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
@@ -1087,9 +1061,7 @@ mod tests {
index.fastfield_cache(),
);
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 0);
assert_eq!(result.num_docs, 0);
@@ -1112,9 +1084,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.example.com");
@@ -1174,9 +1144,7 @@ mod tests {
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.dr.dk");
}
@@ -1222,9 +1190,7 @@ mod tests {
index.commit().expect("failed to commit index");
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let result = search(&index, &query, ranker.collector()).expect("Search failed");
assert_eq!(result.documents.len(), 1);
let schema = result.documents[0].schema_org.clone();
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index ff86e8e9..822cae6f 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -205,10 +205,10 @@ impl tantivy::query::Query for Query {
#[cfg(test)]
mod tests {
use crate::{
- inverted_index::InvertedIndex,
- ranking::Ranker,
+ index::Index,
schema::create_schema,
- webpage::{region::RegionCount, Webpage},
+ searcher::{LocalSearcher, SearchQuery},
+ webpage::Webpage,
};
use super::*;
@@ -311,19 +311,11 @@ mod tests {
#[test]
fn not_query() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
- let query = Query::parse(
- "test -website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
+ let mut index = Index::temporary().expect("Unable to open index");
+ let query = SearchQuery {
+ original: "test -website".to_string(),
+ ..Default::default()
+ };
index
.insert(Webpage::new(
@@ -356,10 +348,14 @@ mod tests {
))
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
-
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let searcher = LocalSearcher::from(index);
+
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.second.com");
@@ -367,7 +363,7 @@ mod tests {
#[test]
fn site_query() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
index
.insert(Webpage::new(
@@ -400,60 +396,46 @@ mod tests {
))
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
-
- let query = Query::parse(
- "test site:first.com",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ original: "test site:first.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.first.com");
- let query = Query::parse(
- "test site:www.first.com",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let query = SearchQuery {
+ original: "test site:www.first.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.first.com");
- let query = Query::parse(
- "test -site:first.com",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let query = SearchQuery {
+ original: "test -site:first.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.second.com");
@@ -461,7 +443,7 @@ mod tests {
#[test]
fn title_query() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
index
.insert(Webpage::new(
@@ -494,22 +476,18 @@ mod tests {
))
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
-
- let query = Query::parse(
- "intitle:website",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ original: "intitle:website".to_string(),
+ ..Default::default()
+ };
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.first.com");
@@ -517,7 +495,7 @@ mod tests {
#[test]
fn url_query() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
index
.insert(Webpage::new(
@@ -550,22 +528,18 @@ mod tests {
))
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
-
- let query = Query::parse(
- "test inurl:forum",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ original: "test inurl:forum".to_string(),
+ ..Default::default()
+ };
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.first.com/forum");
@@ -588,7 +562,7 @@ mod tests {
#[test]
fn query_term_only_special_char() {
- let index = InvertedIndex::temporary().expect("Unable to open index");
+ let index = Index::temporary().expect("Unable to open index");
let query = Query::parse(
"&",
@@ -603,7 +577,7 @@ mod tests {
#[test]
fn site_query_split_domain() {
- let mut index = InvertedIndex::temporary().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
index
.insert(Webpage::new(
@@ -636,59 +610,45 @@ mod tests {
))
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
-
- let query = Query::parse(
- "test site:first.com",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ original: "test site:first.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 0);
assert_eq!(result.documents.len(), 0);
- let query = Query::parse(
- "test site:the-first.com",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let query = SearchQuery {
+ original: "test site:the-first.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.the-first.com");
- let query = Query::parse(
- "test site:www.the-first.com",
- index.schema(),
- index.tokenizers(),
- &SignalAggregator::default(),
- )
- .expect("Failed to parse query");
- let ranker = Ranker::new(
- RegionCount::default(),
- SignalAggregator::default(),
- index.fastfield_cache(),
- );
- let result = index
- .search(&query, ranker.collector())
- .expect("Search failed");
+ let query = SearchQuery {
+ original: "test site:www.the-first.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher
+ .search(&query)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
assert_eq!(result.documents[0].url, "https://www.the-first.com");
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index ca96c14e..44178f20 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -18,6 +18,7 @@ pub mod bm25;
pub mod centrality_store;
pub mod initial;
pub mod optics;
+pub mod pipeline;
pub mod signal;
use std::sync::Arc;
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
new file mode 100644
index 00000000..ac64b34d
--- /dev/null
+++ b/core/src/ranking/pipeline.rs
@@ -0,0 +1,179 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::cmp::Ordering;
+
+#[derive(Clone, Debug)]
+struct RankingWebsite {
+ address: tantivy::DocAddress,
+ bm25: f64,
+ host_centrality: f64,
+ page_centrality: f64,
+ topic_centrality: f64,
+ personal_centrality: f64,
+ query_centrality: f64,
+ title: String,
+ clean_body: String,
+ score: f64,
+}
+
+trait Scorer {
+ fn score(&self, websites: &mut [RankingWebsite]);
+}
+
+struct ReRanker {}
+
+impl Scorer for ReRanker {
+ fn score(&self, websites: &mut [RankingWebsite]) {
+ // TODO: Implement actual scoring
+ // todo!();
+ }
+}
+
+struct SignalFocusText {}
+
+impl Scorer for SignalFocusText {
+ fn score(&self, websites: &mut [RankingWebsite]) {
+ // TODO: Implement actual scoring
+ // todo!();
+ }
+}
+
+enum Prev {
+ Initial,
+ Node(Box<RankingStage>),
+}
+
+struct RankingStage {
+ scorer: Box<dyn Scorer>,
+ prev: Prev,
+ top_n: usize,
+ memory: Option<Vec<RankingWebsite>>,
+}
+
+impl RankingStage {
+ fn initial_top_n(&self) -> usize {
+ match &self.prev {
+ Prev::Initial => self.top_n,
+ Prev::Node(n) => n.initial_top_n(),
+ }
+ }
+
+ pub fn populate(&mut self, websites: Vec<RankingWebsite>) {
+ match &mut self.prev {
+ Prev::Initial => self.memory = Some(websites),
+ Prev::Node(n) => n.populate(websites),
+ }
+ }
+
+ fn apply(self, top_n: usize) -> Vec<RankingWebsite> {
+ let mut chunk = match self.prev {
+ Prev::Initial => self.memory.unwrap(),
+ Prev::Node(n) => n.apply(self.top_n),
+ };
+
+ self.scorer.score(&mut chunk);
+ chunk.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal));
+
+ chunk.into_iter().take(top_n).collect()
+ }
+}
+
+struct Pipeline {
+ last_stage: RankingStage,
+}
+
+impl Default for Pipeline {
+ fn default() -> Self {
+ let last_stage = RankingStage {
+ scorer: Box::new(ReRanker {}),
+ prev: Prev::Node(Box::new(RankingStage {
+ scorer: Box::new(SignalFocusText {}),
+ prev: Prev::Initial,
+ memory: None,
+ top_n: 100,
+ })),
+ memory: None,
+ top_n: 50,
+ };
+ Self { last_stage }
+ }
+}
+
+impl Pipeline {
+ pub fn with_offset(&mut self, offset: usize) {
+ todo!();
+ }
+
+ pub fn apply(mut self, websites: Vec<RankingWebsite>) -> Vec<RankingWebsite> {
+ self.last_stage.populate(websites);
+
+ self.last_stage.apply(20)
+ }
+
+ pub fn initial_top_n(&self) -> usize {
+ self.last_stage.initial_top_n()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn sample_websites(n: usize) -> Vec<RankingWebsite> {
+ (0..n)
+ .map(|i| -> RankingWebsite {
+ RankingWebsite {
+ address: tantivy::DocAddress {
+ segment_ord: 0,
+ doc_id: i as u32,
+ },
+ bm25: 0.0,
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ topic_centrality: 0.0,
+ personal_centrality: 0.0,
+ query_centrality: 0.0,
+ title: String::new(),
+ clean_body: String::new(),
+ score: i as f64,
+ }
+ })
+ .collect()
+ }
+
+ #[test]
+ fn simple() {
+ let pipeline = Pipeline::default();
+ assert_eq!(pipeline.initial_top_n(), 100);
+
+ let sample = sample_websites(pipeline.initial_top_n());
+ let res: Vec<_> = pipeline
+ .apply(sample)
+ .into_iter()
+ .map(|w| w.address)
+ .collect();
+
+ let expected: Vec<_> = sample_websites(100)
+ .into_iter()
+ .rev()
+ .take(20)
+ .map(|w| w.address)
+ .collect();
+
+ assert_eq!(res, expected);
+ }
+}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index a652a0b2..a4c75a87 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -343,6 +343,12 @@ impl SignalAggregator {
.as_ref()
.and_then(|scorer| host_id.map(|host| scorer.score(host) - SHIFT));
+ let personal_centrality = self
+ .personal_centrality
+ .iter()
+ .map(|scorer| scorer.score(doc as u64))
+ .sum();
+
ALL_SIGNALS
.into_iter()
.map(|signal| {
@@ -352,12 +358,6 @@ impl SignalAggregator {
.and_then(|cache| cache.get_doc_cache(&field).get_u64(&doc))
});
- let personal_centrality = self
- .personal_centrality
- .iter()
- .map(|scorer| scorer.score(doc as u64))
- .sum();
-
self.coefficients().get(&signal)
* signal.value(
region_count,