web search done right
fixed page offset bug and prepared for multi stage ranking
Mikkel Denkerf07245834a05
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex b880ab93..ce665b87 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -96,7 +96,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {}searcher::InitialSearchResult::Bang(bang) => {req.respond(sonic::Response::Content(- searcher::InitialPrettifiedSearchResult::Bang(bang),+ searcher::InitialSearchResult::Bang(bang),)).await.ok();diff --git a/core/src/frontend/api.rs b/core/src/frontend/api.rsdeleted file mode 100644index be360bce..00000000--- a/core/src/frontend/api.rs+++ /dev/null@@ -1,54 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{collections::HashMap, sync::Arc};--use crate::{searcher::SearchQuery, webpage::region::Region};--use super::State;-use axum::{extract, response::IntoResponse, Extension, Json};--#[allow(clippy::unused_async)]-pub async fn search(- extract::Query(params): extract::Query<HashMap<String, String>>,- Extension(state): Extension<Arc<State>>,-) -> impl IntoResponse {- let query = params.get("q").cloned().unwrap_or_default();-- let skip_pages = params.get("p").and_then(|p| p.parse().ok());-- let selected_region = params.get("gl").and_then(|gl| {- if let Ok(region) = Region::from_gl(gl) {- Some(region)- } else {- None- }- });-- match state- .searcher- .search_api(&SearchQuery {- original: query.to_string(),- selected_region,- skip_pages,- ..Default::default()- })- .await- {- Ok(result) => Json(result),- Err(_) => panic!("Search failed"), // TODO: show 500 status to user here- }-}diff --git a/core/src/frontend/mod.rs b/core/src/frontend/mod.rsindex b7c9475b..5275680a 100644--- a/core/src/frontend/mod.rs+++ b/core/src/frontend/mod.rs@@ -32,7 +32,6 @@ use axum::{};mod about;-mod api;mod autosuggest;mod index;mod opensearch;@@ -100,7 +99,6 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router.route("/settings/optics", get(optics::route)).route("/settings/sites", get(sites::route)).route("/privacy-and-happy-lawyers", get(privacy::route))- .route("/api/beta/search", get(api::search)).route("/opensearch.xml", get(opensearch::route)).fallback(get_service(ServeDir::new("frontend/dist/")).handle_error(|error: std::io::Error| async move {diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rsindex 8a97a0d3..eed51e39 100644--- a/core/src/frontend/search.rs+++ b/core/src/frontend/search.rs@@ -25,7 +25,7 @@ use crate::{thousand_sep_number, CodeOrText, DisplayedSidebar, DisplayedWebpage,HighlightedSpellCorrection, Snippet,},- searcher::{self, PrettifiedSearchResult, SearchQuery},+ searcher::{self, PrettifiedSearchResult, SearchQuery, NUM_RESULTS_PER_PAGE},webpage::region::{Region, ALL_REGIONS},};@@ -71,7 +71,11 @@ pub async fn route() -> impl IntoResponse {let query = params.get("q").cloned().unwrap_or_default();- let skip_pages = params.get("p").and_then(|p| p.parse().ok());+ let skip_pages: usize = params+ .get("p")+ .and_then(|p| p.parse().ok())+ .unwrap_or_default();+ let offset = skip_pages * NUM_RESULTS_PER_PAGE;let mut optic = None;let mut current_optic_url = None;@@ -116,13 +120,13 @@ pub async fn route(match state.searcher- .search_prettified(&SearchQuery {+ .search(&SearchQuery {original: query.clone(),selected_region,optic_program: optic,- skip_pages,+ offset,site_rankings,- custom_num_results: None,+ num_results: NUM_RESULTS_PER_PAGE,}).await{@@ -151,10 +155,10 @@ pub async fn route(}).collect();- let current_page = skip_pages.unwrap_or(0) + 1;+ let current_page = skip_pages + 1;let mut next_page_params = params.clone();- next_page_params.insert("p".to_string(), (skip_pages.unwrap_or(0) + 1).to_string());+ next_page_params.insert("p".to_string(), (skip_pages + 1).to_string());let next_page_url = uri.path().to_string()+ "?"+ serde_urlencoded::to_string(&next_page_params)@@ -163,8 +167,7 @@ pub async fn route(let prev_page_url = if current_page > 1 {let mut prev_page_params = params;- prev_page_params- .insert("p".to_string(), (skip_pages.unwrap_or(0) - 1).to_string());+ prev_page_params.insert("p".to_string(), (skip_pages - 1).to_string());Some(uri.path().to_string()+ "?"diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 3bd9b99c..2ab40cc7 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -16,15 +16,27 @@use std::cmp::Ordering;+use itertools::Itertools;use serde::{Deserialize, Serialize};use crate::{inverted_index::WebsitePointer,schema::{FastField, Field, TextField, ALL_FIELDS, FLOAT_SCALING},+ searcher::SearchQuery,};use super::SignalAggregator;+pub trait AsRankingWebsite: Clone {+ fn as_ranking(&self) -> &RankingWebsite;+}++impl AsRankingWebsite for RankingWebsite {+ fn as_ranking(&self) -> &RankingWebsite {+ self+ }+}+#[derive(Clone, Debug, Serialize, Deserialize)]pub struct RankingWebsite {pub pointer: WebsitePointer,@@ -45,7 +57,6 @@ impl RankingWebsite {aggregator: &SignalAggregator,) -> Self {let mut res = RankingWebsite {- pointer,host_centrality: 0.0,page_centrality: 0.0,topic_centrality: 0.0,@@ -53,7 +64,8 @@ impl RankingWebsite {query_centrality: 0.0,title: String::new(),clean_body: String::new(),- score: 0.0,+ score: pointer.score.total,+ pointer,};for value in doc.field_values() {@@ -69,7 +81,7 @@ impl RankingWebsite {Field::Fast(FastField::HostNodeID) => {let node = value.value.as_u64().unwrap();- res.personal_centrality = dbg!(aggregator.personal_centrality(node));+ res.personal_centrality = aggregator.personal_centrality(node);res.topic_centrality = aggregator.topic_centrality(node).unwrap_or_default();res.query_centrality = aggregator.query_centrality(node).unwrap_or_default();}@@ -87,14 +99,14 @@ impl RankingWebsite {}}-trait Scorer {- fn score(&self, websites: &mut [RankingWebsite]);+trait Scorer<T: AsRankingWebsite>: Send + Sync {+ fn score(&self, websites: &mut [T]);}struct ReRanker {}-impl Scorer for ReRanker {- fn score(&self, websites: &mut [RankingWebsite]) {+impl<T: AsRankingWebsite> Scorer<T> for ReRanker {+ fn score(&self, websites: &mut [T]) {// TODO: Implement actual scoring// todo!();}@@ -102,26 +114,26 @@ impl Scorer for ReRanker {struct SignalFocusText {}-impl Scorer for SignalFocusText {- fn score(&self, websites: &mut [RankingWebsite]) {+impl<T: AsRankingWebsite> Scorer<T> for SignalFocusText {+ fn score(&self, websites: &mut [T]) {// TODO: Implement actual scoring// todo!();}}-enum Prev {+enum Prev<T: AsRankingWebsite> {Initial,- Node(Box<RankingStage>),+ Node(Box<RankingStage<T>>),}-struct RankingStage {- scorer: Box<dyn Scorer>,- prev: Prev,+struct RankingStage<T: AsRankingWebsite> {+ scorer: Box<dyn Scorer<T>>,+ prev: Prev<T>,top_n: usize,- memory: Option<Vec<RankingWebsite>>,+ memory: Option<(Vec<T>, Vec<T>)>,}-impl RankingStage {+impl<T: AsRankingWebsite> RankingStage<T> {fn initial_top_n(&self) -> usize {match &self.prev {Prev::Initial => self.top_n,@@ -129,32 +141,90 @@ impl RankingStage {}}- pub fn populate(&mut self, websites: Vec<RankingWebsite>) {+ pub fn populate(&mut self, websites: Vec<T>) {match &mut self.prev {- Prev::Initial => self.memory = Some(websites),+ Prev::Initial => {+ let a: Vec<_> = websites.clone().into_iter().take(self.top_n).collect();+ let b: Vec<_> = websites+ .into_iter()+ .skip(self.top_n)+ .take(self.top_n)+ .collect();++ self.memory = Some((a, b));+ }Prev::Node(n) => n.populate(websites),}}- fn apply(self, top_n: usize) -> Vec<RankingWebsite> {- let mut chunk = match self.prev {- Prev::Initial => self.memory.unwrap(),- Prev::Node(n) => n.apply(self.top_n),- };-- self.scorer.score(&mut chunk);- chunk.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(Ordering::Equal));-- chunk.into_iter().take(top_n).collect()+ fn apply(&self, top_n: usize, offset: usize) -> Vec<T> {+ match &self.prev {+ Prev::Initial => {+ let (mut a, mut b) = self.memory.clone().unwrap();++ self.scorer.score(&mut a);+ a.sort_by(|a, b| {+ b.as_ranking()+ .score+ .partial_cmp(&a.as_ranking().score)+ .unwrap_or(Ordering::Equal)+ });++ self.scorer.score(&mut b);+ b.sort_by(|a, b| {+ b.as_ranking()+ .score+ .partial_cmp(&a.as_ranking().score)+ .unwrap_or(Ordering::Equal)+ });++ a.into_iter()+ .chain(b.into_iter())+ .skip(offset % self.top_n)+ .take(top_n)+ .collect()+ }+ Prev::Node(n) => {+ let k = offset / self.top_n;+ let (mut a, mut b) = (+ n.apply(self.top_n, k * self.top_n),+ n.apply(self.top_n, (k + 1) * self.top_n),+ );++ self.scorer.score(&mut a);+ self.scorer.score(&mut b);++ a.sort_by(|a, b| {+ b.as_ranking()+ .score+ .partial_cmp(&a.as_ranking().score)+ .unwrap_or(Ordering::Equal)+ });+ b.sort_by(|a, b| {+ b.as_ranking()+ .score+ .partial_cmp(&a.as_ranking().score)+ .unwrap_or(Ordering::Equal)+ });++ a.into_iter()+ .chain(b.into_iter())+ .skip(offset % self.top_n)+ .take(top_n)+ .collect()+ }+ }}}-struct Pipeline {- last_stage: RankingStage,+pub struct RankingPipeline<T: AsRankingWebsite> {+ last_stage: RankingStage<T>,+ offset: usize,+ top_n: usize,}-impl Default for Pipeline {- fn default() -> Self {+impl<T: AsRankingWebsite> RankingPipeline<T> {+ fn create() -> Self {let last_stage = RankingStage {scorer: Box::new(ReRanker {}),prev: Prev::Node(Box::new(RankingStage {@@ -166,24 +236,43 @@ impl Default for Pipeline {memory: None,top_n: 50,};- Self { last_stage }++ Self {+ last_stage,+ offset: 0,+ top_n: 0,+ }}-}-impl Pipeline {- pub fn with_offset(&mut self, offset: usize) {- todo!();+ pub fn for_query(query: &mut SearchQuery) -> Self {+ let mut pipeline = Self::create();++ pipeline.offset = query.offset;+ pipeline.top_n = query.num_results;++ query.num_results = pipeline.collector_top_n();+ query.offset = pipeline.collector_offset();++ pipeline}- pub fn apply(mut self, websites: Vec<RankingWebsite>) -> Vec<RankingWebsite> {+ pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {self.last_stage.populate(websites);- self.last_stage.apply(20)+ self.last_stage.apply(20, self.offset)+ }++ pub fn collector_top_n(&self) -> usize {+ 2 * self.initial_top_n()}pub fn initial_top_n(&self) -> usize {self.last_stage.initial_top_n()}++ pub fn collector_offset(&self) -> usize {+ (self.offset / self.initial_top_n()) * self.initial_top_n()+ }}#[cfg(test)]@@ -222,7 +311,7 @@ mod tests {query_centrality: 0.0,title: String::new(),clean_body: String::new(),- score: 0.0,+ score: 1.0 / i as f64,}}).collect()@@ -230,10 +319,12 @@ mod tests {#[test]fn simple() {- let pipeline = Pipeline::default();- assert_eq!(pipeline.initial_top_n(), 100);+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {+ ..Default::default()+ });+ assert_eq!(pipeline.collector_top_n(), 200);- let sample = sample_websites(pipeline.initial_top_n());+ let sample = sample_websites(pipeline.collector_top_n());let res: Vec<_> = pipeline.apply(sample).into_iter()@@ -242,11 +333,93 @@ mod tests {let expected: Vec<_> = sample_websites(100).into_iter()- .rev().take(20).map(|w| w.pointer.address).collect();assert_eq!(res, expected);}++ #[test]+ fn offsets() {+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {+ offset: 0,+ ..Default::default()+ });++ let sample: Vec<_> =+ sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())+ .into_iter()+ .skip(pipeline.collector_offset())+ .collect();+ let mut prev: Vec<_> = pipeline.apply(sample);++ for offset in 1..1_000 {+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {+ offset,+ ..Default::default()+ });++ let sample: Vec<_> =+ sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())+ .into_iter()+ .skip(pipeline.collector_offset())+ .collect();+ let res: Vec<_> = pipeline.apply(sample);++ assert_eq!(res.len(), 20, "Every page should have 20 results");++ if let Some(first) = prev.first() {+ assert!(!res+ .iter()+ .any(|r| r.pointer.address.doc_id == first.pointer.address.doc_id));+ }++ // assert_eq!(+ // prev.iter()+ // .map(|p| usize::from(+ // res.iter()+ // .any(|r| r.pointer.address.doc_id == p.pointer.address.doc_id)+ // ))+ // .sum::<usize>(),+ // 19,+ // "Only the top result from previous offset should be removed in current ranking"+ // );++ prev = res;+ }++ let pipeline = RankingPipeline::for_query(&mut SearchQuery {+ offset: 0,+ ..Default::default()+ });+ let sample: Vec<_> =+ sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())+ .into_iter()+ .skip(pipeline.collector_offset())+ .collect();+ let mut prev: Vec<_> = pipeline.apply(sample);+ for p in 1..100 {+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {+ offset: p * 20,+ ..Default::default()+ });++ let sample: Vec<_> =+ sample_websites(pipeline.collector_top_n() + pipeline.collector_offset())+ .into_iter()+ .skip(pipeline.collector_offset())+ .collect();+ let res: Vec<_> = pipeline.apply(sample).into_iter().collect();++ assert_eq!(res.len(), 20, "Every page should have 20 results");++ assert!(prev+ .iter()+ .zip_eq(res.iter())+ .all(|(p, r)| p.pointer.address.doc_id + 20 == r.pointer.address.doc_id));++ prev = res;+ }+ }}diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex c00e2965..b991c20e 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -360,10 +360,6 @@ impl SignalAggregator {let personal_centrality = host_id.map(|host_id| self.personal_centrality(host_id));- if personal_centrality.unwrap() > 1.0 {- dbg!(personal_centrality);- }-let score = ALL_SIGNALS.into_iter().map(|signal| {diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rsindex 2d2f8802..ec6d3a21 100644--- a/core/src/search_prettifier.rs+++ b/core/src/search_prettifier.rs@@ -23,7 +23,8 @@ use tracing::log::debug;use crate::{entity_index::{entity::Span, StoredEntity},- inverted_index::{self, RetrievedWebpage},+ inverted_index::RetrievedWebpage,+ ranking::pipeline::RankingWebsite,searcher::{self, LocalSearcher, Sidebar},spell::CorrectionTerm,webpage::{@@ -76,6 +77,7 @@ pub fn initial(InitialWebsiteResult {spell_corrected_query,websites: result.websites,+ num_websites: result.num_websites,sidebar,}}@@ -112,7 +114,8 @@ pub struct HighlightedSpellCorrection {#[derive(Debug, Serialize, Deserialize)]pub struct InitialWebsiteResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,- pub websites: inverted_index::InitialSearchResult,+ pub num_websites: usize,+ pub websites: Vec<RankingWebsite>,pub sidebar: Option<DisplayedSidebar>,}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex b87998c5..89471de2 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -17,9 +17,10 @@use crate::{collector::{self, BucketCollector},exponential_backoff::ExponentialBackoff,- inverted_index::{self, RetrievedWebpage},+ inverted_index::{self},+ ranking::pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},search_prettifier::DisplayedWebpage,- searcher::{PrettifiedWebsitesResult, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE},+ searcher::PrettifiedWebsitesResult,};use std::{net::SocketAddr, time::Instant};@@ -31,9 +32,7 @@ use thiserror::Error;use crate::sonic;-use super::{- InitialPrettifiedSearchResult, InitialSearchResult, PrettifiedSearchResult, SearchQuery,-};+use super::{InitialPrettifiedSearchResult, PrettifiedSearchResult, SearchQuery};type Result<T> = std::result::Result<T, Error>;@@ -51,25 +50,7 @@ pub enum Error {}impl RemoteSearcher {- async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResult> {- for timeout in ExponentialBackoff::from_millis(30).take(5) {- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await- {- if let Ok(sonic::Response::Content(body)) =- connection.send(Request::Search(query.clone())).await- {- return Ok(body);- }- }- }-- Err(Error::SearchFailed)- }-- async fn search_prettified(- &self,- query: &SearchQuery,- ) -> Result<InitialPrettifiedSearchResult> {+ async fn search(&self, query: &SearchQuery) -> Result<InitialPrettifiedSearchResult> {for timeout in ExponentialBackoff::from_millis(30).take(5) {if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await{@@ -89,29 +70,6 @@ impl RemoteSearcher {&self,pointers: &[inverted_index::WebsitePointer],original_query: &str,- ) -> Result<Vec<RetrievedWebpage>> {- for timeout in ExponentialBackoff::from_millis(30).take(5) {- if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await- {- if let Ok(sonic::Response::Content(body)) = connection- .send(Request::RetrieveWebites {- websites: pointers.to_vec(),- query: original_query.to_string(),- })- .await- {- return Ok(body);- }- }- }-- Err(Error::SearchFailed)- }-- async fn retrieve_websites_prettified(- &self,- pointers: &[inverted_index::WebsitePointer],- original_query: &str,) -> Result<Vec<DisplayedWebpage>> {for timeout in ExponentialBackoff::from_millis(30).take(5) {if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await@@ -173,53 +131,15 @@ impl Shard {}}- async fn search_prettified(- &self,- query: &SearchQuery,- ) -> Result<InitialPrettifiedSearchResultShard> {- match self- .replicas- .iter()- .map(|remote| remote.search_prettified(query))- .collect::<FuturesUnordered<_>>()- .next()- .await- {- Some(result) => Ok(InitialPrettifiedSearchResultShard {- local_result: result?,- shard: self.id.clone(),- }),- None => Err(Error::SearchFailed),- }- }-async fn retrieve_websites(&self,pointers: &[inverted_index::WebsitePointer],original_query: &str,- ) -> Result<Vec<RetrievedWebpage>> {- match self- .replicas- .iter()- .map(|remote| remote.retrieve_websites(pointers, original_query))- .collect::<FuturesUnordered<_>>()- .next()- .await- {- Some(Ok(websites)) => Ok(websites),- _ => Err(Error::SearchFailed),- }- }-- async fn retrieve_websites_prettified(- &self,- pointers: &[inverted_index::WebsitePointer],- original_query: &str,) -> Result<Vec<DisplayedWebpage>> {match self.replicas.iter()- .map(|remote| remote.retrieve_websites_prettified(pointers, original_query))+ .map(|remote| remote.retrieve_websites(pointers, original_query)).collect::<FuturesUnordered<_>>().next().await@@ -231,11 +151,6 @@ impl Shard {}struct InitialSearchResultShard {- local_result: InitialSearchResult,- shard: ShardId,-}--struct InitialPrettifiedSearchResultShard {local_result: InitialPrettifiedSearchResult,shard: ShardId,}@@ -260,21 +175,27 @@ pub struct DistributedSearcher {#[derive(Clone)]struct ScoredWebsitePointer {- local_pointer: inverted_index::WebsitePointer,+ website: RankingWebsite,shard: ShardId,}+impl AsRankingWebsite for ScoredWebsitePointer {+ fn as_ranking(&self) -> &RankingWebsite {+ &self.website+ }+}+impl collector::Doc for ScoredWebsitePointer {fn score(&self) -> &f64 {- &self.local_pointer.score.total+ &self.website.pointer.score.total}fn id(&self) -> &tantivy::DocId {- &self.local_pointer.address.doc_id+ &self.website.pointer.address.doc_id}fn hashes(&self) -> collector::Hashes {- self.local_pointer.hashes+ self.website.pointer.hashes}}@@ -283,139 +204,22 @@ impl DistributedSearcher {Self { shards }}- pub async fn search_api(&self, query: &SearchQuery) -> Result<SearchResult> {+ pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {let start = Instant::now();if query.is_empty() {return Err(Error::EmptyQuery);}- // search shards- let initial_results = self- .shards- .iter()- .map(|shard| shard.search(query))- .collect::<FuturesUnordered<_>>()- .collect::<Vec<_>>()- .await- .into_iter()- .filter_map(|result| result.ok())- .collect::<Vec<_>>();-- // check if any result has a bang hit- if let Some(result) = initial_results- .iter()- .find(|result| matches!(result.local_result, InitialSearchResult::Bang(_)))- {- if let InitialSearchResult::Bang(bang) = &result.local_result {- return Ok(SearchResult::Bang(bang.clone()));- }- }-- let spell_corrected_query = initial_results.first().and_then(|result| {- if let InitialSearchResult::Websites(result) = &result.local_result {- result.spell_corrected_query.clone()- } else {- None- }- });-- let sidebar = initial_results.first().and_then(|result| {- if let InitialSearchResult::Websites(result) = &result.local_result {- result.sidebar.clone()- } else {- None- }- });-- let num_docs = initial_results- .iter()- .map(|result| {- if let InitialSearchResult::Websites(result) = &result.local_result {- result.websites.num_websites- } else {- 0- }- })- .sum();-- // combine results- let top_n = NUM_RESULTS_PER_PAGE + query.skip_pages.unwrap_or(0);- let mut collector = BucketCollector::new(top_n);-- for result in initial_results {- if let InitialSearchResult::Websites(local_result) = result.local_result {- for website in local_result.websites.top_websites {- let pointer = ScoredWebsitePointer {- local_pointer: website,- shard: result.shard.clone(),- };-- collector.insert(pointer);- }- }- }-- let top_websites = collector- .into_sorted_vec(true)- .into_iter()- .skip(query.skip_pages.unwrap_or(0))- .take(NUM_RESULTS_PER_PAGE)- .collect::<Vec<_>>();-- // retrieve webpages- let mut retrieved_webpages = Vec::new();-- for _ in 0..top_websites.len() {- retrieved_webpages.push(None);- }-- for shard in &self.shards {- let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites- .iter()- .enumerate()- .filter(|(_, pointer)| pointer.shard == shard.id)- .map(|(idx, pointer)| (idx, pointer.local_pointer.clone()))- .unzip();-- if let Ok(websites) = shard.retrieve_websites(&pointers, &query.original).await {- for (index, website) in indexes.into_iter().zip(websites.into_iter()) {- retrieved_webpages[index] = Some(website);- }- }- }-- let retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().flatten().collect();-- debug_assert_eq!(retrieved_webpages.len(), top_websites.len());-- if retrieved_webpages.is_empty() && !top_websites.is_empty() {- return Err(Error::SearchFailed);- }-- Ok(SearchResult::Websites(WebsitesResult {- spell_corrected_query,- webpages: inverted_index::SearchResult {- num_docs,- documents: retrieved_webpages,- },- sidebar,- search_duration_ms: start.elapsed().as_millis(),- }))- }-- pub async fn search_prettified(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {- let start = Instant::now();-- if query.is_empty() {- return Err(Error::EmptyQuery);- }+ let mut search_query = query.clone();+ let pipeline: RankingPipeline<ScoredWebsitePointer> =+ RankingPipeline::for_query(&mut search_query);// search shardslet initial_results = self.shards.iter()- .map(|shard| shard.search_prettified(query))+ .map(|shard| shard.search(&search_query)).collect::<FuturesUnordered<_>>().collect::<Vec<_>>().await@@ -453,7 +257,7 @@ impl DistributedSearcher {.iter().map(|result| {if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {- result.websites.num_websites+ result.num_websites} else {0}@@ -461,14 +265,14 @@ impl DistributedSearcher {.sum();// combine results- let top_n = NUM_RESULTS_PER_PAGE + query.skip_pages.unwrap_or(0);- let mut collector = BucketCollector::new(top_n);+ let mut collector =+ BucketCollector::new(pipeline.collector_top_n() + pipeline.collector_offset());for result in initial_results {if let InitialPrettifiedSearchResult::Websites(local_result) = result.local_result {- for website in local_result.websites.top_websites {+ for website in local_result.websites {let pointer = ScoredWebsitePointer {- local_pointer: website,+ website,shard: result.shard.clone(),};@@ -480,10 +284,12 @@ impl DistributedSearcher {let top_websites = collector.into_sorted_vec(true).into_iter()- .skip(query.skip_pages.unwrap_or(0))- .take(NUM_RESULTS_PER_PAGE)+ .skip(pipeline.collector_offset())+ .take(pipeline.collector_top_n()).collect::<Vec<_>>();+ let top_websites = pipeline.apply(top_websites);+// retrieve webpageslet mut retrieved_webpages = Vec::new();@@ -496,13 +302,10 @@ impl DistributedSearcher {.iter().enumerate().filter(|(_, pointer)| pointer.shard == shard.id)- .map(|(idx, pointer)| (idx, pointer.local_pointer.clone()))+ .map(|(idx, pointer)| (idx, pointer.website.pointer.clone())).unzip();- if let Ok(websites) = shard- .retrieve_websites_prettified(&pointers, &query.original)- .await- {+ if let Ok(websites) = shard.retrieve_websites(&pointers, &query.original).await {for (index, website) in indexes.into_iter().zip(websites.into_iter()) {retrieved_webpages[index] = Some(website);}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex fa06b391..0983ad26 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -28,6 +28,7 @@ use crate::index::Index;use crate::query::Query;use crate::ranking::centrality_store::CentralityStore;use crate::ranking::optics::CreateAggregator;+use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};use crate::ranking::{Ranker, SignalAggregator};use crate::spell::Correction;use crate::webgraph::centrality::topic::TopicCentrality;@@ -35,9 +36,7 @@ use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};-use super::{- InitialSearchResult, SearchQuery, SearchResult, Sidebar, WebsitesResult, NUM_RESULTS_PER_PAGE,-};+use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar, WebsitesResult};const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;@@ -103,9 +102,7 @@ impl LocalSearcher {&query_aggregator,)?;- if let Some(num_results) = query.custom_num_results {- parsed_query.set_num_results(num_results)- }+ parsed_query.set_num_results(query.num_results);if parsed_query.is_empty() {Err(Error::EmptyQuery)@@ -118,7 +115,7 @@ impl LocalSearcher {&self,query: &SearchQuery,de_rank_similar: bool,- ) -> Result<inverted_index::InitialSearchResult> {+ ) -> Result<(Vec<RankingWebsite>, usize)> {let optic = match query.optic_program.as_ref() {Some(program) => Some(optics::parse(program)?),None => None,@@ -181,13 +178,19 @@ impl LocalSearcher {ranker = ranker.with_max_docs(10_000_000, self.index.num_segments())- .with_num_results(parsed_query.num_results());+ .with_num_results(parsed_query.num_results())+ .with_offset(query.offset);- if let Some(skip_pages) = query.skip_pages {- ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);- }+ let res = self+ .index+ .search_initial(&parsed_query, ranker.collector())?;++ let ranking_websites = self+ .index+ .inverted_index+ .retrieve_ranking_websites(res.top_websites, &ranker.aggregator())?;- self.index.search_initial(&parsed_query, ranker.collector())+ Ok((ranking_websites, res.num_websites))}fn spell_correction(&self, query: &SearchQuery) -> Result<Option<Correction>> {@@ -215,14 +218,14 @@ impl LocalSearcher {fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {let mut query = query.clone();query.optic_program = Some(include_str!("stackoverflow.optic").to_string());- query.custom_num_results = Some(1);+ query.num_results = 1;- let mut res = self.search_inverted_index(&query, false)?;+ let (mut top_websites, num_websites) = self.search_inverted_index(&query, false)?;- if res.num_websites > 0 && !res.top_websites.is_empty() {- let top = res.top_websites.remove(0);- if top.score.total > self.stackoverflow_sidebar_threshold {- let mut retrieved = self.retrieve_websites(&[top], &query.original)?;+ if num_websites > 0 && !top_websites.is_empty() {+ let top = top_websites.remove(0);+ if top.score > self.stackoverflow_sidebar_threshold {+ let mut retrieved = self.retrieve_websites(&[top.pointer], &query.original)?;let retrieved = retrieved.remove(0);return Ok(Some(Sidebar::StackOverflow {schema_org: retrieved.schema_org,@@ -255,13 +258,14 @@ impl LocalSearcher {return Ok(InitialSearchResult::Bang(bang));}- let webpages = self.search_inverted_index(query, de_rank_similar)?;+ let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;let correction = self.spell_correction(query)?;let sidebar = self.sidebar(query)?;Ok(InitialSearchResult::Websites(InitialWebsiteResult {spell_corrected_query: correction,- websites: webpages,+ websites,+ num_websites,sidebar,}))}@@ -288,18 +292,25 @@ impl LocalSearcher {pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {let start = Instant::now();- let query_text = query.original.clone();- let initial_result = self.search_initial(query, true)?;+ let mut search_query = query.clone();++ let pipeline = RankingPipeline::for_query(&mut search_query);++ let initial_result = self.search_initial(&search_query, true)?;match initial_result {InitialSearchResult::Websites(search_result) => {- let retrieved_sites =- self.retrieve_websites(&search_result.websites.top_websites, &query_text)?;+ let top_websites = pipeline.apply(search_result.websites);+ let pointers: Vec<_> = top_websites+ .into_iter()+ .map(|website| website.pointer)+ .collect();+ let retrieved_sites = self.retrieve_websites(&pointers, &search_query.original)?;Ok(SearchResult::Websites(WebsitesResult {spell_corrected_query: search_result.spell_corrected_query,webpages: inverted_index::SearchResult {- num_docs: search_result.websites.num_websites,+ num_docs: search_result.num_websites,documents: retrieved_sites,},sidebar: search_result.sidebar,@@ -337,7 +348,8 @@ impl LocalSearcher {#[derive(Debug, Serialize, Deserialize)]pub struct InitialWebsiteResult {pub spell_corrected_query: Option<Correction>,- pub websites: inverted_index::InitialSearchResult,+ pub num_websites: usize,+ pub websites: Vec<RankingWebsite>,pub sidebar: Option<Sidebar>,}@@ -354,13 +366,16 @@ impl SearchResult {#[cfg(test)]mod tests {- use crate::webpage::{Html, Webpage};+ use crate::{+ searcher::NUM_RESULTS_PER_PAGE,+ webpage::{Html, Webpage},+ };use super::*;#[test]fn offset_page() {- const NUM_PAGES: usize = 10;+ const NUM_PAGES: usize = 50;const NUM_WEBSITES: usize = NUM_PAGES * NUM_RESULTS_PER_PAGE;let mut index = Index::temporary().expect("Unable to open index");@@ -403,7 +418,7 @@ mod tests {let urls: Vec<_> = searcher.search(&SearchQuery {original: "test".to_string(),- skip_pages: Some(p),+ offset: p * NUM_RESULTS_PER_PAGE,..Default::default()}).unwrap()@@ -417,6 +432,8 @@ mod tests {assert!(!urls.is_empty());+ dbg!(&urls);+for (i, url) in urls.into_iter().enumerate() {assert_eq!(url,diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 5f4ab6fd..ce96059d 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -83,14 +83,27 @@ pub enum InitialPrettifiedSearchResult {Bang(BangHit),}-#[derive(Debug, Serialize, Deserialize, Clone, Default)]+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct SearchQuery {pub original: String,+ pub offset: usize,+ pub num_results: usize,pub selected_region: Option<Region>,pub optic_program: Option<String>,- pub skip_pages: Option<usize>,pub site_rankings: Option<SiteRankings>,- pub custom_num_results: Option<usize>,+}++impl Default for SearchQuery {+ fn default() -> Self {+ Self {+ original: Default::default(),+ offset: Default::default(),+ num_results: NUM_RESULTS_PER_PAGE,+ selected_region: Default::default(),+ optic_program: Default::default(),+ site_rankings: Default::default(),+ }+ }}impl SearchQuery {