web search done right
[api-searcher] refactor functionality into helper structs to keep the searcher a bit more clean
Mikkel Denker9dce1a3931c7
diff --git a/core/src/bangs.rs b/core/src/bangs.rsindex 7ac34e7d..70b3b57e 100644--- a/core/src/bangs.rs+++ b/core/src/bangs.rs@@ -28,7 +28,7 @@ use utoipa::ToSchema;use crate::query::parser::Term;-pub const BANG_PREFIX: char = '!';+pub const BANG_PREFIXES: [char; 2] = ['!', '!'];#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]#[serde(rename_all = "camelCase")]diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 0f1f2840..b2b2f43f 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -147,6 +147,29 @@ impl IndexingWorker {.into_iter().map(|edge| edge.label).filter(|label| !label.is_empty())+ .filter(|label| {+ let label = label.to_lowercase();+ let stopwords = [+ "click",+ "click here",+ "here",+ "link",+ "website",+ "webpage",+ "page",+ "site",+ "url",+ "web",+ "visit",+ "more",+ "info",+ "information",+ "read",+ "read more",+ ];++ !stopwords.contains(&label.as_str())+ }).collect()}).unwrap_or_default();diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 444f0235..56c6abc2 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -589,6 +589,7 @@ pub struct RetrievedWebpage {pub region: Region,pub likely_has_ads: bool,pub likely_has_paywall: bool,+ pub recipe_first_ingredient_tag_id: Option<String>,}impl RetrievedWebpage {pub fn description(&self) -> Option<&String> {@@ -688,6 +689,18 @@ impl From<TantivyDocument> for RetrievedWebpage {webpage.likely_has_paywall =value.value().as_value().as_u64().unwrap_or_default() != 0;}+ Field::Text(TextField::RecipeFirstIngredientTagId) => {+ let tag_id = value+ .value()+ .as_value()+ .as_str()+ .expect("Recipe first ingredient tag id field should be stored as text")+ .to_string();++ if !tag_id.is_empty() {+ webpage.recipe_first_ingredient_tag_id = Some(tag_id);+ }+ }_ => {}}}diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex f621040c..c875889a 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -1084,7 +1084,9 @@ mod tests {let res = searcher.search(&SearchQuery {query: "avengers endgame".to_string(),- optic: Some(Optic::parse(include_str!("../searcher/discussions.optic")).unwrap()),+ optic: Some(+ Optic::parse(include_str!("../searcher/api/discussions.optic")).unwrap(),+ ),..Default::default()}).unwrap()diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex c9a478f5..18b70777 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -20,7 +20,7 @@ use tantivy::{};use crate::{- bangs::BANG_PREFIX,+ bangs::BANG_PREFIXES,floor_char_boundary,schema::{Field, TextField, ALL_FIELDS},};@@ -87,7 +87,7 @@ impl ToString for Term {Term::Title(title) => "intitle:".to_string() + title.as_str(),Term::Body(body) => "inbody:".to_string() + body.as_str(),Term::Url(url) => "inurl:".to_string() + url.as_str(),- Term::PossibleBang(bang) => BANG_PREFIX.to_string() + bang.as_str(),+ Term::PossibleBang(bang) => BANG_PREFIXES[0].to_string() + bang.as_str(),}}}@@ -233,7 +233,7 @@ impl Term {Term::PossibleBang(text) => {let mut term = String::new();- term.push(BANG_PREFIX);+ term.push(BANG_PREFIXES[0]);term.push_str(text);simple_into_tantivy(&term.into(), &[], fields)@@ -372,9 +372,13 @@ fn parse_term(term: &str) -> Box<Term> {} else {Box::new(Term::Simple(term.to_string().into()))}- } else if let Some(bang) = term.strip_prefix(BANG_PREFIX) {- Box::new(Term::PossibleBang(bang.to_string()))} else {+ for bang_prefix in BANG_PREFIXES {+ if let Some(bang) = term.strip_prefix(bang_prefix) {+ return Box::new(Term::PossibleBang(bang.to_string()));+ }+ }+Box::new(Term::Simple(term.to_string().into()))}}diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 3970de21..06d55e93 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -22,7 +22,7 @@ use crate::{collector::{self, BucketCollector},config::CollectorConfig,enum_map::EnumMap,- inverted_index::WebsitePointer,+ inverted_index::{RetrievedWebpage, WebsitePointer},searcher::SearchQuery,Result,};@@ -72,12 +72,44 @@ impl lambdamart::AsValue for SignalScore {}}+#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct RetrievedWebpageRanking {+ retrieved_webpage: RetrievedWebpage,+ ranking: RankingWebsite,+}++impl RetrievedWebpageRanking {+ pub fn new(retrieved_webpage: RetrievedWebpage, ranking: RankingWebsite) -> Self {+ let mut ranking = ranking;+ ranking.snippet = Some(retrieved_webpage.snippet.unhighlighted_string());++ Self {+ retrieved_webpage,+ ranking,+ }+ }++ pub fn into_retrieved_webpage(self) -> RetrievedWebpage {+ self.retrieved_webpage+ }+}++impl AsRankingWebsite for RetrievedWebpageRanking {+ fn as_ranking(&self) -> &RankingWebsite {+ &self.ranking+ }++ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {+ &mut self.ranking+ }+}+#[derive(Clone, Debug, Serialize, Deserialize)]pub struct RankingWebsite {pub pointer: WebsitePointer,pub signals: EnumMap<Signal, SignalScore>,pub title: Option<String>,- pub clean_body: Option<String>,+ pub snippet: Option<String>,pub optic_boost: Option<f64>,pub score: f64,}@@ -87,9 +119,9 @@ impl RankingWebsite {let mut res = RankingWebsite {signals: EnumMap::new(),title: None,- clean_body: None,score: pointer.score.total,optic_boost: None,+ snippet: None,pointer: pointer.clone(),};@@ -129,36 +161,51 @@ impl<M: CrossEncoder> ReRanker<M> {}fn crossencoder_score_websites<T: AsRankingWebsite>(&self, websites: &mut [T]) {- let mut bodies = Vec::with_capacity(websites.len());+ let mut snippets = Vec::with_capacity(websites.len());+ let mut titles = Vec::with_capacity(websites.len());for website in websites.iter_mut() {- let website = website.as_mut_ranking();- let title = website.title.clone().unwrap_or_default();- let body = website.clean_body.clone().unwrap_or_default();- let text = title + ". " + body.as_str();- bodies.push(text);+ titles.push(website.as_mut_ranking().title.clone().unwrap_or_default());+ snippets.push(website.as_mut_ranking().snippet.clone().unwrap_or_default());}let query = &self.query.as_ref().unwrap().query;- let scores = self.crossencoder.run(query, &bodies);+ let snippet_scores = self.crossencoder.run(query, &snippets);+ let title_scores = self.crossencoder.run(query, &titles);- for (website, score) in websites.iter_mut().zip(scores) {+ for ((website, snippet), title) in websites.iter_mut().zip(snippet_scores).zip(title_scores)+ {let website = website.as_mut_ranking();website.signals.insert(- Signal::CrossEncoder,+ Signal::CrossEncoderSnippet,+ SignalScore {+ coefficient: self.crossencoder_snippet_coeff(),+ value: snippet,+ },+ );++ website.signals.insert(+ Signal::CrossEncoderTitle,SignalScore {- coefficient: self.crossencoder_coeff(),- value: score,+ coefficient: self.crossencoder_title_coeff(),+ value: title,},);}}- fn crossencoder_coeff(&self) -> f64 {+ fn crossencoder_snippet_coeff(&self) -> f64 {self.signal_coefficients.as_ref()- .and_then(|coeffs| coeffs.get(&Signal::CrossEncoder))- .unwrap_or(Signal::CrossEncoder.default_coefficient())+ .and_then(|coeffs| coeffs.get(&Signal::CrossEncoderSnippet))+ .unwrap_or(Signal::CrossEncoderSnippet.default_coefficient())+ }++ fn crossencoder_title_coeff(&self) -> f64 {+ self.signal_coefficients+ .as_ref()+ .and_then(|coeffs| coeffs.get(&Signal::CrossEncoderSnippet))+ .unwrap_or(Signal::CrossEncoderSnippet.default_coefficient())}}@@ -293,10 +340,11 @@ pub struct RankingPipeline<T: AsRankingWebsite> {}impl<T: AsRankingWebsite> RankingPipeline<T> {- fn create_reranking<M: CrossEncoder + 'static>(+ fn create_second_stage<M: CrossEncoder + 'static>(crossencoder: Option<Arc<M>>,lambda: Option<Arc<LambdaMART>>,collector_config: CollectorConfig,+ top_n_considered: usize,) -> Result<Self> {let scorer = match crossencoder {Some(cross_encoder) => {@@ -307,7 +355,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {let stage = RankingStage {scorer,- stage_top_n: 20,+ stage_top_n: top_n_considered,derank_similar: true,};@@ -319,25 +367,31 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {})}- pub fn reranking_for_query<M: CrossEncoder + 'static>(+ pub fn reranker<M: CrossEncoder + 'static>(query: &mut SearchQuery,crossencoder: Option<Arc<M>>,lambda: Option<Arc<LambdaMART>>,collector_config: CollectorConfig,+ top_n_considered: usize,) -> Result<Self> {- let mut pipeline = Self::create_reranking(crossencoder, lambda, collector_config)?;+ let mut pipeline =+ Self::create_second_stage(crossencoder, lambda, collector_config, top_n_considered)?;pipeline.set_query_info(query);Ok(pipeline)}- fn create_ltr(model: Option<Arc<LambdaMART>>, collector_config: CollectorConfig) -> Self {+ fn create_first_stage(+ model: Option<Arc<LambdaMART>>,+ collector_config: CollectorConfig,+ stage_top_n: usize,+ ) -> Self {let last_stage = RankingStage {scorer: Box::new(Initial {model,signal_coefficients: None,}),- stage_top_n: 100,+ stage_top_n,derank_similar: true,};@@ -349,12 +403,13 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}}- pub fn initial_for_query(+ pub fn first_stage(query: &mut SearchQuery,model: Option<Arc<LambdaMART>>,collector_config: CollectorConfig,+ top_n_considered: usize,) -> Self {- let mut pipeline = Self::create_ltr(model, collector_config);+ let mut pipeline = Self::create_first_stage(model, collector_config, top_n_considered);pipeline.set_query_info(query);pipeline@@ -436,7 +491,7 @@ mod tests {signals,optic_boost: None,title: None,- clean_body: None,+ snippet: None,score: 1.0 / i as f64,}})@@ -445,13 +500,14 @@ mod tests {#[test]fn simple() {- let pipeline = RankingPipeline::reranking_for_query(+ let pipeline = RankingPipeline::reranker(&mut SearchQuery {..Default::default()},Some(Arc::new(DummyCrossEncoder {})),None,CollectorConfig::default(),+ 20,).unwrap();assert_eq!(pipeline.collector_top_n(), 20);@@ -475,7 +531,7 @@ mod tests {#[test]fn top_n() {let num_results = 100;- let pipeline = RankingPipeline::reranking_for_query(+ let pipeline = RankingPipeline::reranker(&mut SearchQuery {num_results,..Default::default()@@ -483,6 +539,7 @@ mod tests {Some(Arc::new(DummyCrossEncoder {})),None,CollectorConfig::default(),+ num_results,).unwrap();@@ -508,7 +565,7 @@ mod tests {#[test]fn offsets() {let num_results = 20;- let pipeline = RankingPipeline::reranking_for_query(+ let pipeline = RankingPipeline::reranker(&mut SearchQuery {page: 0,num_results,@@ -517,13 +574,14 @@ mod tests {Some(Arc::new(DummyCrossEncoder {})),None,CollectorConfig::default(),+ num_results,).unwrap();let sample: Vec<_> = sample_websites(pipeline.collector_top_n());let mut prev: Vec<_> = pipeline.apply(sample);for p in 1..1_000 {- let pipeline = RankingPipeline::reranking_for_query(+ let pipeline = RankingPipeline::reranker(&mut SearchQuery {page: p,..Default::default()@@ -531,6 +589,7 @@ mod tests {Some(Arc::new(DummyCrossEncoder {})),None,CollectorConfig::default(),+ num_results,).unwrap();diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex f6c556df..c769a502 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -98,8 +98,10 @@ pub enum Signal {Bm25TitleIfHomepage,#[serde(rename = "bm25_backlink_text")]Bm25BacklinkText,- #[serde(rename = "cross_encoder")]- CrossEncoder,+ #[serde(rename = "cross_encoder_snippet")]+ CrossEncoderSnippet,+ #[serde(rename = "cross_encoder_title")]+ CrossEncoderTitle,#[serde(rename = "host_centrality")]HostCentrality,#[serde(rename = "page_centrality")]@@ -134,7 +136,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 34] = [+pub const ALL_SIGNALS: [Signal; 35] = [Signal::Bm25Title,Signal::Bm25TitleBigrams,Signal::Bm25TitleTrigrams,@@ -155,7 +157,8 @@ pub const ALL_SIGNALS: [Signal; 34] = [Signal::Bm25DomainIfHomepageNoTokenizer,Signal::Bm25TitleIfHomepage,Signal::Bm25BacklinkText,- Signal::CrossEncoder,+ Signal::CrossEncoderSnippet,+ Signal::CrossEncoderTitle,Signal::HostCentrality,Signal::PageCentrality,Signal::IsHomepage,@@ -278,7 +281,8 @@ impl Signal {Signal::Bm25DomainIfHomepageNoTokenizer => 0.0036,Signal::Bm25TitleIfHomepage => 0.00022,Signal::Bm25BacklinkText => 0.003,- Signal::CrossEncoder => 0.17,+ Signal::CrossEncoderSnippet => 0.17,+ Signal::CrossEncoderTitle => 0.17,Signal::HostCentrality => 0.5,Signal::PageCentrality => 0.25,Signal::QueryCentrality => 0.0,@@ -453,7 +457,8 @@ impl Signal {.unwrap_or(0.0)}),- Signal::CrossEncoder => None, // this is calculated in a later step+ Signal::CrossEncoderSnippet => None, // this is calculated in a later step+ Signal::CrossEncoderTitle => None, // this is calculated in a later stepSignal::LambdaMART => None,};@@ -553,7 +558,8 @@ impl Signal {| Signal::Bm25DomainIfHomepageNoTokenizer| Signal::Bm25TitleIfHomepage| Signal::Bm25BacklinkText- | Signal::CrossEncoder+ | Signal::CrossEncoderSnippet+ | Signal::CrossEncoderTitle| Signal::InboundSimilarity| Signal::LambdaMART| Signal::QueryCentrality => {diff --git a/core/src/schema.rs b/core/src/schema.rsindex 5db77aa3..63b87caa 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -60,6 +60,7 @@ pub enum TextField {/// can either be NSFW or SFW (see safety classifier)SafetyClassification,InsertionTimestamp,+ RecipeFirstIngredientTagId,}impl From<TextField> for usize {@@ -111,6 +112,7 @@ impl TextField {TextField::MicroformatTags => Tokenizer::default(),TextField::SafetyClassification => Tokenizer::Identity(Identity {}),TextField::InsertionTimestamp => Tokenizer::Identity(Identity {}),+ TextField::RecipeFirstIngredientTagId => Tokenizer::Identity(Identity {}),}}@@ -154,6 +156,7 @@ impl TextField {TextField::MicroformatTags => true,TextField::SafetyClassification => false,TextField::InsertionTimestamp => false,+ TextField::RecipeFirstIngredientTagId => false,}}@@ -189,6 +192,7 @@ impl TextField {TextField::MicroformatTags => "microformat_tags",TextField::SafetyClassification => "safety_classification",TextField::InsertionTimestamp => "insertion_timestamp",+ TextField::RecipeFirstIngredientTagId => "recipe_first_ingredient_tag_id",}}}@@ -459,6 +463,9 @@ impl Field {Field::Text(TextField::SafetyClassification) => {IndexingOption::Text(self.default_text_options())}+ Field::Text(TextField::RecipeFirstIngredientTagId) => {+ IndexingOption::Text(self.default_text_options().set_stored())+ }Field::Text(TextField::InsertionTimestamp) => {IndexingOption::DateTime(tantivy::schema::DateOptions::default().set_indexed())}@@ -634,6 +641,7 @@ impl Field {| Field::Text(TextField::SiteWithout) // will match url| Field::Text(TextField::Domain) // will match url| Field::Text(TextField::InsertionTimestamp)+ | Field::Text(TextField::RecipeFirstIngredientTagId)) && !self.is_fast()}diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsdeleted file mode 100644index 85135ba2..00000000--- a/core/src/searcher/api.rs+++ /dev/null@@ -1,733 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::cmp::Ordering;-use std::collections::HashMap;-use std::ops::Range;-use std::sync::Arc;-use std::time::Instant;--use itertools::{intersperse, Itertools};-use optics::Optic;-use url::Url;--use crate::bangs::{Bang, BangHit};-use crate::config::{ApiConfig, ApiThresholds, CollectorConfig};-use crate::image_store::Image;-use crate::inverted_index::RetrievedWebpage;-#[cfg(not(feature = "libtorch"))]-use crate::ranking::models::cross_encoder::DummyCrossEncoder;-use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite};-use crate::ranking::ALL_SIGNALS;-use crate::search_prettifier::{- create_stackoverflow_sidebar, DisplayedAnswer, DisplayedEntity, DisplayedSidebar,- DisplayedWebpage, HighlightedSpellCorrection,-};-use crate::web_spell::SpellChecker;-use crate::widgets::{Widget, Widgets};-use crate::{- bangs::Bangs,- collector::BucketCollector,- distributed::cluster::Cluster,- ranking::{models::lambdamart::LambdaMART, pipeline::RankingPipeline},-};-use crate::{ceil_char_boundary, floor_char_boundary, query, Result};-#[cfg(feature = "libtorch")]-use crate::{qa_model::QaModel, ranking::models::cross_encoder::CrossEncoderModel};--use super::live::LiveSearcher;-use super::{distributed, live, DistributedSearcher, SearchQuery, SearchResult, WebsitesResult};--#[derive(Clone)]-enum ScoredWebsitePointer {- Normal(distributed::ScoredWebsitePointer),- Live(live::ScoredWebsitePointer),-}--impl AsRankingWebsite for ScoredWebsitePointer {- fn as_ranking(&self) -> &RankingWebsite {- match self {- ScoredWebsitePointer::Normal(p) => &p.website,- ScoredWebsitePointer::Live(p) => &p.website,- }- }-- fn as_mut_ranking(&mut self) -> &mut RankingWebsite {- match self {- ScoredWebsitePointer::Normal(p) => &mut p.website,- ScoredWebsitePointer::Live(p) => &mut p.website,- }- }-}--#[cfg(feature = "libtorch")]-pub struct ApiSearcher {- distributed_searcher: DistributedSearcher,- live_searcher: LiveSearcher,- cross_encoder: Option<Arc<CrossEncoderModel>>,- lambda_model: Option<Arc<LambdaMART>>,- qa_model: Option<Arc<QaModel>>,- bangs: Bangs,- collector_config: CollectorConfig,- thresholds: ApiThresholds,- widgets: Widgets,- spell_checker: Option<SpellChecker>,-}--#[cfg(not(feature = "libtorch"))]-pub struct ApiSearcher {- distributed_searcher: DistributedSearcher,- live_searcher: LiveSearcher,- lambda_model: Option<Arc<LambdaMART>>,- bangs: Bangs,- collector_config: CollectorConfig,- thresholds: ApiThresholds,- widgets: Widgets,- spell_checker: Option<SpellChecker>,-}--impl ApiSearcher {- #[cfg(feature = "libtorch")]- pub fn new(- cluster: Arc<Cluster>,- cross_encoder: Option<CrossEncoderModel>,- lambda_model: Option<LambdaMART>,- qa_model: Option<QaModel>,- bangs: Bangs,- config: ApiConfig,- ) -> Self {- Self {- distributed_searcher: DistributedSearcher::new(Arc::clone(&cluster)),- live_searcher: LiveSearcher::new(Arc::clone(&cluster)),- cross_encoder: cross_encoder.map(Arc::new),- lambda_model: lambda_model.map(Arc::new),- qa_model: qa_model.map(Arc::new),- bangs,- collector_config: config.collector,- thresholds: config.thresholds,- widgets: Widgets::new(config.widgets).unwrap(),- spell_checker: config- .spell_checker_path- .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),- }- }-- #[cfg(not(feature = "libtorch"))]- pub fn new(- cluster: Arc<Cluster>,- lambda_model: Option<LambdaMART>,- bangs: Bangs,- config: ApiConfig,- ) -> Self {- Self {- distributed_searcher: DistributedSearcher::new(Arc::clone(&cluster)),- live_searcher: LiveSearcher::new(Arc::clone(&cluster)),- lambda_model: lambda_model.map(Arc::new),- bangs,- collector_config: config.collector,- thresholds: config.thresholds,- widgets: Widgets::new(config.widgets).unwrap(),- spell_checker: config- .spell_checker_path- .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),- }- }-- fn combine_results(- &self,- initial_results: Vec<distributed::InitialSearchResultShard>,- live_results: Vec<live::InitialSearchResultSplit>,- pipeline: RankingPipeline<ScoredWebsitePointer>,- ) -> (Vec<ScoredWebsitePointer>, bool) {- let mut collector =- BucketCollector::new(pipeline.collector_top_n(), self.collector_config.clone());-- let mut num_sites: usize = 0;- for result in initial_results {- for website in result.local_result.websites {- num_sites += 1;- let pointer = distributed::ScoredWebsitePointer {- website,- shard: result.shard,- };-- let pointer = ScoredWebsitePointer::Normal(pointer);-- collector.insert(pointer);- }- }-- for result in live_results {- for website in result.local_result.websites {- num_sites += 1;- let pointer = live::ScoredWebsitePointer {- website,- split_id: result.split_id.clone(),- };-- let pointer = ScoredWebsitePointer::Live(pointer);-- collector.insert(pointer);- }- }-- let top_websites = collector- .into_sorted_vec(true)- .into_iter()- .take(pipeline.collector_top_n())- .collect::<Vec<_>>();-- let offset = pipeline.offset();-- let res = pipeline.apply(top_websites);- let has_more = num_sites.saturating_sub(offset) > res.len();-- (res, has_more)- }-- async fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<DisplayedSidebar>> {- let query = SearchQuery {- query: query.query.clone(),- num_results: 1,- optic: Some(Optic::parse(include_str!("stackoverflow.optic")).unwrap()),- ..Default::default()- };-- let mut results: Vec<_> = self- .distributed_searcher- .search_initial(&query)- .await- .into_iter()- .filter_map(|result| {- result- .local_result- .websites- .first()- .cloned()- .map(|website| (result.shard, website))- })- .collect();-- results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));-- if let Some((shard, website)) = results.pop() {- tracing::debug!(?website.score, ?self.thresholds.stackoverflow, "stackoverflow score");- if website.score > self.thresholds.stackoverflow {- let scored_websites =- vec![(0, distributed::ScoredWebsitePointer { website, shard })];- let mut retrieved = self- .distributed_searcher- .retrieve_webpages(&scored_websites, &query.query)- .await;-- if let Some((_, res)) = retrieved.pop() {- return Ok(Some(create_stackoverflow_sidebar(- res.schema_org,- Url::parse(&res.url).unwrap(),- )?));- }- }- }-- Ok(None)- }-- fn sidebar(- &self,- initial_results: &[distributed::InitialSearchResultShard],- stackoverflow: Option<DisplayedSidebar>,- ) -> Option<DisplayedSidebar> {- let entity = initial_results- .iter()- .filter_map(|res| res.local_result.entity_sidebar.clone())- .map(DisplayedEntity::from)- .filter(|entity| entity.match_score as f64 > self.thresholds.entity_sidebar)- .max_by(|a, b| {- a.match_score- .partial_cmp(&b.match_score)- .unwrap_or(Ordering::Equal)- });-- match entity {- Some(entity) => Some(DisplayedSidebar::Entity(entity)),- None => stackoverflow,- }- }-- async fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {- let parsed_terms = query::parser::parse(&query.query);-- if parsed_terms.iter().any(|term| match term.as_ref() {- query::parser::Term::PossibleBang(t) => t.is_empty(),- _ => false,- }) {- let q: String = intersperse(- parsed_terms- .iter()- .filter(|term| !matches!(term.as_ref(), query::parser::Term::PossibleBang(_)))- .map(|term| term.to_string()),- " ".to_string(),- )- .collect();-- let mut query = query.clone();- query.query = q;-- let res = self.search_websites(&query).await?;-- return Ok(res.webpages.first().map(|webpage| BangHit {- bang: Bang {- category: None,- sub_category: None,- domain: None,- ranking: None,- site: None,- tag: String::new(),- url: webpage.url.clone(),- },- redirect_to: Url::parse(&webpage.url).unwrap().into(),- }));- }-- Ok(self.bangs.get(&parsed_terms))- }-- async fn discussions_widget(- &self,- query: &SearchQuery,- ) -> Result<Option<Vec<DisplayedWebpage>>> {- if !query.fetch_discussions || query.optic.is_some() || query.page > 0 {- return Ok(None);- }-- const NUM_RESULTS: usize = 10;-- let mut query = SearchQuery {- query: query.query.clone(),- num_results: NUM_RESULTS,- optic: Some(Optic::parse(include_str!("discussions.optic")).unwrap()),- site_rankings: query.site_rankings.clone(),- return_ranking_signals: query.return_ranking_signals,- ..Default::default()- };-- #[cfg(feature = "libtorch")]- let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::reranking_for_query(- &mut query,- self.cross_encoder.as_ref().map(Arc::clone),- self.lambda_model.clone(),- self.collector_config.clone(),- )?;-- #[cfg(not(feature = "libtorch"))]- let pipeline: RankingPipeline<ScoredWebsitePointer> =- RankingPipeline::reranking_for_query::<DummyCrossEncoder>(- &mut query,- None,- self.lambda_model.clone(),- self.collector_config.clone(),- )?;-- let initial_results = self.distributed_searcher.search_initial(&query).await;-- if initial_results.is_empty() {- return Ok(None);- }-- let num_results: usize = initial_results- .iter()- .map(|res| res.local_result.websites.len())- .sum();-- if num_results < NUM_RESULTS / 2 {- return Ok(None);- }-- let (top_websites, _) = self.combine_results(initial_results, vec![], pipeline);-- let scores: Vec<_> = top_websites- .iter()- .map(|pointer| pointer.as_ranking().score)- .collect();-- let median = if scores.len() % 2 == 0 {- (scores[(scores.len() / 2) - 1] + scores[scores.len() / 2]) / 2.0- } else {- scores[scores.len() / 2]- };-- if median < self.thresholds.discussions_widget {- return Ok(None);- }-- let mut result: Vec<DisplayedWebpage> = self- .distributed_searcher- .retrieve_webpages(- &top_websites- .clone()- .into_iter()- .filter_map(|pointer| {- if let ScoredWebsitePointer::Normal(p) = pointer {- Some(p)- } else {- None- }- })- .enumerate()- .collect_vec(),- &query.query,- )- .await- .into_iter()- .map(|(_, webpage)| webpage)- .map(DisplayedWebpage::from)- .collect();-- if query.return_ranking_signals {- self.add_ranking_signals(&mut result, &top_websites);- }-- Ok(Some(result))- }-- fn add_ranking_signals(- &self,- websites: &mut [DisplayedWebpage],- pointers: &[ScoredWebsitePointer],- ) {- for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {- let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());-- for signal in ALL_SIGNALS {- if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {- signals.insert(signal, *signal_value);- }- }-- website.ranking_signals = Some(signals);- }- }-- async fn search_websites(&self, query: &SearchQuery) -> Result<WebsitesResult> {- let start = Instant::now();-- if query.is_empty() {- return Err(distributed::Error::EmptyQuery.into());- }-- let mut search_query = query.clone();- #[cfg(feature = "libtorch")]- let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::reranking_for_query(- &mut search_query,- self.cross_encoder.as_ref().map(Arc::clone),- self.lambda_model.clone(),- self.collector_config.clone(),- )?;-- #[cfg(not(feature = "libtorch"))]- let pipeline: RankingPipeline<ScoredWebsitePointer> =- RankingPipeline::reranking_for_query::<DummyCrossEncoder>(- &mut search_query,- None,- self.lambda_model.clone(),- self.collector_config.clone(),- )?;-- let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(- self.distributed_searcher.search_initial(&search_query),- self.live_searcher.search_initial(&search_query),- self.widget(query),- self.discussions_widget(query),- self.stackoverflow_sidebar(query),- );-- let discussions = discussions?;- let stackoverflow = stackoverflow?;- let sidebar = self.sidebar(&initial_results, stackoverflow);-- let spell_corrected_query = if widget.is_none() {- self.spell_checker- .as_ref()- .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))- .map(HighlightedSpellCorrection::from)- } else {- None- };-- let num_docs = initial_results- .iter()- .map(|result| result.local_result.num_websites)- .sum();-- let (top_websites, has_more_results) =- self.combine_results(initial_results, live_results, pipeline);-- // retrieve webpages- let normal: Vec<_> = top_websites- .iter()- .enumerate()- .filter_map(|(i, pointer)| {- if let ScoredWebsitePointer::Normal(p) = pointer {- Some((i, p.clone()))- } else {- None- }- })- .collect();-- let live: Vec<_> = top_websites- .iter()- .enumerate()- .filter_map(|(i, pointer)| {- if let ScoredWebsitePointer::Live(p) = pointer {- Some((i, p.clone()))- } else {- None- }- })- .collect();-- let (retrieved_normal, retrieved_live) = tokio::join!(- self.distributed_searcher- .retrieve_webpages(&normal, &query.query),- self.live_searcher.retrieve_webpages(&live, &query.query),- );-- let mut retrieved_webpages: Vec<_> =- retrieved_normal.into_iter().chain(retrieved_live).collect();- retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));-- let mut retrieved_webpages: Vec<_> = retrieved_webpages- .into_iter()- .map(|(_, webpage)| webpage)- .map(DisplayedWebpage::from)- .collect();-- if retrieved_webpages.len() != top_websites.len() {- return Err(distributed::Error::SearchFailed.into());- }-- if query.return_ranking_signals {- self.add_ranking_signals(&mut retrieved_webpages, &top_websites);- }-- let direct_answer = self.answer(&query.query, &mut retrieved_webpages);-- let search_duration_ms = start.elapsed().as_millis();-- Ok(WebsitesResult {- spell_corrected_query,- num_hits: num_docs,- webpages: retrieved_webpages,- direct_answer,- sidebar,- widget,- discussions,- search_duration_ms,- has_more_results,- })- }-- pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {- if let Some(bang) = self.check_bangs(query).await? {- return Ok(SearchResult::Bang(Box::new(bang)));- }-- Ok(SearchResult::Websites(self.search_websites(query).await?))- }-- async fn widget(&self, query: &SearchQuery) -> Option<Widget> {- if query.page > 0 {- return None;- }-- let parsed_terms = query::parser::parse(&query.query);-- self.widgets.widget(- parsed_terms- .into_iter()- .filter_map(|term| {- if let query::parser::Term::Simple(simple) = *term {- Some(String::from(simple))- } else {- None- }- })- .join(" ")- .as_str(),- )- }-- #[cfg(feature = "libtorch")]- fn answer(&self, query: &str, webpages: &mut Vec<DisplayedWebpage>) -> Option<DisplayedAnswer> {- self.qa_model.as_ref().and_then(|qa_model| {- let contexts: Vec<_> = webpages- .iter()- .take(1)- .filter_map(|webpage| webpage.snippet.text())- .map(|t| t.fragments.iter().map(|f| f.text()).join(""))- .collect();-- match qa_model.run(query, &contexts) {- Some(answer) => {- let answer_webpage = webpages.remove(answer.context_idx);- let snip = answer_webpage- .snippet- .text()- .unwrap()- .fragments- .iter()- .map(|f| f.text())- .join("");- Some(DisplayedAnswer {- title: answer_webpage.title,- url: answer_webpage.url,- pretty_url: answer_webpage.pretty_url,- snippet: generate_answer_snippet(&snip, answer.offset.clone()),- answer: snip[answer.offset].to_string(),- })- }- None => None,- }- })- }-- #[cfg(not(feature = "libtorch"))]- fn answer(&self, query: &str, webpages: &mut Vec<DisplayedWebpage>) -> Option<DisplayedAnswer> {- None- }-- pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- self.distributed_searcher.get_webpage(url).await- }-- pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {- self.distributed_searcher.get_entity_image(image_id).await- }-}--fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {- let mut best_start = 0;- let mut best_end = 0;- const SNIPPET_LENGTH: usize = 200;-- if body.is_empty() || answer_offset.start > body.len() - 1 {- return body.to_string();- }-- for (idx, _) in body.char_indices().filter(|(_, c)| *c == '.') {- if idx > answer_offset.end + SNIPPET_LENGTH {- break;- }-- if idx < answer_offset.start {- best_start = idx;- }-- if idx > answer_offset.end {- best_end = idx;- }- }-- if (answer_offset.end - best_start > SNIPPET_LENGTH) || (best_start >= best_end) {- if answer_offset.end - answer_offset.start >= SNIPPET_LENGTH {- let end = floor_char_boundary(body, answer_offset.start + SNIPPET_LENGTH);-- return "<b>".to_string() + &body[answer_offset.start..end] + "</b>";- }-- let chars_either_side = (SNIPPET_LENGTH - (answer_offset.end - answer_offset.start)) / 2;-- let start = ceil_char_boundary(- body,- answer_offset- .start- .checked_sub(chars_either_side)- .unwrap_or_default(),- );- let mut end = ceil_char_boundary(body, answer_offset.end + chars_either_side);-- if end >= body.len() {- end = floor_char_boundary(body, body.len());- }-- body[start..answer_offset.start].to_string()- + "<b>"- + &body[answer_offset.clone()]- + "</b>"- + &body[answer_offset.end..end]- } else {- let mut res = body[best_start..answer_offset.start].to_string()- + "<b>"- + &body[answer_offset.clone()]- + "</b>";-- let remaining_chars = SNIPPET_LENGTH - (res.len() - 7);- let end = ceil_char_boundary(body, (remaining_chars + answer_offset.end).min(best_end));-- res += &body[answer_offset.end..end];-- res- }-}--#[cfg(test)]-mod tests {- use super::*;- #[test]- fn test_generate_answer_snippet() {- assert_eq!(- generate_answer_snippet("this is a test", 0..4),- "<b>this</b> is a test".to_string()- );-- assert_eq!(- generate_answer_snippet("this is a test", 0..1000),- "<b>this is a test</b>".to_string()- );- assert_eq!(- generate_answer_snippet("this is a test", 1000..2000),- "this is a test".to_string()- );- let input = r#"-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test- "#;-- let res = generate_answer_snippet(input, 0..500);- assert!(!res.is_empty());- assert!(res.len() > 100);- assert!(res.len() < input.len());- assert!(res.starts_with("<b>"));- assert!(res.ends_with("</b>"));-- assert_eq!(generate_answer_snippet("", 0..2000), "".to_string());- }-}diff --git a/core/src/searcher/discussions.optic b/core/src/searcher/api/discussions.opticsimilarity index 100%rename from core/src/searcher/discussions.opticrename to core/src/searcher/api/discussions.opticdiff --git a/core/src/searcher/api/mod.rs b/core/src/searcher/api/mod.rsnew file mode 100644index 00000000..c233e63c--- /dev/null+++ b/core/src/searcher/api/mod.rs@@ -0,0 +1,450 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod sidebar;+mod widget;++use std::collections::HashMap;+use std::sync::Arc;+use std::time::Instant;++use itertools::intersperse;+use url::Url;++use crate::bangs::{Bang, BangHit};+use crate::config::{ApiConfig, CollectorConfig};+use crate::image_store::Image;+use crate::inverted_index::RetrievedWebpage;+#[cfg(not(feature = "libtorch"))]+use crate::ranking::models::cross_encoder::DummyCrossEncoder;+use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite, RetrievedWebpageRanking};+use crate::ranking::ALL_SIGNALS;+use crate::search_prettifier::{DisplayedWebpage, HighlightedSpellCorrection};+use crate::web_spell::SpellChecker;+use crate::widgets::Widgets;+use crate::{+ bangs::Bangs,+ collector::BucketCollector,+ distributed::cluster::Cluster,+ ranking::{models::lambdamart::LambdaMART, pipeline::RankingPipeline},+};+#[cfg(feature = "libtorch")]+use crate::{qa_model::QaModel, ranking::models::cross_encoder::CrossEncoderModel};+use crate::{query, Result};++use self::sidebar::SidebarManager;+use self::widget::WidgetManager;++use super::live::LiveSearcher;+use super::{distributed, live, DistributedSearcher, SearchQuery, SearchResult, WebsitesResult};++#[derive(Clone)]+pub enum ScoredWebsitePointer {+ Normal(distributed::ScoredWebsitePointer),+ Live(live::ScoredWebsitePointer),+}++impl AsRankingWebsite for ScoredWebsitePointer {+ fn as_ranking(&self) -> &RankingWebsite {+ match self {+ ScoredWebsitePointer::Normal(p) => &p.website,+ ScoredWebsitePointer::Live(p) => &p.website,+ }+ }++ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {+ match self {+ ScoredWebsitePointer::Normal(p) => &mut p.website,+ ScoredWebsitePointer::Live(p) => &mut p.website,+ }+ }+}++pub fn combine_results(+ collector_config: CollectorConfig,+ initial_results: Vec<distributed::InitialSearchResultShard>,+ live_results: Vec<live::InitialSearchResultSplit>,+ pipeline: RankingPipeline<ScoredWebsitePointer>,+) -> (Vec<ScoredWebsitePointer>, bool) {+ let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);++ let mut num_sites: usize = 0;+ for result in initial_results {+ for website in result.local_result.websites {+ num_sites += 1;+ let pointer = distributed::ScoredWebsitePointer {+ website,+ shard: result.shard,+ };++ let pointer = ScoredWebsitePointer::Normal(pointer);++ collector.insert(pointer);+ }+ }++ for result in live_results {+ for website in result.local_result.websites {+ num_sites += 1;+ let pointer = live::ScoredWebsitePointer {+ website,+ split_id: result.split_id.clone(),+ };++ let pointer = ScoredWebsitePointer::Live(pointer);++ collector.insert(pointer);+ }+ }++ let top_websites = collector+ .into_sorted_vec(true)+ .into_iter()+ .take(pipeline.collector_top_n())+ .collect::<Vec<_>>();++ let offset = pipeline.offset();++ let res = pipeline.apply(top_websites);+ let has_more = num_sites.saturating_sub(offset) > res.len();++ (res, has_more)+}+pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[ScoredWebsitePointer]) {+ for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {+ let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());++ for signal in ALL_SIGNALS {+ if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {+ signals.insert(signal, *signal_value);+ }+ }++ website.ranking_signals = Some(signals);+ }+}++#[cfg(feature = "libtorch")]+pub struct ApiSearcher {+ distributed_searcher: Arc<DistributedSearcher>,+ sidebar_manager: SidebarManager,+ live_searcher: LiveSearcher,+ cross_encoder: Option<Arc<CrossEncoderModel>>,+ lambda_model: Option<Arc<LambdaMART>>,+ bangs: Bangs,+ collector_config: CollectorConfig,+ widget_manager: WidgetManager,+ spell_checker: Option<SpellChecker>,+}++#[cfg(not(feature = "libtorch"))]+pub struct ApiSearcher {+ distributed_searcher: Arc<DistributedSearcher>,+ live_searcher: LiveSearcher,+ sidebar_manager: SidebarManager,+ lambda_model: Option<Arc<LambdaMART>>,+ bangs: Bangs,+ collector_config: CollectorConfig,+ widget_manager: WidgetManager,+ spell_checker: Option<SpellChecker>,+}++impl ApiSearcher {+ #[cfg(feature = "libtorch")]+ pub fn new(+ cluster: Arc<Cluster>,+ cross_encoder: Option<CrossEncoderModel>,+ lambda_model: Option<LambdaMART>,+ qa_model: Option<QaModel>,+ bangs: Bangs,+ config: ApiConfig,+ ) -> Self {+ let dist_searcher = Arc::new(DistributedSearcher::new(Arc::clone(&cluster)));+ let sidebar_manager =+ SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());++ let lambda_model = lambda_model.map(Arc::new);+ let qa_model = qa_model.map(Arc::new);++ let widget_manager = WidgetManager::new(+ Widgets::new(config.widgets).unwrap(),+ Arc::clone(&dist_searcher),+ config.thresholds.clone(),+ config.collector.clone(),+ lambda_model.clone(),+ qa_model,+ );++ Self {+ distributed_searcher: dist_searcher,+ sidebar_manager,+ live_searcher: LiveSearcher::new(Arc::clone(&cluster)),+ cross_encoder: cross_encoder.map(Arc::new),+ lambda_model,+ bangs,+ collector_config: config.collector,+ widget_manager,+ spell_checker: config+ .spell_checker_path+ .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),+ }+ }++ #[cfg(not(feature = "libtorch"))]+ pub fn new(+ cluster: Arc<Cluster>,+ lambda_model: Option<LambdaMART>,+ bangs: Bangs,+ config: ApiConfig,+ ) -> Self {+ let dist_searcher = Arc::new(DistributedSearcher::new(Arc::clone(&cluster)));+ let sidebar_manager =+ SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());+ let lambda_model = lambda_model.map(Arc::new);++ let widget_manager = WidgetManager::new(+ Widgets::new(config.widgets).unwrap(),+ Arc::clone(&dist_searcher),+ config.thresholds.clone(),+ config.collector.clone(),+ lambda_model.clone(),+ );++ Self {+ distributed_searcher: dist_searcher,+ sidebar_manager,+ live_searcher: LiveSearcher::new(Arc::clone(&cluster)),+ lambda_model,+ bangs,+ collector_config: config.collector,+ widget_manager,+ spell_checker: config+ .spell_checker_path+ .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),+ }+ }++ async fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {+ let parsed_terms = query::parser::parse(&query.query);++ if parsed_terms.iter().any(|term| match term.as_ref() {+ query::parser::Term::PossibleBang(t) => t.is_empty(),+ _ => false,+ }) {+ let q: String = intersperse(+ parsed_terms+ .iter()+ .filter(|term| !matches!(term.as_ref(), query::parser::Term::PossibleBang(_)))+ .map(|term| term.to_string()),+ " ".to_string(),+ )+ .collect();++ let mut query = query.clone();+ query.query = q;++ let res = self.search_websites(&query).await?;++ return Ok(res.webpages.first().map(|webpage| BangHit {+ bang: Bang {+ category: None,+ sub_category: None,+ domain: None,+ ranking: None,+ site: None,+ tag: String::new(),+ url: webpage.url.clone(),+ },+ redirect_to: Url::parse(&webpage.url).unwrap().into(),+ }));+ }++ Ok(self.bangs.get(&parsed_terms))+ }++ async fn retrieve_webpages(+ &self,+ query: &str,+ top_websites: &[ScoredWebsitePointer],+ ) -> Vec<RetrievedWebpageRanking> {+ let normal: Vec<_> = top_websites+ .iter()+ .enumerate()+ .filter_map(|(i, pointer)| {+ if let ScoredWebsitePointer::Normal(p) = pointer {+ Some((i, p.clone()))+ } else {+ None+ }+ })+ .collect();++ let live: Vec<_> = top_websites+ .iter()+ .enumerate()+ .filter_map(|(i, pointer)| {+ if let ScoredWebsitePointer::Live(p) = pointer {+ Some((i, p.clone()))+ } else {+ None+ }+ })+ .collect();++ let (retrieved_normal, retrieved_live) = tokio::join!(+ self.distributed_searcher.retrieve_webpages(&normal, query),+ self.live_searcher.retrieve_webpages(&live, query),+ );++ let mut retrieved_webpages: Vec<_> =+ retrieved_normal.into_iter().chain(retrieved_live).collect();+ retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));++ retrieved_webpages+ .into_iter()+ .map(|(_, webpage)| webpage)+ .collect::<Vec<_>>()+ }++ async fn search_websites(&self, query: &SearchQuery) -> Result<WebsitesResult> {+ let start = Instant::now();++ if query.is_empty() {+ return Err(distributed::Error::EmptyQuery.into());+ }++ let mut search_query = query.clone();++ // This pipeline should be created before the first search is performed+ // so the query knows how many results to fetch from the indices+ let first_pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::first_stage(+ &mut search_query,+ self.lambda_model.clone(),+ self.collector_config.clone(),+ 20,+ );++ let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(+ self.distributed_searcher.search_initial(&search_query),+ self.live_searcher.search_initial(&search_query),+ self.widget_manager.widget(query),+ self.widget_manager.discussions(query),+ self.sidebar_manager.stackoverflow(query),+ );++ let discussions = discussions?;+ let stackoverflow = stackoverflow?;+ let sidebar = self+ .sidebar_manager+ .sidebar(&initial_results, stackoverflow);++ let spell_corrected_query = if widget.is_none() {+ self.spell_checker+ .as_ref()+ .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))+ .map(HighlightedSpellCorrection::from)+ } else {+ None+ };++ let num_docs = initial_results+ .iter()+ .map(|result| result.local_result.num_websites)+ .sum();++ let (top_websites, has_more_results) = combine_results(+ self.collector_config.clone(),+ initial_results,+ live_results,+ first_pipeline,+ );++ let retrieved_webpages = self+ .retrieve_webpages(&search_query.query, &top_websites)+ .await;++ let mut search_query = query.clone();++ #[cfg(feature = "libtorch")]+ let pipeline: RankingPipeline<RetrievedWebpageRanking> = RankingPipeline::reranker(+ &mut search_query,+ self.cross_encoder.clone(),+ self.lambda_model.clone(),+ self.collector_config.clone(),+ query.num_results,+ )?;++ #[cfg(not(feature = "libtorch"))]+ let pipeline: RankingPipeline<RetrievedWebpageRanking> =+ RankingPipeline::reranker::<DummyCrossEncoder>(+ &mut search_query,+ None,+ self.lambda_model.clone(),+ self.collector_config.clone(),+ query.num_results,+ )?;++ let retrieved_webpages = pipeline.apply(retrieved_webpages);++ let mut retrieved_webpages: Vec<_> = retrieved_webpages+ .into_iter()+ .map(|webpage| webpage.into_retrieved_webpage())+ .map(DisplayedWebpage::from)+ .collect();++ if retrieved_webpages.len() != top_websites.len() {+ return Err(distributed::Error::SearchFailed.into());+ }++ if query.return_ranking_signals {+ add_ranking_signals(&mut retrieved_webpages, &top_websites);+ }++ let direct_answer = self+ .widget_manager+ .answer(&query.query, &mut retrieved_webpages);++ let search_duration_ms = start.elapsed().as_millis();++ Ok(WebsitesResult {+ spell_corrected_query,+ num_hits: num_docs,+ webpages: retrieved_webpages,+ direct_answer,+ sidebar,+ widget,+ discussions,+ search_duration_ms,+ has_more_results,+ })+ }++ pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {+ if let Some(bang) = self.check_bangs(query).await? {+ return Ok(SearchResult::Bang(Box::new(bang)));+ }++ Ok(SearchResult::Websites(self.search_websites(query).await?))+ }++ pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {+ self.distributed_searcher.get_webpage(url).await+ }++ pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {+ self.distributed_searcher.get_entity_image(image_id).await+ }+}diff --git a/core/src/searcher/api/sidebar.rs b/core/src/searcher/api/sidebar.rsnew file mode 100644index 00000000..205bfac4--- /dev/null+++ b/core/src/searcher/api/sidebar.rs@@ -0,0 +1,113 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{config::ApiThresholds, Result};+use std::{cmp::Ordering, sync::Arc};++use optics::Optic;+use url::Url;++use crate::{+ search_prettifier::{create_stackoverflow_sidebar, DisplayedEntity, DisplayedSidebar},+ searcher::{distributed, DistributedSearcher, SearchQuery},+};++pub struct SidebarManager {+ distributed_searcher: Arc<DistributedSearcher>,+ thresholds: ApiThresholds,+}++impl SidebarManager {+ pub fn new(+ distributed_searcher: Arc<DistributedSearcher>,+ thresholds: ApiThresholds,+ ) -> SidebarManager {+ Self {+ distributed_searcher,+ thresholds,+ }+ }++ pub async fn stackoverflow(&self, query: &SearchQuery) -> Result<Option<DisplayedSidebar>> {+ let query = SearchQuery {+ query: query.query.clone(),+ num_results: 1,+ optic: Some(Optic::parse(include_str!("stackoverflow.optic")).unwrap()),+ ..Default::default()+ };++ let mut results: Vec<_> = self+ .distributed_searcher+ .search_initial(&query)+ .await+ .into_iter()+ .filter_map(|result| {+ result+ .local_result+ .websites+ .first()+ .cloned()+ .map(|website| (result.shard, website))+ })+ .collect();++ results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));++ if let Some((shard, website)) = results.pop() {+ tracing::debug!(?website.score, ?self.thresholds.stackoverflow, "stackoverflow score");+ if website.score > self.thresholds.stackoverflow {+ let scored_websites =+ vec![(0, distributed::ScoredWebsitePointer { website, shard })];+ let mut retrieved = self+ .distributed_searcher+ .retrieve_webpages(&scored_websites, &query.query)+ .await;++ if let Some((_, res)) = retrieved.pop() {+ let res = res.into_retrieved_webpage();+ return Ok(Some(create_stackoverflow_sidebar(+ res.schema_org,+ Url::parse(&res.url).unwrap(),+ )?));+ }+ }+ }++ Ok(None)+ }++ pub fn sidebar(+ &self,+ initial_results: &[distributed::InitialSearchResultShard],+ stackoverflow: Option<DisplayedSidebar>,+ ) -> Option<DisplayedSidebar> {+ let entity = initial_results+ .iter()+ .filter_map(|res| res.local_result.entity_sidebar.clone())+ .map(DisplayedEntity::from)+ .filter(|entity| entity.match_score as f64 > self.thresholds.entity_sidebar)+ .max_by(|a, b| {+ a.match_score+ .partial_cmp(&b.match_score)+ .unwrap_or(Ordering::Equal)+ });++ match entity {+ Some(entity) => Some(DisplayedSidebar::Entity(entity)),+ None => stackoverflow,+ }+ }+}diff --git a/core/src/searcher/stackoverflow.optic b/core/src/searcher/api/stackoverflow.opticsimilarity index 100%rename from core/src/searcher/stackoverflow.opticrename to core/src/searcher/api/stackoverflow.opticdiff --git a/core/src/searcher/api/widget.rs b/core/src/searcher/api/widget.rsnew file mode 100644index 00000000..f27e4bcf--- /dev/null+++ b/core/src/searcher/api/widget.rs@@ -0,0 +1,367 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::ops::Range;+use std::sync::Arc;++use itertools::Itertools;+use optics::Optic;++#[cfg(feature = "libtorch")]+use crate::qa_model::QaModel;+use crate::ranking::models::lambdamart::LambdaMART;+use crate::ranking::pipeline::{AsRankingWebsite, RankingPipeline};+use crate::search_prettifier::DisplayedAnswer;+use crate::search_prettifier::DisplayedWebpage;+use crate::searcher::api::{add_ranking_signals, combine_results, ScoredWebsitePointer};+use crate::searcher::{DistributedSearcher, SearchQuery};+use crate::widgets::Widget;+use crate::{ceil_char_boundary, floor_char_boundary, query, Result};+use crate::{+ config::{ApiThresholds, CollectorConfig},+ widgets::Widgets,+};++#[cfg(feature = "libtorch")]+pub struct WidgetManager {+ widgets: Widgets,+ distributed_searcher: Arc<DistributedSearcher>,+ thresholds: ApiThresholds,+ collector_config: CollectorConfig,+ lambda_model: Option<Arc<LambdaMART>>,+ qa_model: Option<Arc<QaModel>>,+}++#[cfg(not(feature = "libtorch"))]+pub struct WidgetManager {+ widgets: Widgets,+ distributed_searcher: Arc<DistributedSearcher>,+ thresholds: ApiThresholds,+ collector_config: CollectorConfig,+ lambda_model: Option<Arc<LambdaMART>>,+}++impl WidgetManager {+ #[cfg(feature = "libtorch")]+ pub fn new(+ widgets: Widgets,+ distributed_searcher: Arc<DistributedSearcher>,+ thresholds: ApiThresholds,+ collector_config: CollectorConfig,+ lambda_model: Option<Arc<LambdaMART>>,+ qa_model: Option<Arc<QaModel>>,+ ) -> WidgetManager {+ Self {+ widgets,+ distributed_searcher,+ thresholds,+ collector_config,+ lambda_model,+ qa_model,+ }+ }++ #[cfg(not(feature = "libtorch"))]+ pub fn new(+ widgets: Widgets,+ distributed_searcher: Arc<DistributedSearcher>,+ thresholds: ApiThresholds,+ collector_config: CollectorConfig,+ lambda_model: Option<Arc<LambdaMART>>,+ ) -> WidgetManager {+ Self {+ widgets,+ distributed_searcher,+ thresholds,+ collector_config,+ lambda_model,+ }+ }++ pub async fn discussions(&self, query: &SearchQuery) -> Result<Option<Vec<DisplayedWebpage>>> {+ if !query.fetch_discussions || query.optic.is_some() || query.page > 0 {+ return Ok(None);+ }++ const NUM_RESULTS: usize = 10;++ let mut query = SearchQuery {+ query: query.query.clone(),+ num_results: NUM_RESULTS,+ optic: Some(Optic::parse(include_str!("discussions.optic")).unwrap()),+ site_rankings: query.site_rankings.clone(),+ return_ranking_signals: query.return_ranking_signals,+ ..Default::default()+ };++ // This pipeline should be created before the first search is performed+ // so the query knows how many results to fetch from the indices+ let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::first_stage(+ &mut query,+ self.lambda_model.clone(),+ self.collector_config.clone(),+ NUM_RESULTS,+ );++ let initial_results = self.distributed_searcher.search_initial(&query).await;++ if initial_results.is_empty() {+ return Ok(None);+ }++ let num_results: usize = initial_results+ .iter()+ .map(|res| res.local_result.websites.len())+ .sum();++ if num_results < NUM_RESULTS / 2 {+ return Ok(None);+ }++ let (top_websites, _) = combine_results(+ self.collector_config.clone(),+ initial_results,+ vec![],+ pipeline,+ );++ let scores: Vec<_> = top_websites+ .iter()+ .map(|pointer| pointer.as_ranking().score)+ .collect();++ let median = if scores.len() % 2 == 0 {+ (scores[(scores.len() / 2) - 1] + scores[scores.len() / 2]) / 2.0+ } else {+ scores[scores.len() / 2]+ };++ if median < self.thresholds.discussions_widget {+ return Ok(None);+ }++ let mut result: Vec<DisplayedWebpage> = self+ .distributed_searcher+ .retrieve_webpages(+ &top_websites+ .clone()+ .into_iter()+ .filter_map(|pointer| {+ if let ScoredWebsitePointer::Normal(p) = pointer {+ Some(p)+ } else {+ None+ }+ })+ .enumerate()+ .collect_vec(),+ &query.query,+ )+ .await+ .into_iter()+ .map(|(_, webpage)| webpage.into_retrieved_webpage())+ .map(DisplayedWebpage::from)+ .collect();++ if query.return_ranking_signals {+ add_ranking_signals(&mut result, &top_websites);+ }++ Ok(Some(result))+ }++ pub async fn widget(&self, query: &SearchQuery) -> Option<Widget> {+ if query.page > 0 {+ return None;+ }++ let parsed_terms = query::parser::parse(&query.query);++ self.widgets.widget(+ parsed_terms+ .into_iter()+ .filter_map(|term| {+ if let query::parser::Term::Simple(simple) = *term {+ Some(String::from(simple))+ } else {+ None+ }+ })+ .join(" ")+ .as_str(),+ )+ }++ #[cfg(feature = "libtorch")]+ pub fn answer(+ &self,+ query: &str,+ webpages: &mut Vec<DisplayedWebpage>,+ ) -> Option<DisplayedAnswer> {+ self.qa_model.as_ref().and_then(|qa_model| {+ let contexts: Vec<_> = webpages+ .iter()+ .take(1)+ .filter_map(|webpage| webpage.snippet.text())+ .map(|t| t.fragments.iter().map(|f| f.text()).join(""))+ .collect();++ match qa_model.run(query, &contexts) {+ Some(answer) => {+ let answer_webpage = webpages.remove(answer.context_idx);+ let snip = answer_webpage+ .snippet+ .text()+ .unwrap()+ .fragments+ .iter()+ .map(|f| f.text())+ .join("");+ Some(DisplayedAnswer {+ title: answer_webpage.title,+ url: answer_webpage.url,+ pretty_url: answer_webpage.pretty_url,+ snippet: generate_answer_snippet(&snip, answer.offset.clone()),+ answer: snip[answer.offset].to_string(),+ })+ }+ None => None,+ }+ })+ }++ #[cfg(not(feature = "libtorch"))]+ pub fn answer(+ &self,+ query: &str,+ webpages: &mut Vec<DisplayedWebpage>,+ ) -> Option<DisplayedAnswer> {+ None+ }+}++fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {+ let mut best_start = 0;+ let mut best_end = 0;+ const SNIPPET_LENGTH: usize = 200;++ if body.is_empty() || answer_offset.start > body.len() - 1 {+ return body.to_string();+ }++ for (idx, _) in body.char_indices().filter(|(_, c)| *c == '.') {+ if idx > answer_offset.end + SNIPPET_LENGTH {+ break;+ }++ if idx < answer_offset.start {+ best_start = idx;+ }++ if idx > answer_offset.end {+ best_end = idx;+ }+ }++ if (answer_offset.end - best_start > SNIPPET_LENGTH) || (best_start >= best_end) {+ if answer_offset.end - answer_offset.start >= SNIPPET_LENGTH {+ let end = floor_char_boundary(body, answer_offset.start + SNIPPET_LENGTH);++ return "<b>".to_string() + &body[answer_offset.start..end] + "</b>";+ }++ let chars_either_side = (SNIPPET_LENGTH - (answer_offset.end - answer_offset.start)) / 2;++ let start = ceil_char_boundary(+ body,+ answer_offset+ .start+ .checked_sub(chars_either_side)+ .unwrap_or_default(),+ );+ let mut end = ceil_char_boundary(body, answer_offset.end + chars_either_side);++ if end >= body.len() {+ end = floor_char_boundary(body, body.len());+ }++ body[start..answer_offset.start].to_string()+ + "<b>"+ + &body[answer_offset.clone()]+ + "</b>"+ + &body[answer_offset.end..end]+ } else {+ let mut res = body[best_start..answer_offset.start].to_string()+ + "<b>"+ + &body[answer_offset.clone()]+ + "</b>";++ let remaining_chars = SNIPPET_LENGTH - (res.len() - 7);+ let end = ceil_char_boundary(body, (remaining_chars + answer_offset.end).min(best_end));++ res += &body[answer_offset.end..end];++ res+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn test_generate_answer_snippet() {+ assert_eq!(+ generate_answer_snippet("this is a test", 0..4),+ "<b>this</b> is a test".to_string()+ );++ assert_eq!(+ generate_answer_snippet("this is a test", 0..1000),+ "<b>this is a test</b>".to_string()+ );+ assert_eq!(+ generate_answer_snippet("this is a test", 1000..2000),+ "this is a test".to_string()+ );+ let input = r#"+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test+ "#;++ let res = generate_answer_snippet(input, 0..500);+ assert!(!res.is_empty());+ assert!(res.len() > 100);+ assert!(res.len() < input.len());+ assert!(res.starts_with("<b>"));+ assert!(res.ends_with("</b>"));++ assert_eq!(generate_answer_snippet("", 0..2000), "".to_string());+ }+}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex eb107a16..39135a3f 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -26,12 +26,13 @@ use crate::{entrypoint::search_server::{self, SearchService},image_store::Image,inverted_index::{RetrievedWebpage, WebsitePointer},- ranking::pipeline::RankingWebsite,+ ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},Result,};use std::{collections::HashMap, sync::Arc};+use fnv::FnvHashMap;use futures::future::join_all;use itertools::Itertools;use serde::{Deserialize, Serialize};@@ -161,7 +162,8 @@ impl DistributedSearcher {&self,top_websites: &[(usize, ScoredWebsitePointer)],query: &str,- ) -> Vec<(usize, RetrievedWebpage)> {+ ) -> Vec<(usize, RetrievedWebpageRanking)> {+ let mut rankings = FnvHashMap::default();let mut pointers: HashMap<_, Vec<_>> = HashMap::new();for (i, pointer) in top_websites {@@ -169,6 +171,8 @@ impl DistributedSearcher {.entry(pointer.shard).or_default().push((*i, pointer.website.pointer.clone()));++ rankings.insert(*i, pointer.website.clone());}let client = self.client().await;@@ -179,7 +183,10 @@ impl DistributedSearcher {let mut retrieved_webpages = Vec::new();for pages in join_all(futures).await {- retrieved_webpages.extend(pages);+ for (i, page) in pages {+ retrieved_webpages+ .push((i, RetrievedWebpageRanking::new(page, rankings[&i].clone())));+ }}debug_assert_eq!(retrieved_webpages.len(), top_websites.len());diff --git a/core/src/searcher/live.rs b/core/src/searcher/live.rsindex 6593d10b..1e90ebd2 100644--- a/core/src/searcher/live.rs+++ b/core/src/searcher/live.rs@@ -26,11 +26,12 @@ use crate::{entrypoint::search_server::{self, SearchService},feed::scheduler::SplitId,inverted_index::{RetrievedWebpage, WebsitePointer},- ranking::pipeline::RankingWebsite,+ ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},};use std::{collections::HashMap, sync::Arc};+use fnv::FnvHashMap;use futures::future::join_all;use itertools::Itertools;@@ -142,7 +143,8 @@ impl LiveSearcher {&self,top_websites: &[(usize, ScoredWebsitePointer)],query: &str,- ) -> Vec<(usize, RetrievedWebpage)> {+ ) -> Vec<(usize, RetrievedWebpageRanking)> {+ let mut rankings = FnvHashMap::default();let mut pointers: HashMap<_, Vec<_>> = HashMap::new();for (i, pointer) in top_websites {@@ -150,17 +152,22 @@ impl LiveSearcher {.entry(pointer.split_id.clone()).or_default().push((*i, pointer.website.pointer.clone()));++ rankings.insert(*i, pointer.website.clone());}let client = self.client().await;let mut futures = Vec::new();- for (split_id, pointers) in pointers {- futures.push(self.retrieve_webpages_from_shard(split_id, &client, query, pointers));+ for (shard, pointers) in pointers {+ futures.push(self.retrieve_webpages_from_shard(shard, &client, query, pointers));}let mut retrieved_webpages = Vec::new();for pages in join_all(futures).await {- retrieved_webpages.extend(pages);+ for (i, page) in pages {+ retrieved_webpages+ .push((i, RetrievedWebpageRanking::new(page, rankings[&i].clone())));+ }}debug_assert_eq!(retrieved_webpages.len(), top_websites.len());diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex d47185a5..b85c0d32 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -17,8 +17,6 @@use std::collections::HashMap;use std::sync::{Arc, RwLockReadGuard};-use tantivy::schema::Value;-use tantivy::TantivyDocument;use url::Url;use crate::config::{CollectorConfig, SnippetConfig};@@ -34,7 +32,6 @@ use crate::ranking::models::lambdamart::LambdaMART;use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS};-use crate::schema::TextField;use crate::search_ctx::Ctx;use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};use crate::webgraph::Node;@@ -116,7 +113,6 @@ impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {pub struct LocalSearcher<I: SearchableIndex> {index: I,- // spell: Option<Spell>,entity_index: Option<EntityIndex>,inbound_similarity: Option<InboundSimilarity>,linear_regression: Option<Arc<LinearRegression>>,@@ -257,10 +253,11 @@ wherede_rank_similar: bool,) -> Result<InvertedIndexResult> {let mut query = query.clone();- let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::initial_for_query(+ let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::first_stage(&mut query,self.lambda_model.clone(),self.collector_config.clone(),+ 100,);let parsed_query = self.parse_query(ctx, guard, &query)?;@@ -319,26 +316,7 @@ where)?;let pipe_top_n = pipeline.top_n;- let mut ranking_websites = pipeline.apply(ranking_websites);-- let schema = guard.inverted_index().schema();- for website in &mut ranking_websites {- let doc: TantivyDocument = ctx.tv_searcher.doc(website.pointer.address.into())?;- website.title = Some(- doc.get_first(schema.get_field(TextField::Title.name()).unwrap())- .map(|text| text.as_value().as_str().unwrap().to_string())- .unwrap_or_default(),- );- website.clean_body = Some(- doc.get_first(- schema- .get_field(TextField::StemmedCleanBody.name())- .unwrap(),- )- .map(|text| text.as_value().as_str().unwrap().to_string())- .unwrap_or_default(),- );- }+ let ranking_websites = pipeline.apply(ranking_websites);let has_more = pipe_top_n == ranking_websites.len();@@ -409,17 +387,19 @@ wherelet pipeline = {use crate::ranking::models::cross_encoder::CrossEncoderModel;match CrossEncoderModel::open("data/cross_encoder") {- Ok(model) => RankingPipeline::reranking_for_query::<CrossEncoderModel>(+ Ok(model) => RankingPipeline::reranker::<CrossEncoderModel>(&mut search_query,Some(Arc::new(model)),None,self.collector_config.clone(),+ query.num_results,)?,- Err(_) => RankingPipeline::reranking_for_query::<CrossEncoderModel>(+ Err(_) => RankingPipeline::reranker::<CrossEncoderModel>(&mut search_query,None,None,self.collector_config.clone(),+ query.num_results,)?,}};@@ -427,11 +407,12 @@ where#[cfg(not(feature = "libtorch"))]let pipeline = {use crate::ranking::models::cross_encoder::DummyCrossEncoder;- RankingPipeline::reranking_for_query::<DummyCrossEncoder>(+ RankingPipeline::reranker::<DummyCrossEncoder>(&mut search_query,None,None,self.collector_config.clone(),+ query.num_results,)?};diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex 7b4568a4..b0498f90 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -77,9 +77,9 @@ impl SimilarSitesFinder {}pub fn find_similar_sites(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {- const SIMILAR_DOMAINS_BUFFER: usize = 30;+ const DEAD_LINKS_BUFFER: usize = 30;let orig_limit = limit.min(self.max_similar_sites);- let limit = orig_limit + nodes.len() + SIMILAR_DOMAINS_BUFFER;+ let limit = orig_limit + nodes.len() + DEAD_LINKS_BUFFER;let nodes: Vec<_> = nodes.iter()@@ -155,6 +155,9 @@ impl SimilarSitesFinder {scored_nodes.into_iter()+ .filter(|ScoredNodeID { node_id, score: _ }| {+ !self.webgraph.raw_outgoing_edges(node_id).is_empty()+ }).filter_map(|ScoredNodeID { node_id, score }| {let node = self.webgraph.id2node(&node_id).unwrap();match Url::parse(&format!("http://{}", &node.name))diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex 263db069..3f0c9f57 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -72,6 +72,16 @@ pub struct TextSnippet {pub fragments: Vec<TextSnippetFragment>,}+impl TextSnippet {+ pub fn unhighlighted_string(&self) -> String {+ self.fragments+ .iter()+ .map(|f| f.text.clone())+ .collect::<Vec<_>>()+ .join("")+ }+}+struct SnippetBuilder {fragment: String,highlights: Vec<Range<usize>>,diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex eb11fc7d..855a26e1 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -582,6 +582,10 @@ impl WebgraphWriter {}pub fn insert(&mut self, from: Node, to: Node, label: String) {+ if from == to {+ return;+ }+let (from_id, to_id) = (self.id_or_assign(from.clone()),self.id_or_assign(to.clone()),diff --git a/core/src/webpage/just_text.rs b/core/src/webpage/just_text.rsindex 0aebd97c..607d3da1 100644--- a/core/src/webpage/just_text.rs+++ b/core/src/webpage/just_text.rs@@ -412,7 +412,7 @@ impl JustText {res}- fn contextual_classification(&self, paragraphs: &mut Vec<ClassifiedParagraph>) {+ fn contextual_classification(&self, paragraphs: &mut [ClassifiedParagraph]) {// good headingslet num_paragraphs = paragraphs.len();for i in 0..num_paragraphs {@@ -602,7 +602,7 @@ impl JustText {.initial_classification(paragraphs, lang).into_iter().filter(|par| par.paragraph.text.chars().any(|c| !c.is_whitespace()))- .collect();+ .collect::<Vec<_>>();self.contextual_classification(&mut classified);diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex c895adb9..99920dd1 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -1015,6 +1015,8 @@ impl Html {.to_string();let schemas: Vec<_> = self.schema_org();+ let first_ingredient_tag_id =+ find_recipe_first_ingredient_tag_id(&schemas, &self.root).unwrap_or_default();let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();@@ -1255,6 +1257,9 @@ impl Html {Field::Text(TextField::AllBody) => {doc.add_pre_tokenized_text(tantivy_field, all_text.clone())}+ Field::Text(TextField::RecipeFirstIngredientTagId) => {+ doc.add_text(tantivy_field, first_ingredient_tag_id.clone());+ }Field::Text(TextField::SchemaOrgJson) => {doc.add_text(tantivy_field, schema_json.clone());}@@ -1713,6 +1718,52 @@ fn stemmer_from_lang(lang: &Lang) -> rust_stemmers::Stemmer {}}+fn find_recipe_first_ingredient_tag_id(+ schemas: &[schema_org::Item],+ root: &NodeRef,+) -> Option<String> {+ schemas+ .iter()+ .filter_map(|schema| {+ if let Some(ingredients) = schema.properties.get("recipeIngredient") {+ if let Some(ingredient) = ingredients.clone().many().first() {+ if let Some(ingredient) = ingredient.try_into_string() {+ let ingredient = ingredient.trim();+ // find first occurrence in html+ if let Some(ingredient_node) = root+ .select("body")+ .unwrap()+ .flat_map(|node| node.as_node().descendants())+ .find(|node| {+ if let Some(text) = node.as_text() {+ text.borrow().trim() == ingredient+ } else {+ false+ }+ })+ {+ // find first parent that has an id+ if let Some(id) = ingredient_node+ .ancestors()+ .filter_map(|node| {+ node.as_element().and_then(|e| {+ e.attributes.borrow().get("id").map(|s| s.to_string())+ })+ })+ .next()+ {+ return Some(id.to_string());+ }+ }+ }+ }+ }++ None+ })+ .next()+}+fn stem_tokens(tokens: &mut [tantivy::tokenizer::Token], lang: Lang) {let stemmer = stemmer_from_lang(&lang);for token in tokens {@@ -2592,4 +2643,65 @@ mod tests {assert!(!microformats.contains(Microformat::HCard));assert!(!microformats.contains(Microformat::HProduct));}++ #[test]+ fn recipe_first_ingredient_tag() {+ let html = Html::parse(+ r#"+ <html>+ <head>+ </head>+ <body>+ <script type="application/ld+json">+ {+ "@context": "https://schema.org",+ "@type": "Recipe",+ "author": "John Smith",+ "cookTime": "PT1H",+ "datePublished": "2009-05-08",+ "description": "This classic banana bread recipe comes from my mom -- the walnuts add a nice texture and flavor to the banana bread.",+ "image": "bananabread.jpg",+ "recipeIngredient": [+ "3 or 4 ripe bananas, smashed",+ "1 egg",+ "3/4 cup of sugar"+ ],+ "interactionStatistic": {+ "@type": "InteractionCounter",+ "interactionType": "https://schema.org/Comment",+ "userInteractionCount": "140"+ },+ "name": "Mom's World Famous Banana Bread",+ "nutrition": {+ "@type": "NutritionInformation",+ "calories": "240 calories",+ "fatContent": "9 grams fat"+ },+ "prepTime": "PT15M",+ "recipeInstructions": "Preheat the oven to 350 degrees. Mix in the ingredients in a bowl. Add the flour last. Pour the mixture into a loaf pan and bake for one hour.",+ "recipeYield": "1 loaf",+ "suitableForDiet": "https://schema.org/LowFatDiet"+ }+ </script>++ <div id="ingredients">+ <h2>Ingredients</h2>+ <ul>+ <li>3 or 4 ripe bananas, smashed</li>+ <li>1 egg</li>+ <li>3/4 cup of sugar</li>+ </ul>+ </body>+ </html>+ "#,+ "https://www.example.com/",+ ).unwrap();++ let schemas = html.schema_org();++ assert_eq!(+ find_recipe_first_ingredient_tag_id(&schemas, &html.root),+ Some("ingredients".to_string())+ );+ }}diff --git a/core/src/widgets/calculator.rs b/core/src/widgets/calculator.rsindex f7012127..3e99b6e3 100644--- a/core/src/widgets/calculator.rs+++ b/core/src/widgets/calculator.rs@@ -159,6 +159,11 @@ impl Calculator {pub fn try_calculate(&self, expr: &str) -> Result<Calculation, Error> {let expr = expr.replace(['"', '\''], "");+ // check if expr has at least one digit+ if !expr.chars().any(|c| c.is_ascii_digit()) {+ return Err(Error::CalculatorParse);+ }+// if expr starts with "d[0-9]+", wrap it in "roll(...)"let expr = if DICE_REGEX.is_match(&expr) {format!("roll({})", expr)diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelteindex f690696c..a7b802d8 100644--- a/frontend/src/routes/search/+page.svelte+++ b/frontend/src/routes/search/+page.svelte@@ -143,10 +143,10 @@{#if results.type == 'websites'}<div- class="m-0 grid w-full gap-y-6 px-5 pt-4 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] md:gap-x-12 md:pl-20 lg:px-36"+ class="m-0 grid w-full gap-y-6 px-5 pt-4 md:grid-cols-[minmax(50ch,48rem)_1fr] md:grid-rows-[auto_1fr] md:gap-x-12 md:pl-20 lg:pl-28"style="text-rendering:optimizeLegibility;font-smooth:always;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;">- <div class="flex max-w-2xl flex-col space-y-3">+ <div class="flex max-w-2xl flex-col space-y-2"><div class="w-full"><Searchbar {query} /></div>@@ -233,7 +233,7 @@{#if results.sidebar}<div- class="row-start-2 mx-auto max-w-[90vw] justify-center md:col-start-2 md:row-span-2 md:row-start-1 md:pt-10"+ class="row-start-2 mx-auto max-w-[90vw] md:max-w-[30vw] justify-center md:col-start-2 md:row-span-2 md:row-start-1 md:pt-5"><Sidebar sidebar={results.sidebar} /></div>diff --git a/frontend/src/routes/search/Discussions.svelte b/frontend/src/routes/search/Discussions.svelteindex 142efeca..8123d2fe 100644--- a/frontend/src/routes/search/Discussions.svelte+++ b/frontend/src/routes/search/Discussions.svelte@@ -51,7 +51,7 @@</div>{/each}<div class="noscript:hidden mt-2">- <Button kind="neutral" pale on:click={() => (showMore = !showMore)}+ <Button _class="py-1" kind="neutral" pale on:click={() => (showMore = !showMore)}>{showMore ? 'Show less' : 'Show more'}</Button></div>diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelteindex 5cf3e43c..e8050ac7 100644--- a/frontend/src/routes/search/Result.svelte+++ b/frontend/src/routes/search/Result.svelte@@ -20,9 +20,9 @@const dispatch = createEventDispatcher<{ modal: HTMLButtonElement }>();</script>-<div class="flex min-w-0 grow flex-col space-y-1">+<div class="flex min-w-0 grow flex-col space-y-0.5"><div class="flex min-w-0">- <div class="flex min-w-0 grow flex-col space-y-1">+ <div class="flex min-w-0 grow flex-col space-y-0.5"><div class="flex items-center text-sm"><aclass="max-w-[calc(100%-100px)] truncate text-neutral-focus"@@ -46,7 +46,7 @@bind:this={button}on:click|stopPropagation={() => dispatch('modal', button)}>- <AdjustVertical class="text-xl" />+ <AdjustVertical class="text-md" /></button></div><div class="text-sm font-normal text-neutral-focus [&>b]:font-bold">diff --git a/frontend/src/routes/search/StackOverflow.svelte b/frontend/src/routes/search/StackOverflow.svelteindex e3683b76..d903908c 100644--- a/frontend/src/routes/search/StackOverflow.svelte+++ b/frontend/src/routes/search/StackOverflow.svelte@@ -20,7 +20,7 @@</div></div></div>- <div class="flex grow justify-between space-x-2">+ <div class="flex grow justify-between space-x-2 text-sm"><div><a href={answer.url}>{answer.url}</a></div>@@ -28,7 +28,7 @@</div></div><hr class="border-stone-700" />- <div class="flex flex-col space-y-3">+ <div class="flex flex-col space-y-3 text-sm">{#each answer.body as part}{#if part.type == 'code'}<Code code={part.value} />