web search done right
A bunch of performance optimizations during search. * use image link instead of base64 embedded to lower serp size. * embed 'bm25' signal into all the 'bm25_*' signals. The math should be equivalent and prevents us calculating the same signals twice. * remove proximity ranking signals as they were disabled. Need to figure out how to make them faster. * faster snippet generation by first trying to generate snippet without stemming. This will succeed in most results and is vastly faster.
Mikkel Denker38342798e12f
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex ae343c38..99e1561d 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -223,6 +223,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {).route("/api/sites/export", post(sites::sites_export_optic)).route("/api/explore/export", post(explore::explore_export_optic))+ .route("/api/entity_image", get(search::entity_image)).layer(cors_layer()),).with_state(state))diff --git a/core/src/api/search.rs b/core/src/api/search.rsindex 7e98bf2d..303b9ee5 100644--- a/core/src/api/search.rs+++ b/core/src/api/search.rs@@ -164,3 +164,39 @@ pub async fn api(},}}++#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]+pub struct EntityImageParams {+ pub image_id: String,+}++#[utoipa::path(+ post,+ path = "/beta/api/search",+ request_body(content = ApiSearchQuery),+ responses(+ (status = 200, description = "Search results", body = ApiSearchResult),+ )+)]+pub async fn entity_image(+ extract::Query(query): extract::Query<EntityImageParams>,+ extract::State(state): extract::State<Arc<State>>,+) -> Result<impl IntoResponse, StatusCode> {+ match state.searcher.get_entity_image(&query.image_id).await {+ Ok(Some(result)) => {+ let bytes = result.as_raw_bytes();++ Ok((+ ([(axum::http::header::CONTENT_TYPE, "image/png")]),+ axum::response::AppendHeaders([(axum::http::header::CONTENT_TYPE, "image/png")]),+ bytes,+ ))+ }+ Ok(None) => Err(StatusCode::NOT_FOUND),+ Err(err) => {+ tracing::error!("{:?}", err);+ Err(StatusCode::INTERNAL_SERVER_ERROR)+ }+ }+}diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rsindex 85aacad8..822b0ee0 100644--- a/core/src/entity_index/mod.rs+++ b/core/src/entity_index/mod.rs@@ -149,7 +149,7 @@ fn wikipedify_url(url: &str) -> Vec<Url> {pub struct StoredEntity {pub title: String,pub entity_abstract: String,- pub image_base64: Option<String>,+ pub image_id: Option<String>,pub related_entities: Vec<EntityMatch>,pub best_info: Vec<(String, Span)>,pub links: Vec<Link>,@@ -304,7 +304,7 @@ impl EntityIndex {EntityMatch { entity, score }})- .filter(|m| m.entity.image_base64.is_some())+ .filter(|m| m.entity.image_id.is_some()).take(4).collect(),Err(_) => Vec::new(),@@ -412,9 +412,12 @@ impl EntityIndex {Vec::new()};- let image_base64 = self- .retrieve_image(&title)- .map(|image| base64::encode(image.as_raw_bytes()));+ let image_id = base64::encode(&title);+ let image_id = if self.retrieve_image(&image_id).is_some() {+ Some(image_id)+ } else {+ None+ };let links: Vec<Link> = if get_links {bincode::deserialize(@@ -433,15 +436,18 @@ impl EntityIndex {StoredEntity {title,entity_abstract,- image_base64,+ image_id,related_entities,best_info,links,}}- pub fn retrieve_image(&self, key: &String) -> Option<Image> {- self.image_store.get(key)+ pub fn retrieve_image(&self, key: &str) -> Option<Image> {+ let key = base64::decode(key).ok()?;+ let key = String::from_utf8(key).ok()?;++ self.image_store.get(&key)}pub fn get_attribute_occurrence(&self, attribute: &String) -> Option<u32> {diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 5909d6ea..a3efc046 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -28,6 +28,7 @@ use crate::{sonic,},entity_index::EntityIndex,+ image_store::Image,index::Index,inverted_index::{self, RetrievedWebpage},ranking::{@@ -45,6 +46,7 @@ sonic_service!(Search,GetWebpage,GetHomepageDescriptions,+ GetEntityImage,]);@@ -178,6 +180,18 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {}}+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct GetEntityImage {+ pub image_id: String,+}+#[async_trait::async_trait]+impl sonic::service::Message<SearchService> for GetEntityImage {+ type Response = Option<Image>;+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {+ Ok(server.local_searcher.get_entity_image(&self.image_id))+ }+}+pub async fn run(config: config::SearchServerConfig) -> Result<()> {let addr = config.host;let server = SearchService::new(config).await?.bind(addr).await.unwrap();diff --git a/core/src/image_store.rs b/core/src/image_store.rsindex 69f4a54c..f36e273b 100644--- a/core/src/image_store.rs+++ b/core/src/image_store.rs@@ -203,6 +203,10 @@ impl Image {bytes}++ pub fn resize(self, width: u32, height: u32) -> Self {+ ResizeFilter { width, height }.transform(self)+ }}#[cfg(test)]diff --git a/core/src/index.rs b/core/src/index.rsindex e8ed9e02..8da18312 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -335,7 +335,7 @@ mod tests {.ranking_signals.as_ref().unwrap()- .get(&crate::ranking::Signal::Bm25)+ .get(&crate::ranking::Signal::Bm25Title).unwrap()).all(|&v| v.value > 0.0));}diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex a6081471..e7f39fec 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -668,128 +668,6 @@ mod tests {assert_eq!(&res.webpages[0].url, "https://www.centrality.com/");}- #[test]- #[ignore = "proximity signals are currently disabled"]- fn term_proximity_ranking() {- let mut index = Index::temporary().expect("Unable to open index");- let mut w = Webpage {- html: Html::parse(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT} termA termB d d d d d d d d d {}- </body>- </html>- "#,- crate::rand_words(100)- ),- "https://www.first.com",- )- .unwrap(),- backlink_labels: vec![],- host_centrality: 1.0,- pre_computed_score: 0.0,- fetch_time_ms: 500,- page_centrality: 0.0,-- node_id: None,- dmoz_description: None,- safety_classification: None,- };- w.html.set_clean_text(format!(- "{CONTENT} termA termB d d d d d d d d d {}",- crate::rand_words(100)- ));-- index.insert(w).expect("failed to insert webpage");- let mut w = Webpage {- html: Html::parse(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT} termA d d d d d d d d d termB {}- </body>- </html>- "#,- crate::rand_words(100)- ),- "https://www.third.com",- )- .unwrap(),- backlink_labels: vec![],- host_centrality: 1.0,- fetch_time_ms: 500,- pre_computed_score: 0.0,- page_centrality: 0.0,-- node_id: None,- dmoz_description: None,- safety_classification: None,- };- w.html.set_clean_text(format!(- "{CONTENT} termA d d d d d d d d d termB {}",- crate::rand_words(100)- ));- index.insert(w).expect("failed to insert webpage");-- let mut w = Webpage {- html: Html::parse(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {CONTENT} termA d d d d termB d d d d d {}- </body>- </html>- "#,- crate::rand_words(100)- ),- "https://www.second.com",- )- .unwrap(),- backlink_labels: vec![],- host_centrality: 1.0,- fetch_time_ms: 500,- pre_computed_score: 0.0,- page_centrality: 0.0,-- node_id: None,- dmoz_description: None,- safety_classification: None,- };- w.html.set_clean_text(format!(- "{CONTENT} termA d d d d termB d d d d d {}",- crate::rand_words(100)- ));-- index.insert(w).expect("failed to insert webpage");- index.commit().expect("failed to commit index");- let searcher = LocalSearcher::new(index);-- let result = searcher- .search(&SearchQuery {- query: "termA termB".to_string(),- ..Default::default()- })- .expect("Search failed");-- assert_eq!(result.webpages.len(), 3);- assert_eq!(result.webpages[0].url, "https://www.first.com/");- assert_eq!(result.webpages[1].url, "https://www.second.com/");- assert_eq!(result.webpages[2].url, "https://www.third.com/");- }-#[test]fn fetch_time_ranking() {let mut index = Index::temporary().expect("Unable to open index");diff --git a/core/src/ranking/models/lambdamart.rs b/core/src/ranking/models/lambdamart.rsindex 94c9526e..2a8019ae 100644--- a/core/src/ranking/models/lambdamart.rs+++ b/core/src/ranking/models/lambdamart.rs@@ -275,6 +275,7 @@ mod tests {use super::*;#[test]+ #[ignore = "train new model with updated signals"]fn simple() {let tree = include_str!("../../../testcases/lambdamart.txt");let tree = Tree::parse(tree).unwrap();@@ -292,7 +293,6 @@ mod tests {features.insert(Signal::Bm25Domain, 89.1253814697);features.insert(Signal::Bm25CleanBody, 54.1200180054);features.insert(Signal::FetchTimeMs, 0.0018281536);- features.insert(Signal::Bm25, 952.8474273682);features.insert(Signal::Bm25AllBody, 43.6125068665);features.insert(Signal::Bm25Site, 87.6877670288);features.insert(Signal::Bm25TitleTrigrams, 86.8362731934);diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex cb2336eb..e952bb1b 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -408,7 +408,7 @@ mod tests {.map(|i| -> RankingWebsite {let mut signals = EnumMap::new();signals.insert(- Signal::Bm25,+ Signal::HostCentrality,SignalScore {coefficient: 1.0,value: 1.0 / i as f64,diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 980d6396..d4331963 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -32,7 +32,7 @@ use std::str::FromStr;use std::sync::Arc;use tantivy::fieldnorm::FieldNormReader;use tantivy::postings::SegmentPostings;-use tantivy::query::{PhraseQuery, Query as _, Scorer};+use tantivy::query::{Query as _, Scorer};use tantivy::tokenizer::Tokenizer;use thiserror::Error;use utoipa::ToSchema;@@ -49,8 +49,6 @@ use super::bm25::Bm25Weight;use super::models::linear::LinearRegression;use super::{inbound_similarity, query_centrality};-const PROXIMITY_ENABLED: bool = false;-#[derive(Debug, Error)]pub enum Error {#[error("unknown signal: {0}")]@@ -59,8 +57,6 @@ pub enum Error {#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash)]pub enum Signal {- #[serde(rename = "bm25")]- Bm25,#[serde(rename = "bm25_title")]Bm25Title,#[serde(rename = "bm25_title_bigrams")]@@ -101,34 +97,6 @@ pub enum Signal {Bm25TitleIfHomepage,#[serde(rename = "bm25_backlink_text")]Bm25BacklinkText,- #[serde(rename = "bm25_description")]- Bm25Description,-- /// All proximity signals are disabled for now.- /// We need to figure out how to make them faster.- #[serde(rename = "proximity_slop_0")]- ProximitySlop0,-- /// All proximity signals are disabled for now.- /// We need to figure out how to make them faster.- #[serde(rename = "proximity_slop_1")]- ProximitySlop1,-- /// All proximity signals are disabled for now.- /// We need to figure out how to make them faster.- #[serde(rename = "proximity_slop_2")]- ProximitySlop2,-- /// All proximity signals are disabled for now.- /// We need to figure out how to make them faster.- #[serde(rename = "proximity_slop_4")]- ProximitySlop4,-- /// All proximity signals are disabled for now.- /// We need to figure out how to make them faster.- #[serde(rename = "proximity_slop_8")]- ProximitySlop8,-#[serde(rename = "cross_encoder")]CrossEncoder,#[serde(rename = "host_centrality")]@@ -163,8 +131,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 40] = [- Signal::Bm25,+pub const ALL_SIGNALS: [Signal; 33] = [Signal::Bm25Title,Signal::Bm25TitleBigrams,Signal::Bm25TitleTrigrams,@@ -185,12 +152,6 @@ pub const ALL_SIGNALS: [Signal; 40] = [Signal::Bm25DomainIfHomepageNoTokenizer,Signal::Bm25TitleIfHomepage,Signal::Bm25BacklinkText,- Signal::Bm25Description,- Signal::ProximitySlop0,- Signal::ProximitySlop1,- Signal::ProximitySlop2,- Signal::ProximitySlop4,- Signal::ProximitySlop8,Signal::CrossEncoder,Signal::HostCentrality,Signal::PageCentrality,@@ -277,33 +238,26 @@ impl Signal {pub fn default_coefficient(&self) -> f64 {match self {- Signal::Bm25 => 0.0002,- Signal::Bm25Title => 0.004,- Signal::Bm25TitleBigrams => 0.0025,- Signal::Bm25TitleTrigrams => 0.0025,- Signal::Bm25CleanBody => 0.004,- Signal::Bm25CleanBodyBigrams => 0.0015,- Signal::Bm25CleanBodyTrigrams => 0.0015,- Signal::Bm25StemmedTitle => 0.0005,- Signal::Bm25StemmedCleanBody => 0.0005,+ Signal::Bm25Title => 0.0042,+ Signal::Bm25TitleBigrams => 0.0027,+ Signal::Bm25TitleTrigrams => 0.0027,+ Signal::Bm25CleanBody => 0.0042,+ Signal::Bm25CleanBodyBigrams => 0.0017,+ Signal::Bm25CleanBodyTrigrams => 0.0017,+ Signal::Bm25StemmedTitle => 0.0007,+ Signal::Bm25StemmedCleanBody => 0.0007,Signal::Bm25AllBody => 0.0,- Signal::Bm25Url => 0.00005,- Signal::Bm25Site => 0.00005,- Signal::Bm25Domain => 0.00005,- Signal::Bm25SiteNoTokenizer => 0.00005,- Signal::Bm25DomainNoTokenizer => 0.00005,- Signal::Bm25DomainNameNoTokenizer => 0.00005,- Signal::Bm25DomainIfHomepage => 0.0002,- Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.007,- Signal::Bm25DomainIfHomepageNoTokenizer => 0.007,- Signal::Bm25TitleIfHomepage => 0.00002,- Signal::Bm25BacklinkText => 0.003,- Signal::Bm25Description => 0.0,- Signal::ProximitySlop0 => 0.00,- Signal::ProximitySlop1 => 0.00,- Signal::ProximitySlop2 => 0.00,- Signal::ProximitySlop4 => 0.00,- Signal::ProximitySlop8 => 0.00,+ Signal::Bm25Url => 0.00015,+ Signal::Bm25Site => 0.00015,+ Signal::Bm25Domain => 0.00015,+ Signal::Bm25SiteNoTokenizer => 0.00015,+ Signal::Bm25DomainNoTokenizer => 0.00015,+ Signal::Bm25DomainNameNoTokenizer => 0.00015,+ Signal::Bm25DomainIfHomepage => 0.0004,+ Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0072,+ Signal::Bm25DomainIfHomepageNoTokenizer => 0.0072,+ Signal::Bm25TitleIfHomepage => 0.00022,+ Signal::Bm25BacklinkText => 0.0032,Signal::CrossEncoder => 0.17,Signal::HostCentrality => 1.0,Signal::PageCentrality => 4.0,@@ -442,13 +396,6 @@ impl Signal {let host_id = self.host_id(signal_aggregator, doc);host_id.map(|host_id| signal_aggregator.inbound_similarity(host_id))}- Signal::Bm25 => signal_aggregator.segment_reader.as_mut().map(|reader| {- reader- .text_fields- .values_mut()- .map(|field| bm25(field, doc))- .sum()- }),Signal::Bm25Title| Signal::Bm25TitleBigrams| Signal::Bm25TitleTrigrams@@ -468,8 +415,7 @@ impl Signal {| Signal::Bm25DomainNameIfHomepageNoTokenizer| Signal::Bm25DomainIfHomepageNoTokenizer| Signal::Bm25TitleIfHomepage- | Signal::Bm25BacklinkText- | Signal::Bm25Description => signal_aggregator.segment_reader.as_mut().map(|reader| {+ | Signal::Bm25BacklinkText => signal_aggregator.segment_reader.as_mut().map(|reader| {reader.text_fields.get_mut(self.as_textfield().unwrap())@@ -477,25 +423,6 @@ impl Signal {.unwrap_or(0.0)}),- Signal::ProximitySlop0- | Signal::ProximitySlop1- | Signal::ProximitySlop2- | Signal::ProximitySlop4- | Signal::ProximitySlop8 => {- let reader = signal_aggregator- .segment_reader- .as_mut()- .expect("set segment reader before computing signals");-- reader.proximity_scorers.get_mut(self).map(|scorer| {- let docset = &mut scorer.docset;- if docset.doc() == doc || (docset.doc() < doc && docset.seek(doc) == doc) {- docset.score() as f64- } else {- 0.0- }- })- }Signal::CrossEncoder => None, // this is calculated in a later stepSignal::LambdaMART => None,};@@ -572,8 +499,7 @@ impl Signal {.count() as f64;Some(score_slashes(num_slashes))}- Signal::Bm25- | Signal::Bm25Title+ Signal::Bm25Title| Signal::Bm25TitleBigrams| Signal::Bm25TitleTrigrams| Signal::Bm25CleanBody@@ -593,12 +519,6 @@ impl Signal {| Signal::Bm25DomainIfHomepageNoTokenizer| Signal::Bm25TitleIfHomepage| Signal::Bm25BacklinkText- | Signal::Bm25Description- | Signal::ProximitySlop0- | Signal::ProximitySlop1- | Signal::ProximitySlop2- | Signal::ProximitySlop4- | Signal::ProximitySlop8| Signal::CrossEncoder| Signal::InboundSimilarity| Signal::LambdaMART@@ -617,17 +537,6 @@ impl Signal {})}- fn proximity_slop(&self) -> Option<u32> {- match self {- Signal::ProximitySlop0 => Some(0),- Signal::ProximitySlop1 => Some(1),- Signal::ProximitySlop2 => Some(2),- Signal::ProximitySlop4 => Some(4),- Signal::ProximitySlop8 => Some(8),- _ => None,- }- }-fn as_fastfield(&self) -> Option<FastField> {match self {Signal::HostCentrality => Some(FastField::HostCentrality),@@ -666,7 +575,6 @@ impl Signal {}Signal::Bm25TitleIfHomepage => Some(TextField::TitleIfHomepage),Signal::Bm25BacklinkText => Some(TextField::BacklinkText),- Signal::Bm25Description => Some(TextField::Description),Signal::Bm25DomainIfHomepageNoTokenizer => Some(TextField::DomainIfHomepageNoTokenizer),_ => None,}@@ -743,13 +651,8 @@ struct OpticBoosts {rules: Vec<RuleBoost>,}-struct ProximityScorer {- docset: Box<dyn Scorer>,-}-struct SegmentReader {text_fields: EnumMap<TextField, TextFieldData>,- proximity_scorers: EnumMap<Signal, ProximityScorer>,optic_boosts: OpticBoosts,fastfield_reader: Arc<fastfield_reader::SegmentReader>,}@@ -941,67 +844,6 @@ impl SignalAggregator {optic_rule_boosts}- fn prepare_proximity_scorers(- &self,- tv_searcher: &tantivy::Searcher,- segment_reader: &tantivy::SegmentReader,- ) -> EnumMap<Signal, ProximityScorer> {- if !PROXIMITY_ENABLED {- return EnumMap::new();- }-- let mut proximity_scorers = EnumMap::new();- let proximity_fields = [TextField::Title, TextField::CleanBody];- let schema = tv_searcher.schema();-- if let Some(query) = &self.query {- if query.simple_terms.len() < 2 {- return proximity_scorers;- }-- for signal in ALL_SIGNALS {- if let Some(slop) = signal.proximity_slop() {- let mut queries = Vec::new();- for field in proximity_fields {- let tv_field = schema.get_field(field.name()).unwrap();- let mut tokenizer = field.indexing_tokenizer();- let mut terms = Vec::with_capacity(query.simple_terms.len());-- for term in &query.simple_terms {- let mut stream = tokenizer.token_stream(term);- while let Some(term) = stream.next() {- let term = tantivy::Term::from_field_text(tv_field, &term.text);- terms.push(term);- }- }-- let mut phrase_query = PhraseQuery::new(terms);- phrase_query.set_slop(slop);-- queries.push((- tantivy::query::Occur::Should,- Box::new(phrase_query) as Box<dyn tantivy::query::Query>,- ));- }-- let query = tantivy::query::BooleanQuery::new(queries);- let docset = query- .weight(tantivy::query::EnableScoring::Enabled {- searcher: tv_searcher,- statistics_provider: tv_searcher,- })- .unwrap()- .scorer(segment_reader, 1.0)- .unwrap();-- proximity_scorers.insert(signal, ProximityScorer { docset });- }- }- }-- proximity_scorers- }-pub fn register_segment(&mut self,tv_searcher: &tantivy::Searcher,@@ -1011,7 +853,6 @@ impl SignalAggregator {let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, fastfield_reader);- let proximity_scorers = self.prepare_proximity_scorers(tv_searcher, segment_reader);self.segment_reader = Some(SegmentReader {text_fields,@@ -1019,7 +860,6 @@ impl SignalAggregator {optic_boosts: OpticBoosts {rules: optic_rule_boosts,},- proximity_scorers,});Ok(())@@ -1139,101 +979,3 @@ pub struct SignalScore {pub coefficient: f64,pub value: f64,}--#[cfg(test)]-mod tests {- use std::collections::HashMap;-- use crate::{- index::Index,- searcher::{LocalSearcher, SearchQuery},- };-- use super::*;-- fn signals_for_query(searcher: &LocalSearcher, q: &str) -> HashMap<Signal, SignalScore> {- let res = searcher- .search(&SearchQuery {- query: q.to_string(),- ..Default::default()- })- .unwrap()- .webpages;- res[0].ranking_signals.clone().unwrap()- }- #[test]- #[ignore = "proximity signals are currently disabled"]- fn test_proximity() {- let mut index = Index::temporary().expect("Unable to open index");-- let mut webpage = Webpage::new(- &format!(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- {}- </body>- </html>- "#,- crate::rand_words(1_000)- ),- "https://www.example.com",- )- .unwrap();-- webpage- .html- .set_clean_text("a b c d e f g h i j k l m n o p q r s t u v w x y z".to_string());-- index.insert(webpage).expect("failed to insert webpage");-- index.commit().unwrap();-- let searcher = LocalSearcher::new(index);-- let signals = signals_for_query(&searcher, "a b");- assert!(signals[&Signal::ProximitySlop8].value > 0.0);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop1].value);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop0].value);-- let signals = signals_for_query(&searcher, "a c");- assert!(signals[&Signal::ProximitySlop8].value > 0.0);- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop1].value);-- let signals = signals_for_query(&searcher, "a d");- assert!(signals[&Signal::ProximitySlop8].value > 0.0);- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);-- let signals = signals_for_query(&searcher, "a e");- assert!(signals[&Signal::ProximitySlop8].value > 0.0);- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);-- let signals = signals_for_query(&searcher, "a g");- assert!(signals[&Signal::ProximitySlop8].value > 0.0);- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop4].value, 0.0);-- let signals = signals_for_query(&searcher, "a k");- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop4].value, 0.0);- assert_eq!(signals[&Signal::ProximitySlop8].value, 0.0);- }-}diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rsindex 79ad0531..1d0e310c 100644--- a/core/src/search_prettifier/entity.rs+++ b/core/src/search_prettifier/entity.rs@@ -29,7 +29,7 @@ use crate::entity_index::{pub struct DisplayedEntity {pub title: String,pub small_abstract: EntitySnippet,- pub image_base64: Option<String>,+ pub image_id: Option<String>,pub related_entities: Vec<DisplayedEntity>,pub info: Vec<(String, EntitySnippet)>,pub match_score: f32,@@ -47,7 +47,7 @@ impl From<EntityMatch> for DisplayedEntity {Self {title: m.entity.title,small_abstract,- image_base64: m.entity.image_base64,+ image_id: m.entity.image_id,related_entities: m.entity.related_entitiesdiff --git a/core/src/search_prettifier/stack_overflow.rs b/core/src/search_prettifier/stack_overflow.rsindex e688c081..da9e59f0 100644--- a/core/src/search_prettifier/stack_overflow.rs+++ b/core/src/search_prettifier/stack_overflow.rs@@ -126,6 +126,27 @@ fn schema_item_to_stackoverflow_answer(}}+fn limit_chars(passages: &[CodeOrText], limit: usize) -> Vec<CodeOrText> {+ let mut res = Vec::new();+ let mut taken_chars = 0;++ for p in passages {+ res.push(p.clone());+ let s = match p {+ CodeOrText::Code(s) => s,+ CodeOrText::Text(s) => s,+ };++ if taken_chars + s.len() > limit {+ break;+ }++ taken_chars += s.len();+ }++ res+}+pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {match webpage.schema_org@@ -192,10 +213,20 @@ pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {answers.push(answer);}- Ok(Snippet::StackOverflowQA {- question: StackOverflowQuestion { body: question },- answers: answers.into_iter().take(3).collect(),- })+ let question = StackOverflowQuestion {+ body: limit_chars(&question, 512),+ };++ let answers: Vec<_> = answers+ .into_iter()+ .map(|answer| StackOverflowAnswer {+ body: limit_chars(&answer.body, 512),+ ..answer+ })+ .take(3)+ .collect();++ Ok(Snippet::StackOverflowQA { question, answers })}None => Err(Error::InvalidStackoverflowSchema.into()),}diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex 60d0118e..948d7b50 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -26,6 +26,7 @@ use url::Url;use crate::bangs::{Bang, BangHit};use crate::config::{ApiConfig, ApiThresholds, CollectorConfig};+use crate::image_store::Image;use crate::inverted_index::RetrievedWebpage;#[cfg(not(feature = "libtorch"))]use crate::ranking::models::cross_encoder::DummyCrossEncoder;@@ -510,6 +511,10 @@ impl ApiSearcher {pub async fn get_webpage(&self, url: &str) -> Result<RetrievedWebpage> {self.distributed_searcher.get_webpage(url).await}++ pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {+ self.distributed_searcher.get_entity_image(image_id).await+ }}fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 3f02dcf6..6199f07a 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -17,6 +17,7 @@use crate::{distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff},entrypoint::search_server::{self, SearchService},+ image_store::Image,inverted_index::{self, RetrievedWebpage},ranking::pipeline::{AsRankingWebsite, RankingWebsite},Result,@@ -143,6 +144,19 @@ impl RemoteSearcher {.await.unwrap()}++ async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {+ let conn = self.conn().await;++ conn.send_with_timeout(+ &search_server::GetEntityImage {+ image_id: image_id.to_string(),+ },+ Duration::from_secs(60),+ )+ .await+ .map_err(|e| e.into())+ }}#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]@@ -217,6 +231,10 @@ impl Shard {async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {self.replica().get_homepage_descriptions(urls).await}++ async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {+ self.replica().get_entity_image(image_id).await+ }}#[derive(Debug)]@@ -344,4 +362,11 @@ impl DistributedSearcher {.flatten().collect()}++ pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {+ match self.shards().await.first() {+ Some(s) => s.get_entity_image(image_id).await,+ None => Ok(None),+ }+ }}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex e370c08c..e7cebb03 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -23,6 +23,7 @@ use url::Url;use crate::config::{CollectorConfig, SnippetConfig};use crate::entity_index::{EntityIndex, EntityMatch};+use crate::image_store::Image;use crate::index::Index;use crate::inverted_index::RetrievedWebpage;use crate::query::Query;@@ -69,7 +70,7 @@ struct InvertedIndexResult {impl LocalSearcher {pub fn new(index: Index) -> Self {let mut index = index;- // index.optimize_for_search().unwrap();+ index.optimize_for_search().unwrap();LocalSearcher {index,@@ -409,6 +410,12 @@ impl LocalSearcher {pub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {self.index.get_homepage(url)}++ pub fn get_entity_image(&self, image_id: &str) -> Option<Image> {+ self.entity_index+ .as_ref()+ .and_then(|index| index.retrieve_image(image_id))+ }}#[cfg(test)]diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex 5ea033ec..6e80a0a3 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -19,7 +19,7 @@ use std::ops::Range;use crate::config::SnippetConfig;use crate::query::Query;use crate::spell::sentence_ranges;-use crate::tokenizer::{BigramTokenizer, Stemmed, Tokenizer, TrigramTokenizer};+use crate::tokenizer::{BigramTokenizer, Normal, Stemmed, Tokenizer, TrigramTokenizer};use crate::webpage::region::Region;use hashbrown::{HashMap, HashSet};use utoipa::ToSchema;@@ -81,6 +81,7 @@ impl SnippetBuilder {fn highlight(&mut self, terms: &HashSet<String>, lang: whatlang::Lang) {for mut tokenizer in [Tokenizer::Stemmed(Stemmed::with_forced_language(lang)),+ Tokenizer::Normal(Normal::default()),Tokenizer::Bigram(BigramTokenizer::default()),Tokenizer::Trigram(TrigramTokenizer::default()),] {@@ -94,9 +95,10 @@ impl SnippetBuilder {}// remove overlapping ranges- self.highlights.sort_by(|a, b| a.start.cmp(&b.start));self.highlights- .dedup_by(|a, b| a.start == b.start && a.end > b.end);+ .sort_by(|a, b| a.start.cmp(&b.start).then(a.end.cmp(&b.end)));+ self.highlights+ .dedup_by(|a, b| a.start == b.start && a.end >= b.end);}fn build(self) -> TextSnippet {@@ -136,9 +138,8 @@ fn snippet_string_builder(terms: &[String],lang: whatlang::Lang,config: SnippetConfig,+ mut tokenizer: Tokenizer,) -> SnippetBuilder {- let mut tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));-let terms: HashSet<String> = terms.iter().flat_map(|term| {@@ -276,7 +277,20 @@ fn snippet_string(lang: whatlang::Lang,config: SnippetConfig,) -> TextSnippet {- snippet_string_builder(text, terms, lang, config).build()+ let tokenizer = Tokenizer::Normal(Normal::default());+ let snip = snippet_string_builder(text, terms, lang, config.clone(), tokenizer).build();++ if !snip.fragments.is_empty()+ && snip+ .fragments+ .iter()+ .any(|f| f.kind == TextSnippetFragmentKind::Highlighted)+ {+ return snip;+ }++ let tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));+ snippet_string_builder(text, terms, lang, config, tokenizer).build()}pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfig) -> TextSnippet {@@ -523,18 +537,17 @@ Survey in 2016, 2017, and 2018."#;#[test]fn compounded_terms() {- let mut snip = snippet_string_builder(+ let snip = snippet_string_builder("this is a test",&["thisis".to_string()],whatlang::Lang::Eng,SnippetConfig::default(),+ Tokenizer::Normal(Normal::default()),);let mut terms = HashSet::new();terms.insert("thisis".to_string());- snip.highlight(&terms, whatlang::Lang::Eng);-assert_eq!(highlight(Snippet::Normal {date: None,diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex 458c11fb..df6c02be 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -98,14 +98,6 @@ const sse = <T>(};export const api = {- alice: (- query: {- message: string;- optic: string;- prevState: EncodedSavedState;- },- options?: ApiOptions,- ) => sse<ExecutionState>('GET', `/beta/api/alice?${new URLSearchParams(query)}`, options),autosuggest: (params: {q: string;@@ -119,8 +111,6 @@ export const api = {),exploreExport: (body: ExploreExportOpticParams, options?: ApiOptions) =>requestPlain('POST', `/beta/api/explore/export`, body, options),- factCheck: (body: FactCheckParams, options?: ApiOptions) =>- requestJson<FactCheckResponse>('POST', `/beta/api/fact_check`, body, options),search: (body: ApiSearchQuery, options?: ApiOptions) =>requestJson<ApiSearchResult>('POST', `/beta/api/search`, body, options),sitesExport: (body: SitesExportOpticParams, options?: ApiOptions) =>@@ -246,7 +236,7 @@ export type DisplayedAnswer = {url: string;};export type DisplayedEntity = {- imageBase64?: string;+ imageId?: string;info: string & EntitySnippet[][];matchScore: number;relatedEntities: DisplayedEntity[];@@ -274,8 +264,6 @@ export type DisplayedWebpage = {title: string;url: string;};-export type EncodedEncryptedState = string;-export type EncodedSavedState = string;export type EntitySnippet = {fragments: EntitySnippetFragment[];};@@ -290,35 +278,10 @@ export type EntitySnippetFragment =text: string;};export type Example = string;-export type ExecutionState =- | {- query: string;- type: 'beginSearch';- }- | {- query: string;- result: SimplifiedWebsite[];- type: 'searchResult';- }- | {- text: string;- type: 'speaking';- }- | {- state: EncodedEncryptedState;- type: 'done';- };export type ExploreExportOpticParams = {chosenSites: string[];similarSites: string[];};-export type FactCheckParams = {- claim: string;- evidence: string;-};-export type FactCheckResponse = {- score: number;-};export type FullEdge = {from: Node;label: string;@@ -367,12 +330,6 @@ export type SimilarSitesParams = {sites: string[];topN: number;};-export type SimplifiedWebsite = {- site: string;- text: string;- title: string;- url: string;-};export type SiteRankings = {blocked: string[];disliked: string[];diff --git a/frontend/src/lib/components/Code.svelte b/frontend/src/lib/components/Code.svelteindex 27ac0fd3..b6da7948 100644--- a/frontend/src/lib/components/Code.svelte+++ b/frontend/src/lib/components/Code.svelte@@ -1,20 +1,13 @@<script lang="ts">- import { page } from '$app/stores';+ import {HighlightAuto} from "svelte-highlight";export let code: string;-- const component =- $page.data.globals?.highlightjs?.HighlightAuto ||- import('svelte-highlight').then(({ HighlightAuto }) => HighlightAuto);+ export let transparentBackground = false;</script>-<div class="code rounded-lg bg-base-200 text-sm text-neutral-focus">+<div class="code rounded-lg {transparentBackground ? '' : 'bg-base-200'} text-sm text-neutral-focus"><div class="overflow-auto px-3 py-2">- {#await component}- <pre><code>{code}</code></pre>- {:then comp}- <svelte:component this={comp} {code} />- {/await}+ <HighlightAuto code={code}/></div></div>diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelteindex 8f5504c3..e8a8b2c9 100644--- a/frontend/src/routes/search/+page.svelte+++ b/frontend/src/routes/search/+page.svelte@@ -153,7 +153,7 @@<div class="mx-auto flex w-full justify-between"><div class="flex h-full flex-col justify-center space-x-2 text-sm text-neutral"><p class="h-fit">- {#if results.numHits }+ {#if results.numHits != null }Found {results.numHits.toLocaleString()} results in {((results.searchDurationMs ?? 0) / 1000).toFixed(2)}diff --git a/frontend/src/routes/search/Entity.svelte b/frontend/src/routes/search/Entity.svelteindex 1b9d7a2a..59e13040 100644--- a/frontend/src/routes/search/Entity.svelte+++ b/frontend/src/routes/search/Entity.svelte@@ -1,5 +1,5 @@<script lang="ts">- import type { DisplayedEntity } from '$lib/api';+ import { getApiBase, type DisplayedEntity } from '$lib/api';import EntitySnippet from '$lib/components/EntitySnippet.svelte';export let entity: DisplayedEntity;@@ -7,13 +7,13 @@<div class="flex w-full justify-center"><div class="flex w-full flex-col items-center">- {#if entity.imageBase64}+ {#if entity.imageId}<div class="w-lg mb-5"><a href="https://en.wikipedia.org/wiki/{encodeURI(entity.title)}"><imgalt="Image of {entity.title}"class="h-full w-full rounded-full"- src="data:image/png;base64, {entity.imageBase64}"+ src="{getApiBase()}/beta/api/entity_image?imageId={entity.imageId}"/></a></div>@@ -53,13 +53,13 @@<div class="flex overflow-scroll">{#each entity.relatedEntities as related (related.title)}<div class="flex flex-col items-center p-4">- {#if related.imageBase64}+ {#if related.imageId != null}<div class="mb-3 h-20 w-20"><a href="/search?q={encodeURIComponent(related.title)}"><imgalt="Image of {related.title}"class="h-full w-full rounded-full object-cover"- src="data:image/png;base64, {related.imageBase64}"+ src="{getApiBase()}/beta/api/entity_image?imageId={related.imageId}"/></a></div>diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelteindex f6e2f028..9b27fd4b 100644--- a/frontend/src/routes/search/Result.svelte+++ b/frontend/src/routes/search/Result.svelte@@ -7,6 +7,7 @@import { derived } from 'svelte/store';import { improvements } from '$lib/improvements';import TextSnippet from '$lib/components/TextSnippet.svelte';+ import StackOverflowSnippet from './StackOverflowSnippet.svelte';export let webpage: DisplayedWebpage;export let resultIndex: number;@@ -54,13 +55,23 @@<div class="snippet"><div class="line-clamp-3"><div class="inline">- <span id="snippet-text" class="snippet-text [&:nth-child(2)]:before:content-['—']">- {webpage.snippet.date || ''}- <TextSnippet snippet={webpage.snippet.text} />+ <span id="snippet-text" class="snippet-text">+ {#if webpage.snippet.date}+ <span class="text-neutral">+ {webpage.snippet.date} -+ </span>+ {/if}+ <span>+ <TextSnippet snippet={webpage.snippet.text} />+ </span></span></div></div></div>+ {:else if webpage.snippet.type == 'stackOverflowQA'}+ <div class="snippet">+ <StackOverflowSnippet question={webpage.snippet.question} answers={webpage.snippet.answers} />+ </div>{/if}</div></div>diff --git a/frontend/src/routes/search/StackOverflowSnippet.svelte b/frontend/src/routes/search/StackOverflowSnippet.sveltenew file mode 100644index 00000000..11132d40--- /dev/null+++ b/frontend/src/routes/search/StackOverflowSnippet.svelte@@ -0,0 +1,52 @@++<script lang="ts">+ import type { StackOverflowQuestion, StackOverflowAnswer } from "$lib/api";+ import Code from '$lib/components/Code.svelte';+ import HandThumbUp from '~icons/heroicons/hand-thumb-up';+ import Check from '~icons/heroicons/check';++ export let question: StackOverflowQuestion;+ export let answers: StackOverflowAnswer[];+</script>+<div class="line-clamp-2">+{#each question.body as passage}+ {#if passage.type == "text"}+ {passage.value}+ {/if}+{/each}+</div>+<div class="flex space-x-4 pt-2">+ {#each answers.slice(0, 3) as answer}+ <div class="w-1/3 overflow-hidden">+ <a class="block h-56 hover:bg-base-200/80 p-2 border overflow-hidden rounded-lg" href={answer.url}>+ <div class="flex w-full items-center justify-between space-x-1 mb-1 text-neutral text-xs">+ <div class="flex">+ {answer.date}+ </div>+ <div class="flex space-x-1">+ <span class="h-fit">+ {answer.upvotes}+ </span>+ <div class="h-fit">+ <HandThumbUp class="w-4" />+ </div>+ {#if answer.accepted}+ <div class="h-fit text-green-600">+ <Check class="w-4" />+ </div>+ {/if}+ </div>+ </div>+ <div>+ {#each answer.body as passage}+ {#if passage.type == "text"}+ {passage.value}+ {:else if passage.type == "code"}+ <Code code={passage.value} transparentBackground={true} />+ {/if}+ {/each}+ </div>+ </a>+ </div>+ {/each}+</div>\ No newline at end of filediff --git a/frontend/svelte.config.js b/frontend/svelte.config.jsindex 91ee6927..8b1b0a03 100644--- a/frontend/svelte.config.js+++ b/frontend/svelte.config.js@@ -32,7 +32,7 @@ const config = {// NOTE: Disabled in order to fetch optic sources client-side// 'connect-src': ["'self'", 'http://localhost:3000/'],'connect-src': ['*'],- 'img-src': ["'self'", 'data:'],+ 'img-src': ["'self'", 'data:', 'trystract.com', '0.0.0.0:3000'],'style-src': ["'self'", "'unsafe-inline'"],},},