Skip to content

web search done right

A bunch of performance optimizations during search. * use image link instead of base64 embedded to lower serp size. * embed 'bm25' signal into all the 'bm25_*' signals. The math should be equivalent and prevents us calculating the same signals twice. * remove proximity ranking signals as they were disabled. Need to figure out how to make them faster. * faster snippet generation by first trying to generate snippet without stemming. This will succeed in most results and is vastly faster.

Mikkel Denker38342798e12f
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index ae343c38..99e1561d 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -223,6 +223,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
)
.route("/api/sites/export", post(sites::sites_export_optic))
.route("/api/explore/export", post(explore::explore_export_optic))
+ .route("/api/entity_image", get(search::entity_image))
.layer(cors_layer()),
)
.with_state(state))
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index 7e98bf2d..303b9ee5 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -164,3 +164,39 @@ pub async fn api(
},
}
}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct EntityImageParams {
+ pub image_id: String,
+}
+
+#[utoipa::path(
+ post,
+ path = "/beta/api/search",
+ request_body(content = ApiSearchQuery),
+ responses(
+ (status = 200, description = "Search results", body = ApiSearchResult),
+ )
+)]
+pub async fn entity_image(
+ extract::Query(query): extract::Query<EntityImageParams>,
+ extract::State(state): extract::State<Arc<State>>,
+) -> Result<impl IntoResponse, StatusCode> {
+ match state.searcher.get_entity_image(&query.image_id).await {
+ Ok(Some(result)) => {
+ let bytes = result.as_raw_bytes();
+
+ Ok((
+ ([(axum::http::header::CONTENT_TYPE, "image/png")]),
+ axum::response::AppendHeaders([(axum::http::header::CONTENT_TYPE, "image/png")]),
+ bytes,
+ ))
+ }
+ Ok(None) => Err(StatusCode::NOT_FOUND),
+ Err(err) => {
+ tracing::error!("{:?}", err);
+ Err(StatusCode::INTERNAL_SERVER_ERROR)
+ }
+ }
+}
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rs
index 85aacad8..822b0ee0 100644
--- a/core/src/entity_index/mod.rs
+++ b/core/src/entity_index/mod.rs
@@ -149,7 +149,7 @@ fn wikipedify_url(url: &str) -> Vec<Url> {
pub struct StoredEntity {
pub title: String,
pub entity_abstract: String,
- pub image_base64: Option<String>,
+ pub image_id: Option<String>,
pub related_entities: Vec<EntityMatch>,
pub best_info: Vec<(String, Span)>,
pub links: Vec<Link>,
@@ -304,7 +304,7 @@ impl EntityIndex {
EntityMatch { entity, score }
})
- .filter(|m| m.entity.image_base64.is_some())
+ .filter(|m| m.entity.image_id.is_some())
.take(4)
.collect(),
Err(_) => Vec::new(),
@@ -412,9 +412,12 @@ impl EntityIndex {
Vec::new()
};
- let image_base64 = self
- .retrieve_image(&title)
- .map(|image| base64::encode(image.as_raw_bytes()));
+ let image_id = base64::encode(&title);
+ let image_id = if self.retrieve_image(&image_id).is_some() {
+ Some(image_id)
+ } else {
+ None
+ };
let links: Vec<Link> = if get_links {
bincode::deserialize(
@@ -433,15 +436,18 @@ impl EntityIndex {
StoredEntity {
title,
entity_abstract,
- image_base64,
+ image_id,
related_entities,
best_info,
links,
}
}
- pub fn retrieve_image(&self, key: &String) -> Option<Image> {
- self.image_store.get(key)
+ pub fn retrieve_image(&self, key: &str) -> Option<Image> {
+ let key = base64::decode(key).ok()?;
+ let key = String::from_utf8(key).ok()?;
+
+ self.image_store.get(&key)
}
pub fn get_attribute_occurrence(&self, attribute: &String) -> Option<u32> {
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 5909d6ea..a3efc046 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -28,6 +28,7 @@ use crate::{
sonic,
},
entity_index::EntityIndex,
+ image_store::Image,
index::Index,
inverted_index::{self, RetrievedWebpage},
ranking::{
@@ -45,6 +46,7 @@ sonic_service!(
Search,
GetWebpage,
GetHomepageDescriptions,
+ GetEntityImage,
]
);
@@ -178,6 +180,18 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
}
}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GetEntityImage {
+ pub image_id: String,
+}
+#[async_trait::async_trait]
+impl sonic::service::Message<SearchService> for GetEntityImage {
+ type Response = Option<Image>;
+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
+ Ok(server.local_searcher.get_entity_image(&self.image_id))
+ }
+}
+
pub async fn run(config: config::SearchServerConfig) -> Result<()> {
let addr = config.host;
let server = SearchService::new(config).await?.bind(addr).await.unwrap();
diff --git a/core/src/image_store.rs b/core/src/image_store.rs
index 69f4a54c..f36e273b 100644
--- a/core/src/image_store.rs
+++ b/core/src/image_store.rs
@@ -203,6 +203,10 @@ impl Image {
bytes
}
+
+ pub fn resize(self, width: u32, height: u32) -> Self {
+ ResizeFilter { width, height }.transform(self)
+ }
}
#[cfg(test)]
diff --git a/core/src/index.rs b/core/src/index.rs
index e8ed9e02..8da18312 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -335,7 +335,7 @@ mod tests {
.ranking_signals
.as_ref()
.unwrap()
- .get(&crate::ranking::Signal::Bm25)
+ .get(&crate::ranking::Signal::Bm25Title)
.unwrap())
.all(|&v| v.value > 0.0));
}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index a6081471..e7f39fec 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -668,128 +668,6 @@ mod tests {
assert_eq!(&res.webpages[0].url, "https://www.centrality.com/");
}
- #[test]
- #[ignore = "proximity signals are currently disabled"]
- fn term_proximity_ranking() {
- let mut index = Index::temporary().expect("Unable to open index");
- let mut w = Webpage {
- html: Html::parse(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT} termA termB d d d d d d d d d {}
- </body>
- </html>
- "#,
- crate::rand_words(100)
- ),
- "https://www.first.com",
- )
- .unwrap(),
- backlink_labels: vec![],
- host_centrality: 1.0,
- pre_computed_score: 0.0,
- fetch_time_ms: 500,
- page_centrality: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
- };
- w.html.set_clean_text(format!(
- "{CONTENT} termA termB d d d d d d d d d {}",
- crate::rand_words(100)
- ));
-
- index.insert(w).expect("failed to insert webpage");
- let mut w = Webpage {
- html: Html::parse(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT} termA d d d d d d d d d termB {}
- </body>
- </html>
- "#,
- crate::rand_words(100)
- ),
- "https://www.third.com",
- )
- .unwrap(),
- backlink_labels: vec![],
- host_centrality: 1.0,
- fetch_time_ms: 500,
- pre_computed_score: 0.0,
- page_centrality: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
- };
- w.html.set_clean_text(format!(
- "{CONTENT} termA d d d d d d d d d termB {}",
- crate::rand_words(100)
- ));
- index.insert(w).expect("failed to insert webpage");
-
- let mut w = Webpage {
- html: Html::parse(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT} termA d d d d termB d d d d d {}
- </body>
- </html>
- "#,
- crate::rand_words(100)
- ),
- "https://www.second.com",
- )
- .unwrap(),
- backlink_labels: vec![],
- host_centrality: 1.0,
- fetch_time_ms: 500,
- pre_computed_score: 0.0,
- page_centrality: 0.0,
-
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
- };
- w.html.set_clean_text(format!(
- "{CONTENT} termA d d d d termB d d d d d {}",
- crate::rand_words(100)
- ));
-
- index.insert(w).expect("failed to insert webpage");
- index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::new(index);
-
- let result = searcher
- .search(&SearchQuery {
- query: "termA termB".to_string(),
- ..Default::default()
- })
- .expect("Search failed");
-
- assert_eq!(result.webpages.len(), 3);
- assert_eq!(result.webpages[0].url, "https://www.first.com/");
- assert_eq!(result.webpages[1].url, "https://www.second.com/");
- assert_eq!(result.webpages[2].url, "https://www.third.com/");
- }
-
#[test]
fn fetch_time_ranking() {
let mut index = Index::temporary().expect("Unable to open index");
diff --git a/core/src/ranking/models/lambdamart.rs b/core/src/ranking/models/lambdamart.rs
index 94c9526e..2a8019ae 100644
--- a/core/src/ranking/models/lambdamart.rs
+++ b/core/src/ranking/models/lambdamart.rs
@@ -275,6 +275,7 @@ mod tests {
use super::*;
#[test]
+ #[ignore = "train new model with updated signals"]
fn simple() {
let tree = include_str!("../../../testcases/lambdamart.txt");
let tree = Tree::parse(tree).unwrap();
@@ -292,7 +293,6 @@ mod tests {
features.insert(Signal::Bm25Domain, 89.1253814697);
features.insert(Signal::Bm25CleanBody, 54.1200180054);
features.insert(Signal::FetchTimeMs, 0.0018281536);
- features.insert(Signal::Bm25, 952.8474273682);
features.insert(Signal::Bm25AllBody, 43.6125068665);
features.insert(Signal::Bm25Site, 87.6877670288);
features.insert(Signal::Bm25TitleTrigrams, 86.8362731934);
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index cb2336eb..e952bb1b 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -408,7 +408,7 @@ mod tests {
.map(|i| -> RankingWebsite {
let mut signals = EnumMap::new();
signals.insert(
- Signal::Bm25,
+ Signal::HostCentrality,
SignalScore {
coefficient: 1.0,
value: 1.0 / i as f64,
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 980d6396..d4331963 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -32,7 +32,7 @@ use std::str::FromStr;
use std::sync::Arc;
use tantivy::fieldnorm::FieldNormReader;
use tantivy::postings::SegmentPostings;
-use tantivy::query::{PhraseQuery, Query as _, Scorer};
+use tantivy::query::{Query as _, Scorer};
use tantivy::tokenizer::Tokenizer;
use thiserror::Error;
use utoipa::ToSchema;
@@ -49,8 +49,6 @@ use super::bm25::Bm25Weight;
use super::models::linear::LinearRegression;
use super::{inbound_similarity, query_centrality};
-const PROXIMITY_ENABLED: bool = false;
-
#[derive(Debug, Error)]
pub enum Error {
#[error("unknown signal: {0}")]
@@ -59,8 +57,6 @@ pub enum Error {
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Signal {
- #[serde(rename = "bm25")]
- Bm25,
#[serde(rename = "bm25_title")]
Bm25Title,
#[serde(rename = "bm25_title_bigrams")]
@@ -101,34 +97,6 @@ pub enum Signal {
Bm25TitleIfHomepage,
#[serde(rename = "bm25_backlink_text")]
Bm25BacklinkText,
- #[serde(rename = "bm25_description")]
- Bm25Description,
-
- /// All proximity signals are disabled for now.
- /// We need to figure out how to make them faster.
- #[serde(rename = "proximity_slop_0")]
- ProximitySlop0,
-
- /// All proximity signals are disabled for now.
- /// We need to figure out how to make them faster.
- #[serde(rename = "proximity_slop_1")]
- ProximitySlop1,
-
- /// All proximity signals are disabled for now.
- /// We need to figure out how to make them faster.
- #[serde(rename = "proximity_slop_2")]
- ProximitySlop2,
-
- /// All proximity signals are disabled for now.
- /// We need to figure out how to make them faster.
- #[serde(rename = "proximity_slop_4")]
- ProximitySlop4,
-
- /// All proximity signals are disabled for now.
- /// We need to figure out how to make them faster.
- #[serde(rename = "proximity_slop_8")]
- ProximitySlop8,
-
#[serde(rename = "cross_encoder")]
CrossEncoder,
#[serde(rename = "host_centrality")]
@@ -163,8 +131,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 40] = [
- Signal::Bm25,
+pub const ALL_SIGNALS: [Signal; 33] = [
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
Signal::Bm25TitleTrigrams,
@@ -185,12 +152,6 @@ pub const ALL_SIGNALS: [Signal; 40] = [
Signal::Bm25DomainIfHomepageNoTokenizer,
Signal::Bm25TitleIfHomepage,
Signal::Bm25BacklinkText,
- Signal::Bm25Description,
- Signal::ProximitySlop0,
- Signal::ProximitySlop1,
- Signal::ProximitySlop2,
- Signal::ProximitySlop4,
- Signal::ProximitySlop8,
Signal::CrossEncoder,
Signal::HostCentrality,
Signal::PageCentrality,
@@ -277,33 +238,26 @@ impl Signal {
pub fn default_coefficient(&self) -> f64 {
match self {
- Signal::Bm25 => 0.0002,
- Signal::Bm25Title => 0.004,
- Signal::Bm25TitleBigrams => 0.0025,
- Signal::Bm25TitleTrigrams => 0.0025,
- Signal::Bm25CleanBody => 0.004,
- Signal::Bm25CleanBodyBigrams => 0.0015,
- Signal::Bm25CleanBodyTrigrams => 0.0015,
- Signal::Bm25StemmedTitle => 0.0005,
- Signal::Bm25StemmedCleanBody => 0.0005,
+ Signal::Bm25Title => 0.0042,
+ Signal::Bm25TitleBigrams => 0.0027,
+ Signal::Bm25TitleTrigrams => 0.0027,
+ Signal::Bm25CleanBody => 0.0042,
+ Signal::Bm25CleanBodyBigrams => 0.0017,
+ Signal::Bm25CleanBodyTrigrams => 0.0017,
+ Signal::Bm25StemmedTitle => 0.0007,
+ Signal::Bm25StemmedCleanBody => 0.0007,
Signal::Bm25AllBody => 0.0,
- Signal::Bm25Url => 0.00005,
- Signal::Bm25Site => 0.00005,
- Signal::Bm25Domain => 0.00005,
- Signal::Bm25SiteNoTokenizer => 0.00005,
- Signal::Bm25DomainNoTokenizer => 0.00005,
- Signal::Bm25DomainNameNoTokenizer => 0.00005,
- Signal::Bm25DomainIfHomepage => 0.0002,
- Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.007,
- Signal::Bm25DomainIfHomepageNoTokenizer => 0.007,
- Signal::Bm25TitleIfHomepage => 0.00002,
- Signal::Bm25BacklinkText => 0.003,
- Signal::Bm25Description => 0.0,
- Signal::ProximitySlop0 => 0.00,
- Signal::ProximitySlop1 => 0.00,
- Signal::ProximitySlop2 => 0.00,
- Signal::ProximitySlop4 => 0.00,
- Signal::ProximitySlop8 => 0.00,
+ Signal::Bm25Url => 0.00015,
+ Signal::Bm25Site => 0.00015,
+ Signal::Bm25Domain => 0.00015,
+ Signal::Bm25SiteNoTokenizer => 0.00015,
+ Signal::Bm25DomainNoTokenizer => 0.00015,
+ Signal::Bm25DomainNameNoTokenizer => 0.00015,
+ Signal::Bm25DomainIfHomepage => 0.0004,
+ Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0072,
+ Signal::Bm25DomainIfHomepageNoTokenizer => 0.0072,
+ Signal::Bm25TitleIfHomepage => 0.00022,
+ Signal::Bm25BacklinkText => 0.0032,
Signal::CrossEncoder => 0.17,
Signal::HostCentrality => 1.0,
Signal::PageCentrality => 4.0,
@@ -442,13 +396,6 @@ impl Signal {
let host_id = self.host_id(signal_aggregator, doc);
host_id.map(|host_id| signal_aggregator.inbound_similarity(host_id))
}
- Signal::Bm25 => signal_aggregator.segment_reader.as_mut().map(|reader| {
- reader
- .text_fields
- .values_mut()
- .map(|field| bm25(field, doc))
- .sum()
- }),
Signal::Bm25Title
| Signal::Bm25TitleBigrams
| Signal::Bm25TitleTrigrams
@@ -468,8 +415,7 @@ impl Signal {
| Signal::Bm25DomainNameIfHomepageNoTokenizer
| Signal::Bm25DomainIfHomepageNoTokenizer
| Signal::Bm25TitleIfHomepage
- | Signal::Bm25BacklinkText
- | Signal::Bm25Description => signal_aggregator.segment_reader.as_mut().map(|reader| {
+ | Signal::Bm25BacklinkText => signal_aggregator.segment_reader.as_mut().map(|reader| {
reader
.text_fields
.get_mut(self.as_textfield().unwrap())
@@ -477,25 +423,6 @@ impl Signal {
.unwrap_or(0.0)
}),
- Signal::ProximitySlop0
- | Signal::ProximitySlop1
- | Signal::ProximitySlop2
- | Signal::ProximitySlop4
- | Signal::ProximitySlop8 => {
- let reader = signal_aggregator
- .segment_reader
- .as_mut()
- .expect("set segment reader before computing signals");
-
- reader.proximity_scorers.get_mut(self).map(|scorer| {
- let docset = &mut scorer.docset;
- if docset.doc() == doc || (docset.doc() < doc && docset.seek(doc) == doc) {
- docset.score() as f64
- } else {
- 0.0
- }
- })
- }
Signal::CrossEncoder => None, // this is calculated in a later step
Signal::LambdaMART => None,
};
@@ -572,8 +499,7 @@ impl Signal {
.count() as f64;
Some(score_slashes(num_slashes))
}
- Signal::Bm25
- | Signal::Bm25Title
+ Signal::Bm25Title
| Signal::Bm25TitleBigrams
| Signal::Bm25TitleTrigrams
| Signal::Bm25CleanBody
@@ -593,12 +519,6 @@ impl Signal {
| Signal::Bm25DomainIfHomepageNoTokenizer
| Signal::Bm25TitleIfHomepage
| Signal::Bm25BacklinkText
- | Signal::Bm25Description
- | Signal::ProximitySlop0
- | Signal::ProximitySlop1
- | Signal::ProximitySlop2
- | Signal::ProximitySlop4
- | Signal::ProximitySlop8
| Signal::CrossEncoder
| Signal::InboundSimilarity
| Signal::LambdaMART
@@ -617,17 +537,6 @@ impl Signal {
})
}
- fn proximity_slop(&self) -> Option<u32> {
- match self {
- Signal::ProximitySlop0 => Some(0),
- Signal::ProximitySlop1 => Some(1),
- Signal::ProximitySlop2 => Some(2),
- Signal::ProximitySlop4 => Some(4),
- Signal::ProximitySlop8 => Some(8),
- _ => None,
- }
- }
-
fn as_fastfield(&self) -> Option<FastField> {
match self {
Signal::HostCentrality => Some(FastField::HostCentrality),
@@ -666,7 +575,6 @@ impl Signal {
}
Signal::Bm25TitleIfHomepage => Some(TextField::TitleIfHomepage),
Signal::Bm25BacklinkText => Some(TextField::BacklinkText),
- Signal::Bm25Description => Some(TextField::Description),
Signal::Bm25DomainIfHomepageNoTokenizer => Some(TextField::DomainIfHomepageNoTokenizer),
_ => None,
}
@@ -743,13 +651,8 @@ struct OpticBoosts {
rules: Vec<RuleBoost>,
}
-struct ProximityScorer {
- docset: Box<dyn Scorer>,
-}
-
struct SegmentReader {
text_fields: EnumMap<TextField, TextFieldData>,
- proximity_scorers: EnumMap<Signal, ProximityScorer>,
optic_boosts: OpticBoosts,
fastfield_reader: Arc<fastfield_reader::SegmentReader>,
}
@@ -941,67 +844,6 @@ impl SignalAggregator {
optic_rule_boosts
}
- fn prepare_proximity_scorers(
- &self,
- tv_searcher: &tantivy::Searcher,
- segment_reader: &tantivy::SegmentReader,
- ) -> EnumMap<Signal, ProximityScorer> {
- if !PROXIMITY_ENABLED {
- return EnumMap::new();
- }
-
- let mut proximity_scorers = EnumMap::new();
- let proximity_fields = [TextField::Title, TextField::CleanBody];
- let schema = tv_searcher.schema();
-
- if let Some(query) = &self.query {
- if query.simple_terms.len() < 2 {
- return proximity_scorers;
- }
-
- for signal in ALL_SIGNALS {
- if let Some(slop) = signal.proximity_slop() {
- let mut queries = Vec::new();
- for field in proximity_fields {
- let tv_field = schema.get_field(field.name()).unwrap();
- let mut tokenizer = field.indexing_tokenizer();
- let mut terms = Vec::with_capacity(query.simple_terms.len());
-
- for term in &query.simple_terms {
- let mut stream = tokenizer.token_stream(term);
- while let Some(term) = stream.next() {
- let term = tantivy::Term::from_field_text(tv_field, &term.text);
- terms.push(term);
- }
- }
-
- let mut phrase_query = PhraseQuery::new(terms);
- phrase_query.set_slop(slop);
-
- queries.push((
- tantivy::query::Occur::Should,
- Box::new(phrase_query) as Box<dyn tantivy::query::Query>,
- ));
- }
-
- let query = tantivy::query::BooleanQuery::new(queries);
- let docset = query
- .weight(tantivy::query::EnableScoring::Enabled {
- searcher: tv_searcher,
- statistics_provider: tv_searcher,
- })
- .unwrap()
- .scorer(segment_reader, 1.0)
- .unwrap();
-
- proximity_scorers.insert(signal, ProximityScorer { docset });
- }
- }
- }
-
- proximity_scorers
- }
-
pub fn register_segment(
&mut self,
tv_searcher: &tantivy::Searcher,
@@ -1011,7 +853,6 @@ impl SignalAggregator {
let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());
let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;
let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, fastfield_reader);
- let proximity_scorers = self.prepare_proximity_scorers(tv_searcher, segment_reader);
self.segment_reader = Some(SegmentReader {
text_fields,
@@ -1019,7 +860,6 @@ impl SignalAggregator {
optic_boosts: OpticBoosts {
rules: optic_rule_boosts,
},
- proximity_scorers,
});
Ok(())
@@ -1139,101 +979,3 @@ pub struct SignalScore {
pub coefficient: f64,
pub value: f64,
}
-
-#[cfg(test)]
-mod tests {
- use std::collections::HashMap;
-
- use crate::{
- index::Index,
- searcher::{LocalSearcher, SearchQuery},
- };
-
- use super::*;
-
- fn signals_for_query(searcher: &LocalSearcher, q: &str) -> HashMap<Signal, SignalScore> {
- let res = searcher
- .search(&SearchQuery {
- query: q.to_string(),
- ..Default::default()
- })
- .unwrap()
- .webpages;
- res[0].ranking_signals.clone().unwrap()
- }
- #[test]
- #[ignore = "proximity signals are currently disabled"]
- fn test_proximity() {
- let mut index = Index::temporary().expect("Unable to open index");
-
- let mut webpage = Webpage::new(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {}
- </body>
- </html>
- "#,
- crate::rand_words(1_000)
- ),
- "https://www.example.com",
- )
- .unwrap();
-
- webpage
- .html
- .set_clean_text("a b c d e f g h i j k l m n o p q r s t u v w x y z".to_string());
-
- index.insert(webpage).expect("failed to insert webpage");
-
- index.commit().unwrap();
-
- let searcher = LocalSearcher::new(index);
-
- let signals = signals_for_query(&searcher, "a b");
- assert!(signals[&Signal::ProximitySlop8].value > 0.0);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop1].value);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop0].value);
-
- let signals = signals_for_query(&searcher, "a c");
- assert!(signals[&Signal::ProximitySlop8].value > 0.0);
- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop1].value);
-
- let signals = signals_for_query(&searcher, "a d");
- assert!(signals[&Signal::ProximitySlop8].value > 0.0);
- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);
-
- let signals = signals_for_query(&searcher, "a e");
- assert!(signals[&Signal::ProximitySlop8].value > 0.0);
- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);
- assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);
-
- let signals = signals_for_query(&searcher, "a g");
- assert!(signals[&Signal::ProximitySlop8].value > 0.0);
- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop4].value, 0.0);
-
- let signals = signals_for_query(&searcher, "a k");
- assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop4].value, 0.0);
- assert_eq!(signals[&Signal::ProximitySlop8].value, 0.0);
- }
-}
diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rs
index 79ad0531..1d0e310c 100644
--- a/core/src/search_prettifier/entity.rs
+++ b/core/src/search_prettifier/entity.rs
@@ -29,7 +29,7 @@ use crate::entity_index::{
pub struct DisplayedEntity {
pub title: String,
pub small_abstract: EntitySnippet,
- pub image_base64: Option<String>,
+ pub image_id: Option<String>,
pub related_entities: Vec<DisplayedEntity>,
pub info: Vec<(String, EntitySnippet)>,
pub match_score: f32,
@@ -47,7 +47,7 @@ impl From<EntityMatch> for DisplayedEntity {
Self {
title: m.entity.title,
small_abstract,
- image_base64: m.entity.image_base64,
+ image_id: m.entity.image_id,
related_entities: m
.entity
.related_entities
diff --git a/core/src/search_prettifier/stack_overflow.rs b/core/src/search_prettifier/stack_overflow.rs
index e688c081..da9e59f0 100644
--- a/core/src/search_prettifier/stack_overflow.rs
+++ b/core/src/search_prettifier/stack_overflow.rs
@@ -126,6 +126,27 @@ fn schema_item_to_stackoverflow_answer(
}
}
+fn limit_chars(passages: &[CodeOrText], limit: usize) -> Vec<CodeOrText> {
+ let mut res = Vec::new();
+ let mut taken_chars = 0;
+
+ for p in passages {
+ res.push(p.clone());
+ let s = match p {
+ CodeOrText::Code(s) => s,
+ CodeOrText::Text(s) => s,
+ };
+
+ if taken_chars + s.len() > limit {
+ break;
+ }
+
+ taken_chars += s.len();
+ }
+
+ res
+}
+
pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
match webpage
.schema_org
@@ -192,10 +213,20 @@ pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
answers.push(answer);
}
- Ok(Snippet::StackOverflowQA {
- question: StackOverflowQuestion { body: question },
- answers: answers.into_iter().take(3).collect(),
- })
+ let question = StackOverflowQuestion {
+ body: limit_chars(&question, 512),
+ };
+
+ let answers: Vec<_> = answers
+ .into_iter()
+ .map(|answer| StackOverflowAnswer {
+ body: limit_chars(&answer.body, 512),
+ ..answer
+ })
+ .take(3)
+ .collect();
+
+ Ok(Snippet::StackOverflowQA { question, answers })
}
None => Err(Error::InvalidStackoverflowSchema.into()),
}
diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rs
index 60d0118e..948d7b50 100644
--- a/core/src/searcher/api.rs
+++ b/core/src/searcher/api.rs
@@ -26,6 +26,7 @@ use url::Url;
use crate::bangs::{Bang, BangHit};
use crate::config::{ApiConfig, ApiThresholds, CollectorConfig};
+use crate::image_store::Image;
use crate::inverted_index::RetrievedWebpage;
#[cfg(not(feature = "libtorch"))]
use crate::ranking::models::cross_encoder::DummyCrossEncoder;
@@ -510,6 +511,10 @@ impl ApiSearcher {
pub async fn get_webpage(&self, url: &str) -> Result<RetrievedWebpage> {
self.distributed_searcher.get_webpage(url).await
}
+
+ pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
+ self.distributed_searcher.get_entity_image(image_id).await
+ }
}
fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 3f02dcf6..6199f07a 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -17,6 +17,7 @@
use crate::{
distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff},
entrypoint::search_server::{self, SearchService},
+ image_store::Image,
inverted_index::{self, RetrievedWebpage},
ranking::pipeline::{AsRankingWebsite, RankingWebsite},
Result,
@@ -143,6 +144,19 @@ impl RemoteSearcher {
.await
.unwrap()
}
+
+ async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
+ let conn = self.conn().await;
+
+ conn.send_with_timeout(
+ &search_server::GetEntityImage {
+ image_id: image_id.to_string(),
+ },
+ Duration::from_secs(60),
+ )
+ .await
+ .map_err(|e| e.into())
+ }
}
#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
@@ -217,6 +231,10 @@ impl Shard {
async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
self.replica().get_homepage_descriptions(urls).await
}
+
+ async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
+ self.replica().get_entity_image(image_id).await
+ }
}
#[derive(Debug)]
@@ -344,4 +362,11 @@ impl DistributedSearcher {
.flatten()
.collect()
}
+
+ pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
+ match self.shards().await.first() {
+ Some(s) => s.get_entity_image(image_id).await,
+ None => Ok(None),
+ }
+ }
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index e370c08c..e7cebb03 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -23,6 +23,7 @@ use url::Url;
use crate::config::{CollectorConfig, SnippetConfig};
use crate::entity_index::{EntityIndex, EntityMatch};
+use crate::image_store::Image;
use crate::index::Index;
use crate::inverted_index::RetrievedWebpage;
use crate::query::Query;
@@ -69,7 +70,7 @@ struct InvertedIndexResult {
impl LocalSearcher {
pub fn new(index: Index) -> Self {
let mut index = index;
- // index.optimize_for_search().unwrap();
+ index.optimize_for_search().unwrap();
LocalSearcher {
index,
@@ -409,6 +410,12 @@ impl LocalSearcher {
pub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
self.index.get_homepage(url)
}
+
+ pub fn get_entity_image(&self, image_id: &str) -> Option<Image> {
+ self.entity_index
+ .as_ref()
+ .and_then(|index| index.retrieve_image(image_id))
+ }
}
#[cfg(test)]
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 5ea033ec..6e80a0a3 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -19,7 +19,7 @@ use std::ops::Range;
use crate::config::SnippetConfig;
use crate::query::Query;
use crate::spell::sentence_ranges;
-use crate::tokenizer::{BigramTokenizer, Stemmed, Tokenizer, TrigramTokenizer};
+use crate::tokenizer::{BigramTokenizer, Normal, Stemmed, Tokenizer, TrigramTokenizer};
use crate::webpage::region::Region;
use hashbrown::{HashMap, HashSet};
use utoipa::ToSchema;
@@ -81,6 +81,7 @@ impl SnippetBuilder {
fn highlight(&mut self, terms: &HashSet<String>, lang: whatlang::Lang) {
for mut tokenizer in [
Tokenizer::Stemmed(Stemmed::with_forced_language(lang)),
+ Tokenizer::Normal(Normal::default()),
Tokenizer::Bigram(BigramTokenizer::default()),
Tokenizer::Trigram(TrigramTokenizer::default()),
] {
@@ -94,9 +95,10 @@ impl SnippetBuilder {
}
// remove overlapping ranges
- self.highlights.sort_by(|a, b| a.start.cmp(&b.start));
self.highlights
- .dedup_by(|a, b| a.start == b.start && a.end > b.end);
+ .sort_by(|a, b| a.start.cmp(&b.start).then(a.end.cmp(&b.end)));
+ self.highlights
+ .dedup_by(|a, b| a.start == b.start && a.end >= b.end);
}
fn build(self) -> TextSnippet {
@@ -136,9 +138,8 @@ fn snippet_string_builder(
terms: &[String],
lang: whatlang::Lang,
config: SnippetConfig,
+ mut tokenizer: Tokenizer,
) -> SnippetBuilder {
- let mut tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));
-
let terms: HashSet<String> = terms
.iter()
.flat_map(|term| {
@@ -276,7 +277,20 @@ fn snippet_string(
lang: whatlang::Lang,
config: SnippetConfig,
) -> TextSnippet {
- snippet_string_builder(text, terms, lang, config).build()
+ let tokenizer = Tokenizer::Normal(Normal::default());
+ let snip = snippet_string_builder(text, terms, lang, config.clone(), tokenizer).build();
+
+ if !snip.fragments.is_empty()
+ && snip
+ .fragments
+ .iter()
+ .any(|f| f.kind == TextSnippetFragmentKind::Highlighted)
+ {
+ return snip;
+ }
+
+ let tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));
+ snippet_string_builder(text, terms, lang, config, tokenizer).build()
}
pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfig) -> TextSnippet {
@@ -523,18 +537,17 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn compounded_terms() {
- let mut snip = snippet_string_builder(
+ let snip = snippet_string_builder(
"this is a test",
&["thisis".to_string()],
whatlang::Lang::Eng,
SnippetConfig::default(),
+ Tokenizer::Normal(Normal::default()),
);
let mut terms = HashSet::new();
terms.insert("thisis".to_string());
- snip.highlight(&terms, whatlang::Lang::Eng);
-
assert_eq!(
highlight(Snippet::Normal {
date: None,
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index 458c11fb..df6c02be 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -98,14 +98,6 @@ const sse = <T>(
};
export const api = {
- alice: (
- query: {
- message: string;
- optic: string;
- prevState: EncodedSavedState;
- },
- options?: ApiOptions,
- ) => sse<ExecutionState>('GET', `/beta/api/alice?${new URLSearchParams(query)}`, options),
autosuggest: (
params: {
q: string;
@@ -119,8 +111,6 @@ export const api = {
),
exploreExport: (body: ExploreExportOpticParams, options?: ApiOptions) =>
requestPlain('POST', `/beta/api/explore/export`, body, options),
- factCheck: (body: FactCheckParams, options?: ApiOptions) =>
- requestJson<FactCheckResponse>('POST', `/beta/api/fact_check`, body, options),
search: (body: ApiSearchQuery, options?: ApiOptions) =>
requestJson<ApiSearchResult>('POST', `/beta/api/search`, body, options),
sitesExport: (body: SitesExportOpticParams, options?: ApiOptions) =>
@@ -246,7 +236,7 @@ export type DisplayedAnswer = {
url: string;
};
export type DisplayedEntity = {
- imageBase64?: string;
+ imageId?: string;
info: string & EntitySnippet[][];
matchScore: number;
relatedEntities: DisplayedEntity[];
@@ -274,8 +264,6 @@ export type DisplayedWebpage = {
title: string;
url: string;
};
-export type EncodedEncryptedState = string;
-export type EncodedSavedState = string;
export type EntitySnippet = {
fragments: EntitySnippetFragment[];
};
@@ -290,35 +278,10 @@ export type EntitySnippetFragment =
text: string;
};
export type Example = string;
-export type ExecutionState =
- | {
- query: string;
- type: 'beginSearch';
- }
- | {
- query: string;
- result: SimplifiedWebsite[];
- type: 'searchResult';
- }
- | {
- text: string;
- type: 'speaking';
- }
- | {
- state: EncodedEncryptedState;
- type: 'done';
- };
export type ExploreExportOpticParams = {
chosenSites: string[];
similarSites: string[];
};
-export type FactCheckParams = {
- claim: string;
- evidence: string;
-};
-export type FactCheckResponse = {
- score: number;
-};
export type FullEdge = {
from: Node;
label: string;
@@ -367,12 +330,6 @@ export type SimilarSitesParams = {
sites: string[];
topN: number;
};
-export type SimplifiedWebsite = {
- site: string;
- text: string;
- title: string;
- url: string;
-};
export type SiteRankings = {
blocked: string[];
disliked: string[];
diff --git a/frontend/src/lib/components/Code.svelte b/frontend/src/lib/components/Code.svelte
index 27ac0fd3..b6da7948 100644
--- a/frontend/src/lib/components/Code.svelte
+++ b/frontend/src/lib/components/Code.svelte
@@ -1,20 +1,13 @@
<script lang="ts">
- import { page } from '$app/stores';
+ import {HighlightAuto} from "svelte-highlight";
export let code: string;
-
- const component =
- $page.data.globals?.highlightjs?.HighlightAuto ||
- import('svelte-highlight').then(({ HighlightAuto }) => HighlightAuto);
+ export let transparentBackground = false;
</script>
-<div class="code rounded-lg bg-base-200 text-sm text-neutral-focus">
+<div class="code rounded-lg {transparentBackground ? '' : 'bg-base-200'} text-sm text-neutral-focus">
<div class="overflow-auto px-3 py-2">
- {#await component}
- <pre><code>{code}</code></pre>
- {:then comp}
- <svelte:component this={comp} {code} />
- {/await}
+ <HighlightAuto code={code}/>
</div>
</div>
diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelte
index 8f5504c3..e8a8b2c9 100644
--- a/frontend/src/routes/search/+page.svelte
+++ b/frontend/src/routes/search/+page.svelte
@@ -153,7 +153,7 @@
<div class="mx-auto flex w-full justify-between">
<div class="flex h-full flex-col justify-center space-x-2 text-sm text-neutral">
<p class="h-fit">
- {#if results.numHits }
+ {#if results.numHits != null }
Found {results.numHits.toLocaleString()} results in {(
(results.searchDurationMs ?? 0) / 1000
).toFixed(2)}
diff --git a/frontend/src/routes/search/Entity.svelte b/frontend/src/routes/search/Entity.svelte
index 1b9d7a2a..59e13040 100644
--- a/frontend/src/routes/search/Entity.svelte
+++ b/frontend/src/routes/search/Entity.svelte
@@ -1,5 +1,5 @@
<script lang="ts">
- import type { DisplayedEntity } from '$lib/api';
+ import { getApiBase, type DisplayedEntity } from '$lib/api';
import EntitySnippet from '$lib/components/EntitySnippet.svelte';
export let entity: DisplayedEntity;
@@ -7,13 +7,13 @@
<div class="flex w-full justify-center">
<div class="flex w-full flex-col items-center">
- {#if entity.imageBase64}
+ {#if entity.imageId}
<div class="w-lg mb-5">
<a href="https://en.wikipedia.org/wiki/{encodeURI(entity.title)}">
<img
alt="Image of {entity.title}"
class="h-full w-full rounded-full"
- src="data:image/png;base64, {entity.imageBase64}"
+ src="{getApiBase()}/beta/api/entity_image?imageId={entity.imageId}"
/>
</a>
</div>
@@ -53,13 +53,13 @@
<div class="flex overflow-scroll">
{#each entity.relatedEntities as related (related.title)}
<div class="flex flex-col items-center p-4">
- {#if related.imageBase64}
+ {#if related.imageId != null}
<div class="mb-3 h-20 w-20">
<a href="/search?q={encodeURIComponent(related.title)}">
<img
alt="Image of {related.title}"
class="h-full w-full rounded-full object-cover"
- src="data:image/png;base64, {related.imageBase64}"
+ src="{getApiBase()}/beta/api/entity_image?imageId={related.imageId}"
/>
</a>
</div>
diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelte
index f6e2f028..9b27fd4b 100644
--- a/frontend/src/routes/search/Result.svelte
+++ b/frontend/src/routes/search/Result.svelte
@@ -7,6 +7,7 @@
import { derived } from 'svelte/store';
import { improvements } from '$lib/improvements';
import TextSnippet from '$lib/components/TextSnippet.svelte';
+ import StackOverflowSnippet from './StackOverflowSnippet.svelte';
export let webpage: DisplayedWebpage;
export let resultIndex: number;
@@ -54,13 +55,23 @@
<div class="snippet">
<div class="line-clamp-3">
<div class="inline">
- <span id="snippet-text" class="snippet-text [&:nth-child(2)]:before:content-['—']">
- {webpage.snippet.date || ''}
- <TextSnippet snippet={webpage.snippet.text} />
+ <span id="snippet-text" class="snippet-text">
+ {#if webpage.snippet.date}
+ <span class="text-neutral">
+ {webpage.snippet.date} -
+ </span>
+ {/if}
+ <span>
+ <TextSnippet snippet={webpage.snippet.text} />
+ </span>
</span>
</div>
</div>
</div>
+ {:else if webpage.snippet.type == 'stackOverflowQA'}
+ <div class="snippet">
+ <StackOverflowSnippet question={webpage.snippet.question} answers={webpage.snippet.answers} />
+ </div>
{/if}
</div>
</div>
diff --git a/frontend/src/routes/search/StackOverflowSnippet.svelte b/frontend/src/routes/search/StackOverflowSnippet.svelte
new file mode 100644
index 00000000..11132d40
--- /dev/null
+++ b/frontend/src/routes/search/StackOverflowSnippet.svelte
@@ -0,0 +1,52 @@
+
+<script lang="ts">
+ import type { StackOverflowQuestion, StackOverflowAnswer } from "$lib/api";
+ import Code from '$lib/components/Code.svelte';
+ import HandThumbUp from '~icons/heroicons/hand-thumb-up';
+ import Check from '~icons/heroicons/check';
+
+ export let question: StackOverflowQuestion;
+ export let answers: StackOverflowAnswer[];
+</script>
+<div class="line-clamp-2">
+{#each question.body as passage}
+ {#if passage.type == "text"}
+ {passage.value}
+ {/if}
+{/each}
+</div>
+<div class="flex space-x-4 pt-2">
+ {#each answers.slice(0, 3) as answer}
+ <div class="w-1/3 overflow-hidden">
+ <a class="block h-56 hover:bg-base-200/80 p-2 border overflow-hidden rounded-lg" href={answer.url}>
+ <div class="flex w-full items-center justify-between space-x-1 mb-1 text-neutral text-xs">
+ <div class="flex">
+ {answer.date}
+ </div>
+ <div class="flex space-x-1">
+ <span class="h-fit">
+ {answer.upvotes}
+ </span>
+ <div class="h-fit">
+ <HandThumbUp class="w-4" />
+ </div>
+ {#if answer.accepted}
+ <div class="h-fit text-green-600">
+ <Check class="w-4" />
+ </div>
+ {/if}
+ </div>
+ </div>
+ <div>
+ {#each answer.body as passage}
+ {#if passage.type == "text"}
+ {passage.value}
+ {:else if passage.type == "code"}
+ <Code code={passage.value} transparentBackground={true} />
+ {/if}
+ {/each}
+ </div>
+ </a>
+ </div>
+ {/each}
+</div>
\ No newline at end of file
diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js
index 91ee6927..8b1b0a03 100644
--- a/frontend/svelte.config.js
+++ b/frontend/svelte.config.js
@@ -32,7 +32,7 @@ const config = {
// NOTE: Disabled in order to fetch optic sources client-side
// 'connect-src': ["'self'", 'http://localhost:3000/'],
'connect-src': ['*'],
- 'img-src': ["'self'", 'data:'],
+ 'img-src': ["'self'", 'data:', 'trystract.com', '0.0.0.0:3000'],
'style-src': ["'self'", "'unsafe-inline'"],
},
},