web search done right
add truncated body in api
Mikkel Denker4951b72417dc
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rsindex 28248c31..c1ab28a9 100644--- a/crates/core/src/api/docs.rs+++ b/crates/core/src/api/docs.rs@@ -46,6 +46,7 @@ use utoipa_swagger_ui::SwaggerUi;search::WidgetQuery,search::SidebarQuery,search::SpellcheckQuery,+ search::ReturnBody,crate::searcher::WebsitesResult,crate::search_prettifier::HighlightedSpellCorrection,crate::search_prettifier::DisplayedWebpage,diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rsindex 836e6446..94551bea 100644--- a/crates/core/src/api/search.rs+++ b/crates/core/src/api/search.rs@@ -33,6 +33,22 @@ use super::State;use axum::{extract, response::IntoResponse};+#[derive(+ Clone,+ Copy,+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ ToSchema,+)]+#[serde(tag = "type", content = "value", rename_all = "camelCase")]+pub enum ReturnBody {+ All,+ Truncated(usize),+}+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,)]@@ -55,6 +71,8 @@ pub struct ApiSearchQuery {#[serde(default = "defaults::SearchQuery::count_results")]pub count_results: bool,++ pub return_body: Option<ReturnBody>,}impl TryFrom<ApiSearchQuery> for SearchQuery {@@ -79,6 +97,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {return_ranking_signals: api.return_ranking_signals,safe_search: api.safe_search.unwrap_or(default.safe_search),count_results: api.count_results,+ return_body: api.return_body,})}}diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rsindex e7ac74fe..5cba0b51 100644--- a/crates/core/src/distributed/sonic/service.rs+++ b/crates/core/src/distributed/sonic/service.rs@@ -139,7 +139,7 @@ macro_rules! sonic_service {#[derive(Debug, Clone, ::bincode::Decode)]pub enum Request {- $($req($req),)*+ $($req(Box<$req>),)*}#[derive(Debug, Clone, ::bincode::Encode)]pub enum RequestRef<'a> {@@ -147,7 +147,7 @@ macro_rules! sonic_service {}#[derive(::bincode::Encode, ::bincode::Decode)]pub enum Response {- $($req(<$req as sonic::service::Message<$service>>::Response),)*+ $($req(Box<<$req as sonic::service::Message<$service>>::Response>),)*}$(impl sonic::service::Wrapper<$service> for $req {@@ -157,7 +157,7 @@ macro_rules! sonic_service {fn unwrap_response(res: <$service as sonic::service::Service>::Response) -> Option<Self::Response> {#[allow(irrefutable_let_patterns)]if let Response::$req(value) = res {- Some(value)+ Some(*value)} else {None}@@ -177,7 +177,7 @@ macro_rules! sonic_service {async move {match req {$(- Request::$req(value) => Response::$req(sonic::service::Message::handle(value, server).await),+ Request::$req(value) => Response::$req(Box::new(sonic::service::Message::handle(*value, server).await)),)*}}diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rsindex d9a18d85..1488431f 100644--- a/crates/core/src/search_prettifier/mod.rs+++ b/crates/core/src/search_prettifier/mod.rs@@ -25,9 +25,11 @@ use url::Url;use utoipa::ToSchema;use crate::{+ api::search::ReturnBody,highlighted::HighlightedFragment,inverted_index::RetrievedWebpage,ranking::{SignalEnumDiscriminants, SignalScore},+ searcher::SearchQuery,snippet::TextSnippet,web_spell::{self, CorrectionTerm},webpage::url_ext::UrlExt,@@ -186,6 +188,7 @@ pub struct DisplayedWebpage {pub domain: String,pub pretty_url: String,pub snippet: Snippet,+ pub body: Option<String>,pub rich_snippet: Option<RichSnippet>,pub ranking_signals: Option<HashMap<SignalEnumDiscriminants, SignalScore>>,pub score: Option<f64>,@@ -205,8 +208,8 @@ pub struct DisplayedAnswer {pub answer: String,}-impl From<RetrievedWebpage> for DisplayedWebpage {- fn from(webpage: RetrievedWebpage) -> Self {+impl DisplayedWebpage {+ pub fn new(webpage: RetrievedWebpage, query: &SearchQuery) -> Self {let snippet = generate_snippet(&webpage);let rich_snippet = generate_rich_snippet(&webpage);@@ -214,6 +217,11 @@ impl From<RetrievedWebpage> for DisplayedWebpage {let domain = url.root_domain().unwrap_or_default().to_string();let pretty_url = prettify_url(&url);+ let body = query.return_body.map(|r| match r {+ ReturnBody::All => webpage.body,+ ReturnBody::Truncated(n) => webpage.body.chars().take(n).collect::<String>(),+ });+Self {title: webpage.title,site: url.normalized_host().unwrap_or_default().to_string(),@@ -221,6 +229,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {pretty_url,domain,snippet,+ body,ranking_signals: None,score: None,likely_has_ads: webpage.likely_has_ads,diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex e6e5f9bf..22fa6506 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -440,7 +440,7 @@ wherelet mut retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().map(|webpage| webpage.into_retrieved_webpage())- .map(DisplayedWebpage::from)+ .map(|webpage| DisplayedWebpage::new(webpage, query)).collect();if retrieved_webpages.len() != top_websites.len() {diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex 57eea597..44affdea 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -186,8 +186,7 @@ wherede_rank_similar: bool,computer: SignalComputer,) -> Result<Ranker> {- let query_centrality_coeff =- computer.coefficient(&ranking::signal::QueryCentrality.into());+ let query_centrality_coeff = computer.coefficient(&ranking::signal::QueryCentrality.into());let mut ranker = Ranker::new(computer,@@ -404,7 +403,7 @@ wherelet mut webpages: Vec<_> = retrieved_sites.into_iter()- .map(DisplayedWebpage::from)+ .map(|webpage| DisplayedWebpage::new(webpage, query)).collect();for (webpage, ranking) in webpages.iter_mut().zip(top_websites) {diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex 60f34470..d985ff27 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -26,8 +26,9 @@ use optics::{HostRankings, Optic};use utoipa::ToSchema;use crate::{- bangs::BangHit, config::defaults, ranking::pipeline::RecallRankingWebpage,- search_prettifier::DisplayedWebpage, webpage::region::Region,+ api::search::ReturnBody, bangs::BangHit, config::defaults,+ ranking::pipeline::RecallRankingWebpage, search_prettifier::DisplayedWebpage,+ webpage::region::Region,};pub const NUM_RESULTS_PER_PAGE: usize = 20;@@ -60,6 +61,7 @@ pub struct SearchQuery {pub return_ranking_signals: bool,pub safe_search: bool,pub count_results: bool,+ pub return_body: Option<ReturnBody>,}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]@@ -84,6 +86,7 @@ impl Default for SearchQuery {return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),safe_search: defaults::SearchQuery::safe_search(),count_results: defaults::SearchQuery::count_results(),+ return_body: None,}}}diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex c7990e56..6cb515ca 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -41,8 +41,6 @@ struct SerializedEdge {label: Vec<u8>,}-pub const MAX_BATCH_SIZE: usize = 100_000;-pub struct EdgeStoreWriter {reversed: bool,db: speedy_kv::Db<Vec<u8>, Vec<u8>>,@@ -89,14 +87,6 @@ impl EdgeStoreWriter {};self.db.insert_raw(key_bytes, value_bytes);-- if self.db.uncommitted_inserts() > MAX_BATCH_SIZE {- self.db.commit().unwrap();- }- }-- if self.db.uncommitted_inserts() > MAX_BATCH_SIZE {- self.db.commit().unwrap();}}diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rsindex f80a87e8..9ac066ab 100644--- a/crates/core/src/webgraph/writer.rs+++ b/crates/core/src/webgraph/writer.rs@@ -19,8 +19,8 @@ use std::{fs, path::Path};use crate::executor::Executor;use super::{- id_node_db::Id2NodeDb, segment::SegmentWriter, store, Compression, FullNodeID, InnerEdge, Meta,- Node, NodeID, Webgraph, MAX_LABEL_LENGTH,+ id_node_db::Id2NodeDb, segment::SegmentWriter, Compression, FullNodeID, InnerEdge, Meta, Node,+ NodeID, Webgraph, MAX_LABEL_LENGTH,};pub struct WebgraphWriter {@@ -60,7 +60,7 @@ impl WebgraphWriter {path: path.as_ref().as_os_str().to_str().unwrap().to_string(),segment,id2node: Id2NodeDb::open(path.as_ref().join("id2node")),- insert_batch: Vec::with_capacity(store::MAX_BATCH_SIZE),+ insert_batch: Vec::new(),executor,meta,compression,@@ -96,10 +96,6 @@ impl WebgraphWriter {};self.insert_batch.push(edge);-- if self.insert_batch.len() >= store::MAX_BATCH_SIZE {- self.commit();- }}pub fn commit(&mut self) {diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex 604e7d1e..1f69ed37 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -196,6 +196,7 @@ export type ApiSearchQuery = {optic?: string;page?: number;query: string;+ returnBody?: ReturnBody;returnRankingSignals?: boolean;safeSearch?: boolean;selectedRegion?: Region;@@ -262,6 +263,7 @@ export type DisplayedSidebar =};};export type DisplayedWebpage = {+ body?: string;domain: string;likelyHasAds: boolean;likelyHasPaywall: boolean;@@ -341,6 +343,14 @@ export type PartOfSpeechMeaning = {};export type Region = 'All' | 'Denmark' | 'France' | 'Germany' | 'Spain' | 'US';export const REGIONS = ['All', 'Denmark', 'France', 'Germany', 'Spain', 'US'] satisfies Region[];+export type ReturnBody =+ | {+ type: 'all';+ }+ | {+ type: 'truncated';+ value: number;+ };export type RichSnippet = {answers: StackOverflowAnswer[];question: StackOverflowQuestion;