Skip to content

web search done right

add truncated body in api

Mikkel Denker4951b72417dc
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs
index 28248c31..c1ab28a9 100644
--- a/crates/core/src/api/docs.rs
+++ b/crates/core/src/api/docs.rs
@@ -46,6 +46,7 @@ use utoipa_swagger_ui::SwaggerUi;
search::WidgetQuery,
search::SidebarQuery,
search::SpellcheckQuery,
+ search::ReturnBody,
crate::searcher::WebsitesResult,
crate::search_prettifier::HighlightedSpellCorrection,
crate::search_prettifier::DisplayedWebpage,
diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rs
index 836e6446..94551bea 100644
--- a/crates/core/src/api/search.rs
+++ b/crates/core/src/api/search.rs
@@ -33,6 +33,22 @@ use super::State;
use axum::{extract, response::IntoResponse};
+#[derive(
+ Clone,
+ Copy,
+ Debug,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ ToSchema,
+)]
+#[serde(tag = "type", content = "value", rename_all = "camelCase")]
+pub enum ReturnBody {
+ All,
+ Truncated(usize),
+}
+
#[derive(
Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,
)]
@@ -55,6 +71,8 @@ pub struct ApiSearchQuery {
#[serde(default = "defaults::SearchQuery::count_results")]
pub count_results: bool,
+
+ pub return_body: Option<ReturnBody>,
}
impl TryFrom<ApiSearchQuery> for SearchQuery {
@@ -79,6 +97,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {
return_ranking_signals: api.return_ranking_signals,
safe_search: api.safe_search.unwrap_or(default.safe_search),
count_results: api.count_results,
+ return_body: api.return_body,
})
}
}
diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rs
index e7ac74fe..5cba0b51 100644
--- a/crates/core/src/distributed/sonic/service.rs
+++ b/crates/core/src/distributed/sonic/service.rs
@@ -139,7 +139,7 @@ macro_rules! sonic_service {
#[derive(Debug, Clone, ::bincode::Decode)]
pub enum Request {
- $($req($req),)*
+ $($req(Box<$req>),)*
}
#[derive(Debug, Clone, ::bincode::Encode)]
pub enum RequestRef<'a> {
@@ -147,7 +147,7 @@ macro_rules! sonic_service {
}
#[derive(::bincode::Encode, ::bincode::Decode)]
pub enum Response {
- $($req(<$req as sonic::service::Message<$service>>::Response),)*
+ $($req(Box<<$req as sonic::service::Message<$service>>::Response>),)*
}
$(
impl sonic::service::Wrapper<$service> for $req {
@@ -157,7 +157,7 @@ macro_rules! sonic_service {
fn unwrap_response(res: <$service as sonic::service::Service>::Response) -> Option<Self::Response> {
#[allow(irrefutable_let_patterns)]
if let Response::$req(value) = res {
- Some(value)
+ Some(*value)
} else {
None
}
@@ -177,7 +177,7 @@ macro_rules! sonic_service {
async move {
match req {
$(
- Request::$req(value) => Response::$req(sonic::service::Message::handle(value, server).await),
+ Request::$req(value) => Response::$req(Box::new(sonic::service::Message::handle(*value, server).await)),
)*
}
}
diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rs
index d9a18d85..1488431f 100644
--- a/crates/core/src/search_prettifier/mod.rs
+++ b/crates/core/src/search_prettifier/mod.rs
@@ -25,9 +25,11 @@ use url::Url;
use utoipa::ToSchema;
use crate::{
+ api::search::ReturnBody,
highlighted::HighlightedFragment,
inverted_index::RetrievedWebpage,
ranking::{SignalEnumDiscriminants, SignalScore},
+ searcher::SearchQuery,
snippet::TextSnippet,
web_spell::{self, CorrectionTerm},
webpage::url_ext::UrlExt,
@@ -186,6 +188,7 @@ pub struct DisplayedWebpage {
pub domain: String,
pub pretty_url: String,
pub snippet: Snippet,
+ pub body: Option<String>,
pub rich_snippet: Option<RichSnippet>,
pub ranking_signals: Option<HashMap<SignalEnumDiscriminants, SignalScore>>,
pub score: Option<f64>,
@@ -205,8 +208,8 @@ pub struct DisplayedAnswer {
pub answer: String,
}
-impl From<RetrievedWebpage> for DisplayedWebpage {
- fn from(webpage: RetrievedWebpage) -> Self {
+impl DisplayedWebpage {
+ pub fn new(webpage: RetrievedWebpage, query: &SearchQuery) -> Self {
let snippet = generate_snippet(&webpage);
let rich_snippet = generate_rich_snippet(&webpage);
@@ -214,6 +217,11 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
let domain = url.root_domain().unwrap_or_default().to_string();
let pretty_url = prettify_url(&url);
+ let body = query.return_body.map(|r| match r {
+ ReturnBody::All => webpage.body,
+ ReturnBody::Truncated(n) => webpage.body.chars().take(n).collect::<String>(),
+ });
+
Self {
title: webpage.title,
site: url.normalized_host().unwrap_or_default().to_string(),
@@ -221,6 +229,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
pretty_url,
domain,
snippet,
+ body,
ranking_signals: None,
score: None,
likely_has_ads: webpage.likely_has_ads,
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index e6e5f9bf..22fa6506 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -440,7 +440,7 @@ where
let mut retrieved_webpages: Vec<_> = retrieved_webpages
.into_iter()
.map(|webpage| webpage.into_retrieved_webpage())
- .map(DisplayedWebpage::from)
+ .map(|webpage| DisplayedWebpage::new(webpage, query))
.collect();
if retrieved_webpages.len() != top_websites.len() {
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index 57eea597..44affdea 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -186,8 +186,7 @@ where
de_rank_similar: bool,
computer: SignalComputer,
) -> Result<Ranker> {
- let query_centrality_coeff =
- computer.coefficient(&ranking::signal::QueryCentrality.into());
+ let query_centrality_coeff = computer.coefficient(&ranking::signal::QueryCentrality.into());
let mut ranker = Ranker::new(
computer,
@@ -404,7 +403,7 @@ where
let mut webpages: Vec<_> = retrieved_sites
.into_iter()
- .map(DisplayedWebpage::from)
+ .map(|webpage| DisplayedWebpage::new(webpage, query))
.collect();
for (webpage, ranking) in webpages.iter_mut().zip(top_websites) {
diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rs
index 60f34470..d985ff27 100644
--- a/crates/core/src/searcher/mod.rs
+++ b/crates/core/src/searcher/mod.rs
@@ -26,8 +26,9 @@ use optics::{HostRankings, Optic};
use utoipa::ToSchema;
use crate::{
- bangs::BangHit, config::defaults, ranking::pipeline::RecallRankingWebpage,
- search_prettifier::DisplayedWebpage, webpage::region::Region,
+ api::search::ReturnBody, bangs::BangHit, config::defaults,
+ ranking::pipeline::RecallRankingWebpage, search_prettifier::DisplayedWebpage,
+ webpage::region::Region,
};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
@@ -60,6 +61,7 @@ pub struct SearchQuery {
pub return_ranking_signals: bool,
pub safe_search: bool,
pub count_results: bool,
+ pub return_body: Option<ReturnBody>,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
@@ -84,6 +86,7 @@ impl Default for SearchQuery {
return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),
safe_search: defaults::SearchQuery::safe_search(),
count_results: defaults::SearchQuery::count_results(),
+ return_body: None,
}
}
}
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index c7990e56..6cb515ca 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -41,8 +41,6 @@ struct SerializedEdge {
label: Vec<u8>,
}
-pub const MAX_BATCH_SIZE: usize = 100_000;
-
pub struct EdgeStoreWriter {
reversed: bool,
db: speedy_kv::Db<Vec<u8>, Vec<u8>>,
@@ -89,14 +87,6 @@ impl EdgeStoreWriter {
};
self.db.insert_raw(key_bytes, value_bytes);
-
- if self.db.uncommitted_inserts() > MAX_BATCH_SIZE {
- self.db.commit().unwrap();
- }
- }
-
- if self.db.uncommitted_inserts() > MAX_BATCH_SIZE {
- self.db.commit().unwrap();
}
}
diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rs
index f80a87e8..9ac066ab 100644
--- a/crates/core/src/webgraph/writer.rs
+++ b/crates/core/src/webgraph/writer.rs
@@ -19,8 +19,8 @@ use std::{fs, path::Path};
use crate::executor::Executor;
use super::{
- id_node_db::Id2NodeDb, segment::SegmentWriter, store, Compression, FullNodeID, InnerEdge, Meta,
- Node, NodeID, Webgraph, MAX_LABEL_LENGTH,
+ id_node_db::Id2NodeDb, segment::SegmentWriter, Compression, FullNodeID, InnerEdge, Meta, Node,
+ NodeID, Webgraph, MAX_LABEL_LENGTH,
};
pub struct WebgraphWriter {
@@ -60,7 +60,7 @@ impl WebgraphWriter {
path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
segment,
id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
- insert_batch: Vec::with_capacity(store::MAX_BATCH_SIZE),
+ insert_batch: Vec::new(),
executor,
meta,
compression,
@@ -96,10 +96,6 @@ impl WebgraphWriter {
};
self.insert_batch.push(edge);
-
- if self.insert_batch.len() >= store::MAX_BATCH_SIZE {
- self.commit();
- }
}
pub fn commit(&mut self) {
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index 604e7d1e..1f69ed37 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -196,6 +196,7 @@ export type ApiSearchQuery = {
optic?: string;
page?: number;
query: string;
+ returnBody?: ReturnBody;
returnRankingSignals?: boolean;
safeSearch?: boolean;
selectedRegion?: Region;
@@ -262,6 +263,7 @@ export type DisplayedSidebar =
};
};
export type DisplayedWebpage = {
+ body?: string;
domain: string;
likelyHasAds: boolean;
likelyHasPaywall: boolean;
@@ -341,6 +343,14 @@ export type PartOfSpeechMeaning = {
};
export type Region = 'All' | 'Denmark' | 'France' | 'Germany' | 'Spain' | 'US';
export const REGIONS = ['All', 'Denmark', 'France', 'Germany', 'Spain', 'US'] satisfies Region[];
+export type ReturnBody =
+ | {
+ type: 'all';
+ }
+ | {
+ type: 'truncated';
+ value: number;
+ };
export type RichSnippet = {
answers: StackOverflowAnswer[];
question: StackOverflowQuestion;