web search done right
Add `#[serde(tag = "type", content = "value")]` to OpenAPI exposed types (#76)
Oliver Bøving50304467c4a7
diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rsindex 02e006bb..21064ee0 100644--- a/core/src/api/webgraph.rs+++ b/core/src/api/webgraph.rs@@ -49,6 +49,7 @@ impl RemoteWebgraph {}#[derive(serde::Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct SimilarSitesParams {pub sites: Vec<String>,pub top_n: usize,diff --git a/core/src/bangs.rs b/core/src/bangs.rsindex 53afb33a..a8a98887 100644--- a/core/src/bangs.rs+++ b/core/src/bangs.rs@@ -31,6 +31,7 @@ use crate::query::parser::Term;pub const BANG_PREFIX: char = '!';#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct Bang {#[serde(rename = "c")]pub(crate) category: Option<String>,@@ -80,6 +81,7 @@ impl From<Url> for UrlWrapper {}#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct BangHit {pub bang: Bang,pub redirect_to: UrlWrapper,diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsindex 7979c40b..97cff0a7 100644--- a/core/src/entrypoint/webgraph_server.rs+++ b/core/src/entrypoint/webgraph_server.rs@@ -39,6 +39,7 @@ use crate::webgraph::WebgraphBuilder;use crate::Result;#[derive(serde::Serialize, serde::Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct ScoredSite {pub site: String,pub score: f64,diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex fb1f034f..a752bb8e 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -1108,6 +1108,7 @@ pub struct ComputedSignal {}#[derive(Debug, Clone, Copy, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct SignalScore {pub coefficient: f64,pub value: f64,diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rsindex 424a0f73..5e3fe643 100644--- a/core/src/search_prettifier/entity.rs+++ b/core/src/search_prettifier/entity.rs@@ -26,6 +26,7 @@ use crate::{};#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct DisplayedEntity {pub title: String,pub small_abstract: String,diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rsindex f1785f87..4160378e 100644--- a/core/src/search_prettifier/mod.rs+++ b/core/src/search_prettifier/mod.rs@@ -38,6 +38,7 @@ pub use entity::DisplayedEntity;pub use self::stack_overflow::{stackoverflow_snippet, StackOverflowAnswer, StackOverflowQuestion};#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[serde(tag = "type", rename_all = "camelCase")]pub enum Snippet {Normal {date: Option<String>,@@ -50,6 +51,7 @@ pub enum Snippet {}#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct HighlightedSpellCorrection {pub raw: String,pub highlighted: String,@@ -159,6 +161,7 @@ fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {}#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct DisplayedWebpage {pub title: String,pub url: String,@@ -171,6 +174,7 @@ pub struct DisplayedWebpage {}#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct DisplayedAnswer {pub title: String,pub url: String,@@ -204,6 +208,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {}#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[serde(tag = "type", content = "value", rename_all = "camelCase")]pub enum Sidebar {Entity(DisplayedEntity),StackOverflow {diff --git a/core/src/search_prettifier/stack_overflow.rs b/core/src/search_prettifier/stack_overflow.rsindex 629bb483..0b7b2319 100644--- a/core/src/search_prettifier/stack_overflow.rs+++ b/core/src/search_prettifier/stack_overflow.rs@@ -28,6 +28,7 @@ use super::{Sidebar, Snippet};use crate::Result;#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct StackOverflowAnswer {pub body: Vec<CodeOrText>,pub date: String,@@ -37,11 +38,13 @@ pub struct StackOverflowAnswer {}#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct StackOverflowQuestion {pub body: Vec<CodeOrText>,}#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[serde(tag = "type", content = "value", rename_all = "camelCase")]pub enum CodeOrText {Code(String),Text(String),diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 9a99f296..83f21086 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -45,6 +45,7 @@ pub enum SearchResult {}#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct WebsitesResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,diff --git a/core/src/widgets/calculator.rs b/core/src/widgets/calculator.rsindex dfcfa4c0..5aea7b1e 100644--- a/core/src/widgets/calculator.rs+++ b/core/src/widgets/calculator.rs@@ -108,6 +108,7 @@ fn parse(expr: &str) -> Result<Box<Expr>> {}#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct Calculation {pub input: String,pub expr: Box<Expr>,diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rsindex b89eb8aa..a814cd21 100644--- a/core/src/widgets/mod.rs+++ b/core/src/widgets/mod.rs@@ -31,6 +31,7 @@ pub enum Error {}#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[serde(tag = "type", content = "value", rename_all = "camelCase")]pub enum Widget {Calculator(Calculation),}diff --git a/frontend/public/js/explore.js b/frontend/public/js/explore.jsindex 8e4cb22a..ca37d73e 100644--- a/frontend/public/js/explore.js+++ b/frontend/public/js/explore.js@@ -125,7 +125,7 @@ function updateSimilarSites() {},body: JSON.stringify({sites: sites,- top_n: parseInt(limit.value)+ topN: parseInt(limit.value)})}).then(response => response.json())diff --git a/ltr/generate_data.py b/ltr/generate_data.pyindex 794faa50..2ff0da53 100644--- a/ltr/generate_data.py+++ b/ltr/generate_data.py@@ -47,8 +47,8 @@ def search(q: str, top_n, optic=None):json={"query": q,"page": 0,- "num_results": top_n,- "return_ranking_signals": True,+ "numResults": top_n,+ "returnRankingSignals": True,"optic": optic,},)@@ -66,7 +66,7 @@ def search(q: str, top_n, optic=None):"domain": r["domain"],"title": r["title"],"url": r["url"],- "ranking_signals": r["ranking_signals"],+ "rankingSignals": r["rankingSignals"],"snippet": snip,"body": r["body"],}@@ -78,7 +78,7 @@ def search(q: str, top_n, optic=None):def score_prompt(query: str, res):r = {k: v for (k, v) in res.items() if k in ["domain", "title", "snippet"]}# r['domain_score'] = '{:f}'.format(- # res['ranking_signals']['host_centrality'])+ # res['rankingSignals']['host_centrality'])encoded = json.dumps(r)inst = "You are a search engine evaluator. Evaluate the following result based on the query. A good search result is relevant to the query and comes from a trustworthy domain. The score for the search result should be between 0.0 and 1.0."@@ -257,7 +257,7 @@ with tqdm(total=NUM_QUERIES) as pbar:continueurl = res[i]["url"]- signals = res[i]["ranking_signals"]+ signals = res[i]["rankingSignals"]scores["query"].append(query)scores["url"].append(url)@@ -274,7 +274,7 @@ df = pd.DataFrame(scores)df = df.groupby(["query", "url"])df = df.mean().sort_values(by="score", ascending=False).reset_index()df["rank"] = df.groupby("query")["score"].rank(ascending=False)-df["ranking_signals"] = df["url"].map(ranking_signals)+df["rankingSignals"] = df["url"].map(ranking_signals)# convert to objectres = json.loads(df.to_json(orient="records"))diff --git a/optics/src/lib.rs b/optics/src/lib.rsindex 5b07d225..6a195a85 100644--- a/optics/src/lib.rs+++ b/optics/src/lib.rs@@ -375,6 +375,7 @@ impl ToString for Rule {}#[derive(Debug, PartialEq, Default, Serialize, Deserialize, Clone, ToSchema)]+#[serde(rename_all = "camelCase")]pub struct SiteRankings {pub liked: Vec<String>,pub disliked: Vec<String>,