web search done right
Simple calculator widget
Mikkel Denker28186fb52609
diff --git a/Cargo.lock b/Cargo.lockindex 53970873..fc0f7067 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -804,6 +804,8 @@ dependencies = ["indicatif 0.16.2","itertools 0.10.5","kuchiki",+ "lalrpop",+ "lalrpop-util","logos","maplit","md5",diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 2000b37f..ba9a28e5 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -71,6 +71,10 @@ crossbeam-channel = { workspace = true }num_cpus = { workspace = true }onnxruntime = { workspace = true }tokenizers = { workspace = true }+lalrpop-util = { workspace = true }++[build-dependencies]+lalrpop = { workspace = true }[dev-dependencies]criterion = { workspace = true }diff --git a/core/build.rs b/core/build.rsnew file mode 100644index 00000000..ca5c2836--- /dev/null+++ b/core/build.rs@@ -0,0 +1,3 @@+fn main() {+ lalrpop::process_root().unwrap();+}diff --git a/core/src/frontend/about.rs b/core/src/api/about.rssimilarity index 100%rename from core/src/frontend/about.rsrename to core/src/api/about.rsdiff --git a/core/src/frontend/autosuggest.rs b/core/src/api/autosuggest.rssimilarity index 100%rename from core/src/frontend/autosuggest.rsrename to core/src/api/autosuggest.rsdiff --git a/core/src/frontend/index.rs b/core/src/api/index.rssimilarity index 100%rename from core/src/frontend/index.rsrename to core/src/api/index.rsdiff --git a/core/src/frontend/mod.rs b/core/src/api/mod.rssimilarity index 100%rename from core/src/frontend/mod.rsrename to core/src/api/mod.rsdiff --git a/core/src/frontend/opensearch.rs b/core/src/api/opensearch.rssimilarity index 100%rename from core/src/frontend/opensearch.rsrename to core/src/api/opensearch.rsdiff --git a/core/src/frontend/optics.rs b/core/src/api/optics.rssimilarity index 100%rename from core/src/frontend/optics.rsrename to core/src/api/optics.rsdiff --git a/core/src/frontend/privacy.rs b/core/src/api/privacy.rssimilarity index 100%rename from core/src/frontend/privacy.rsrename to core/src/api/privacy.rsdiff --git a/core/src/frontend/search.rs b/core/src/api/search.rssimilarity index 97%rename from core/src/frontend/search.rsrename to core/src/api/search.rsindex 07d613e6..7d65eddf 100644--- a/core/src/frontend/search.rs+++ b/core/src/api/search.rs@@ -28,6 +28,7 @@ use crate::{},searcher::{self, SearchQuery, SearchResult, NUM_RESULTS_PER_PAGE},webpage::region::{Region, ALL_REGIONS},+ widgets::Widget,};use super::{@@ -47,6 +48,7 @@ struct SearchTemplate {discussions: Option<Vec<DisplayedWebpage>>,query: String,sidebar: Option<Sidebar>,+ widget: Option<Widget>,spell_correction: Option<HighlightedSpellCorrection>,num_matches: String,search_duration_sec: String,@@ -139,9 +141,6 @@ pub async fn route({Ok(result) => match result {SearchResult::Websites(result) => {- let sidebar = result.sidebar;- let spell_correction = result.spell_corrected_query;-let num_matches = thousand_sep_number(result.num_hits);let search_duration_sec =@@ -157,8 +156,9 @@ pub async fn route(search_result: result.webpages,discussions: result.discussions,query,- sidebar,- spell_correction,+ sidebar: result.sidebar,+ widget: result.widget,+ spell_correction: result.spell_corrected_query,num_matches,search_duration_sec,all_regions,diff --git a/core/src/frontend/sites.rs b/core/src/api/sites.rssimilarity index 100%rename from core/src/frontend/sites.rsrename to core/src/api/sites.rsdiff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rsindex e3f0003f..0b827ea6 100644--- a/core/src/entrypoint/frontend.rs+++ b/core/src/entrypoint/frontend.rs@@ -16,7 +16,7 @@use anyhow::Result;-use crate::{frontend::router, FrontendConfig};+use crate::{api::router, FrontendConfig};pub async fn run(config: FrontendConfig) -> Result<()> {let app = router(diff --git a/core/src/lib.rs b/core/src/lib.rsindex a688eb7d..513f02b9 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -31,6 +31,7 @@ mod inverted_index;pub mod mapreduce;+mod api;mod autosuggest;mod bangs;mod collector;@@ -38,7 +39,6 @@ mod directory;mod entity_index;mod exponential_backoff;mod fastfield_cache;-mod frontend;mod human_website_annotations;mod hyperloglog;mod image_downloader;@@ -64,6 +64,7 @@ mod ttl_cache;mod warc;pub mod webgraph;pub mod webpage;+mod widgets;#[derive(Debug, Deserialize, Clone)]pub struct IndexingMasterConfig {@@ -220,9 +221,6 @@ pub enum Error {#[error("Query cannot be completely empty")]EmptyQuery,- #[error("Error parsing the query")]- QueryParser,-#[error("Unknown region")]UnknownRegion,diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 1ea9b84c..c1a0c801 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -28,6 +28,7 @@ use crate::{create_stackoverflow_sidebar, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,},searcher::WebsitesResult,+ widgets::Widget,};use std::{@@ -475,10 +476,16 @@ impl DistributedSearcher {let sidebar = self.sidebar(&initial_results, query).await?;let discussions = self.discussions_widget(query).await?;- let spell_corrected_query = initial_results- .first()- .and_then(|result| result.local_result.spell_corrected_query.clone())- .map(HighlightedSpellCorrection::from);+ let widget = self.widget(query);++ let spell_corrected_query = if widget.is_none() {+ initial_results+ .first()+ .and_then(|result| result.local_result.spell_corrected_query.clone())+ .map(HighlightedSpellCorrection::from)+ } else {+ None+ };let num_docs = initial_results.iter()@@ -506,6 +513,7 @@ impl DistributedSearcher {num_hits: num_docs,webpages: retrieved_webpages,sidebar,+ widget,discussions,search_duration_ms,})@@ -518,4 +526,12 @@ impl DistributedSearcher {Ok(SearchResult::Websites(self.search_websites(query).await?))}++ fn widget(&self, query: &SearchQuery) -> Option<Widget> {+ if query.offset > 0 {+ return None;+ }++ Widget::try_new(&query.query)+ }}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 40a06f4a..8552e6c4 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -327,6 +327,7 @@ impl LocalSearcher {.map(DisplayedWebpage::from).collect(),discussions: None,+ widget: None,sidebar: search_result.entity_sidebar.map(Sidebar::Entity),search_duration_ms: start.elapsed().as_millis(),})diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 8de162fb..8de8f49a 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -28,6 +28,7 @@ use crate::{search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection, Sidebar},spell::Correction,webpage::region::Region,+ widgets::Widget,};pub const NUM_RESULTS_PER_PAGE: usize = 20;@@ -44,6 +45,7 @@ pub struct WebsitesResult {pub webpages: Vec<DisplayedWebpage>,pub num_hits: usize,pub sidebar: Option<Sidebar>,+ pub widget: Option<Widget>,pub discussions: Option<Vec<DisplayedWebpage>>,pub search_duration_ms: u128,}diff --git a/core/src/widgets/calculator.lalrpop b/core/src/widgets/calculator.lalrpopnew file mode 100644index 00000000..6eef0641--- /dev/null+++ b/core/src/widgets/calculator.lalrpop@@ -0,0 +1,33 @@+use std::str::FromStr;+use crate::widgets::calculator::{Expr, Opcode};++grammar;++pub Expr: Box<Expr> = {+ Expr ExprOp Factor => Box::new(Expr::Op(<>)),+ Factor,+};++ExprOp: Opcode = {+ "+" => Opcode::Add,+ "-" => Opcode::Sub,+};++Factor: Box<Expr> = {+ Factor FactorOp Term => Box::new(Expr::Op(<>)),+ Term,+};++FactorOp: Opcode = {+ "*" => Opcode::Mul,+ "/" => Opcode::Div,+};++Term: Box<Expr> = {+ Num => Box::new(Expr::Number(<>)),+ "(" <Expr> ")"+};++Num: f64 = {+ r"([0-9]*[.])?[0-9]+" => f64::from_str(<>).unwrap()+};\ No newline at end of filediff --git a/core/src/widgets/calculator.rs b/core/src/widgets/calculator.rsnew file mode 100644index 00000000..3a3a90a7--- /dev/null+++ b/core/src/widgets/calculator.rs@@ -0,0 +1,145 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::widgets::{Error, Result};+use lalrpop_util::lalrpop_mod;+use serde::Serialize;+use std::fmt::{Debug, Display, Formatter};++lalrpop_mod!(pub parser, "/widgets/calculator.rs");+pub static PARSER: once_cell::sync::Lazy<parser::ExprParser> =+ once_cell::sync::Lazy::new(parser::ExprParser::new);++#[derive(Serialize)]+pub enum Expr {+ Number(f64),+ Op(Box<Expr>, Opcode, Box<Expr>),+}+impl Expr {+ fn calculate(&self) -> f64 {+ match self {+ Expr::Number(n) => *n,+ Expr::Op(lhs, op, rhs) => {+ let lhs = lhs.calculate();+ let rhs = rhs.calculate();++ match op {+ Opcode::Mul => lhs * rhs,+ Opcode::Div => lhs / rhs,+ Opcode::Add => lhs + rhs,+ Opcode::Sub => lhs - rhs,+ }+ }+ }+ }+}++#[derive(Copy, Clone, Serialize)]+pub enum Opcode {+ Mul,+ Div,+ Add,+ Sub,+}++impl Debug for Expr {+ fn fmt(&self, fmt: &mut Formatter) -> std::result::Result<(), std::fmt::Error> {+ use self::Expr::*;+ match *self {+ Number(n) => write!(fmt, "{:?}", n),+ Op(ref l, op, ref r) => write!(fmt, "({:?} {:?} {:?})", l, op, r),+ }+ }+}++impl Display for Expr {+ fn fmt(&self, fmt: &mut Formatter<'_>) -> std::fmt::Result {+ use self::Expr::*;+ match *self {+ Number(n) => write!(fmt, "{}", n),+ Op(ref l, op, ref r) => write!(fmt, "{} {} {}", l, op, r),+ }+ }+}++impl Debug for Opcode {+ fn fmt(&self, fmt: &mut Formatter) -> std::result::Result<(), std::fmt::Error> {+ use self::Opcode::*;+ match *self {+ Mul => write!(fmt, "*"),+ Div => write!(fmt, "/"),+ Add => write!(fmt, "+"),+ Sub => write!(fmt, "-"),+ }+ }+}++impl Display for Opcode {+ fn fmt(&self, fmt: &mut Formatter) -> std::result::Result<(), std::fmt::Error> {+ use self::Opcode::*;+ match *self {+ Mul => write!(fmt, "*"),+ Div => write!(fmt, "/"),+ Add => write!(fmt, "+"),+ Sub => write!(fmt, "-"),+ }+ }+}++fn parse(expr: &str) -> Result<Box<Expr>> {+ match PARSER.parse(expr) {+ Ok(expr) => Ok(expr),+ Err(_) => Err(Error::CalculatorParse),+ }+}++#[derive(Debug, Serialize)]+pub struct Calculation {+ pub input: String,+ pub expr: Box<Expr>,+ pub result: f64,+}++pub fn try_calculate(expr: &str) -> Result<Calculation> {+ let input = expr.to_string();+ let expr = parse(expr)?;++ Ok(Calculation {+ input,+ result: expr.calculate(),+ expr,+ })+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn it_calculates_simple_expressions() {+ assert_eq!(try_calculate("2+2").unwrap().result, 4.0);+ assert_eq!(try_calculate("2*2").unwrap().result, 4.0);+ assert_eq!(try_calculate("2*3").unwrap().result, 6.0);+ assert!(try_calculate("2.1-3").unwrap().result + 0.9 < 0.0001);+ assert_eq!(try_calculate("6/2").unwrap().result, 3.0);+ }++ #[test]+ fn it_respects_paranthesis() {+ assert_eq!(try_calculate("2+2*6").unwrap().result, 14.0);+ assert_eq!(try_calculate("(2+2)*6").unwrap().result, 24.0);+ }+}diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rsnew file mode 100644index 00000000..8c7417cc--- /dev/null+++ b/core/src/widgets/mod.rs@@ -0,0 +1,45 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use serde::Serialize;+use thiserror::Error;++use self::calculator::{try_calculate, Calculation};++mod calculator;++pub type Result<T> = std::result::Result<T, Error>;++#[derive(Error, Debug)]+pub enum Error {+ #[error("Calculator parse")]+ CalculatorParse,+}++#[derive(Debug, Serialize)]+pub enum Widget {+ Calculator(Calculation),+}++impl Widget {+ pub fn try_new(query: &str) -> Option<Widget> {+ Self::calculator(query)+ }++ fn calculator(query: &str) -> Option<Widget> {+ try_calculate(query).ok().map(Widget::Calculator)+ }+}diff --git a/frontend/src/components/CalculatorWidget.astro b/frontend/src/components/CalculatorWidget.astronew file mode 100644index 00000000..be5ab4a3--- /dev/null+++ b/frontend/src/components/CalculatorWidget.astro@@ -0,0 +1,16 @@+---+import { askama } from "../askama";++export interface Props {+ calculation: string;+}+---++<div class="border rounded-xl p-5 flex flex-col items-end">+ <div class="text-xs text-gray-500 flex w-fit">+ {askama`calculation.input`} =+ </div>+ <div class="font-bold text-3xl flex w-fit">+ {askama`calculation.result`}+ </div>+</div>\ No newline at end of filediff --git a/frontend/src/components/Widget.astro b/frontend/src/components/Widget.astronew file mode 100644index 00000000..2251bac7--- /dev/null+++ b/frontend/src/components/Widget.astro@@ -0,0 +1,22 @@+---+import { askama } from "../askama";+import CalculatorWidget from "./CalculatorWidget.astro";++export interface Props {+ widget: string;+}+---++{+askama.if_("let Some(widget) = widget", (+ <div>+ {+ askama.match("widget", {+ "Widget::Calculator(calculation)": (+ <CalculatorWidget calculation="calculation" />+ )+ })+ }+ </div>+))+}\ No newline at end of filediff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astroindex 3e7b4a55..0009bd8d 100644--- a/frontend/src/pages/search.astro+++ b/frontend/src/pages/search.astro@@ -8,6 +8,7 @@ import OpticSelector from "../components/OpticSelector.astro";import Header from "../components/Header.astro";import RankingModal from "../components/RankingModal.astro";import Discussions from "../components/Discussions.astro";+import Widget from "../components/Widget.astro";import { askama } from "../askama";---@@ -78,6 +79,8 @@ import { askama } from "../askama";))}+ <Widget widget="widget" />+{askama.for_("item in search_result", () => (<>diff --git a/scripts/export_crossencoder b/scripts/export_crossencoderindex 8b02b183..50475818 100755--- a/scripts/export_crossencoder+++ b/scripts/export_crossencoder@@ -1,7 +1,7 @@#!/usr/bin/env python3import osfrom optimum.onnxruntime import ORTModelForSequenceClassification, ORTQuantizer-from optimum.onnxruntime.configuration import AutoQuantizationConfig, QuantizationMode+from optimum.onnxruntime.configuration import AutoQuantizationConfigimport transformersfrom functools import partialdiff --git a/scripts/requirements.txt b/scripts/requirements.txtindex 1991a5aa..5200fc7e 100644--- a/scripts/requirements.txt+++ b/scripts/requirements.txt@@ -1,4 +1,5 @@transformers[onnx]+tensorboard>1.15.0torchoptimumdatasets\ No newline at end of file