Skip to content

web search done right

Simple calculator widget

Mikkel Denker28186fb52609
diff --git a/Cargo.lock b/Cargo.lock
index 53970873..fc0f7067 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -804,6 +804,8 @@ dependencies = [
"indicatif 0.16.2",
"itertools 0.10.5",
"kuchiki",
+ "lalrpop",
+ "lalrpop-util",
"logos",
"maplit",
"md5",
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 2000b37f..ba9a28e5 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -71,6 +71,10 @@ crossbeam-channel = { workspace = true }
num_cpus = { workspace = true }
onnxruntime = { workspace = true }
tokenizers = { workspace = true }
+lalrpop-util = { workspace = true }
+
+[build-dependencies]
+lalrpop = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
diff --git a/core/build.rs b/core/build.rs
new file mode 100644
index 00000000..ca5c2836
--- /dev/null
+++ b/core/build.rs
@@ -0,0 +1,3 @@
+fn main() {
+ lalrpop::process_root().unwrap();
+}
diff --git a/core/src/frontend/about.rs b/core/src/api/about.rs
similarity index 100%
rename from core/src/frontend/about.rs
rename to core/src/api/about.rs
diff --git a/core/src/frontend/autosuggest.rs b/core/src/api/autosuggest.rs
similarity index 100%
rename from core/src/frontend/autosuggest.rs
rename to core/src/api/autosuggest.rs
diff --git a/core/src/frontend/index.rs b/core/src/api/index.rs
similarity index 100%
rename from core/src/frontend/index.rs
rename to core/src/api/index.rs
diff --git a/core/src/frontend/mod.rs b/core/src/api/mod.rs
similarity index 100%
rename from core/src/frontend/mod.rs
rename to core/src/api/mod.rs
diff --git a/core/src/frontend/opensearch.rs b/core/src/api/opensearch.rs
similarity index 100%
rename from core/src/frontend/opensearch.rs
rename to core/src/api/opensearch.rs
diff --git a/core/src/frontend/optics.rs b/core/src/api/optics.rs
similarity index 100%
rename from core/src/frontend/optics.rs
rename to core/src/api/optics.rs
diff --git a/core/src/frontend/privacy.rs b/core/src/api/privacy.rs
similarity index 100%
rename from core/src/frontend/privacy.rs
rename to core/src/api/privacy.rs
diff --git a/core/src/frontend/search.rs b/core/src/api/search.rs
similarity index 97%
rename from core/src/frontend/search.rs
rename to core/src/api/search.rs
index 07d613e6..7d65eddf 100644
--- a/core/src/frontend/search.rs
+++ b/core/src/api/search.rs
@@ -28,6 +28,7 @@ use crate::{
},
searcher::{self, SearchQuery, SearchResult, NUM_RESULTS_PER_PAGE},
webpage::region::{Region, ALL_REGIONS},
+ widgets::Widget,
};
use super::{
@@ -47,6 +48,7 @@ struct SearchTemplate {
discussions: Option<Vec<DisplayedWebpage>>,
query: String,
sidebar: Option<Sidebar>,
+ widget: Option<Widget>,
spell_correction: Option<HighlightedSpellCorrection>,
num_matches: String,
search_duration_sec: String,
@@ -139,9 +141,6 @@ pub async fn route(
{
Ok(result) => match result {
SearchResult::Websites(result) => {
- let sidebar = result.sidebar;
- let spell_correction = result.spell_corrected_query;
-
let num_matches = thousand_sep_number(result.num_hits);
let search_duration_sec =
@@ -157,8 +156,9 @@ pub async fn route(
search_result: result.webpages,
discussions: result.discussions,
query,
- sidebar,
- spell_correction,
+ sidebar: result.sidebar,
+ widget: result.widget,
+ spell_correction: result.spell_corrected_query,
num_matches,
search_duration_sec,
all_regions,
diff --git a/core/src/frontend/sites.rs b/core/src/api/sites.rs
similarity index 100%
rename from core/src/frontend/sites.rs
rename to core/src/api/sites.rs
diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rs
index e3f0003f..0b827ea6 100644
--- a/core/src/entrypoint/frontend.rs
+++ b/core/src/entrypoint/frontend.rs
@@ -16,7 +16,7 @@
use anyhow::Result;
-use crate::{frontend::router, FrontendConfig};
+use crate::{api::router, FrontendConfig};
pub async fn run(config: FrontendConfig) -> Result<()> {
let app = router(
diff --git a/core/src/lib.rs b/core/src/lib.rs
index a688eb7d..513f02b9 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -31,6 +31,7 @@ mod inverted_index;
pub mod mapreduce;
+mod api;
mod autosuggest;
mod bangs;
mod collector;
@@ -38,7 +39,6 @@ mod directory;
mod entity_index;
mod exponential_backoff;
mod fastfield_cache;
-mod frontend;
mod human_website_annotations;
mod hyperloglog;
mod image_downloader;
@@ -64,6 +64,7 @@ mod ttl_cache;
mod warc;
pub mod webgraph;
pub mod webpage;
+mod widgets;
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingMasterConfig {
@@ -220,9 +221,6 @@ pub enum Error {
#[error("Query cannot be completely empty")]
EmptyQuery,
- #[error("Error parsing the query")]
- QueryParser,
-
#[error("Unknown region")]
UnknownRegion,
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 1ea9b84c..c1a0c801 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -28,6 +28,7 @@ use crate::{
create_stackoverflow_sidebar, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,
},
searcher::WebsitesResult,
+ widgets::Widget,
};
use std::{
@@ -475,10 +476,16 @@ impl DistributedSearcher {
let sidebar = self.sidebar(&initial_results, query).await?;
let discussions = self.discussions_widget(query).await?;
- let spell_corrected_query = initial_results
- .first()
- .and_then(|result| result.local_result.spell_corrected_query.clone())
- .map(HighlightedSpellCorrection::from);
+ let widget = self.widget(query);
+
+ let spell_corrected_query = if widget.is_none() {
+ initial_results
+ .first()
+ .and_then(|result| result.local_result.spell_corrected_query.clone())
+ .map(HighlightedSpellCorrection::from)
+ } else {
+ None
+ };
let num_docs = initial_results
.iter()
@@ -506,6 +513,7 @@ impl DistributedSearcher {
num_hits: num_docs,
webpages: retrieved_webpages,
sidebar,
+ widget,
discussions,
search_duration_ms,
})
@@ -518,4 +526,12 @@ impl DistributedSearcher {
Ok(SearchResult::Websites(self.search_websites(query).await?))
}
+
+ fn widget(&self, query: &SearchQuery) -> Option<Widget> {
+ if query.offset > 0 {
+ return None;
+ }
+
+ Widget::try_new(&query.query)
+ }
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 40a06f4a..8552e6c4 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -327,6 +327,7 @@ impl LocalSearcher {
.map(DisplayedWebpage::from)
.collect(),
discussions: None,
+ widget: None,
sidebar: search_result.entity_sidebar.map(Sidebar::Entity),
search_duration_ms: start.elapsed().as_millis(),
})
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 8de162fb..8de8f49a 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -28,6 +28,7 @@ use crate::{
search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection, Sidebar},
spell::Correction,
webpage::region::Region,
+ widgets::Widget,
};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
@@ -44,6 +45,7 @@ pub struct WebsitesResult {
pub webpages: Vec<DisplayedWebpage>,
pub num_hits: usize,
pub sidebar: Option<Sidebar>,
+ pub widget: Option<Widget>,
pub discussions: Option<Vec<DisplayedWebpage>>,
pub search_duration_ms: u128,
}
diff --git a/core/src/widgets/calculator.lalrpop b/core/src/widgets/calculator.lalrpop
new file mode 100644
index 00000000..6eef0641
--- /dev/null
+++ b/core/src/widgets/calculator.lalrpop
@@ -0,0 +1,33 @@
+use std::str::FromStr;
+use crate::widgets::calculator::{Expr, Opcode};
+
+grammar;
+
+pub Expr: Box<Expr> = {
+ Expr ExprOp Factor => Box::new(Expr::Op(<>)),
+ Factor,
+};
+
+ExprOp: Opcode = {
+ "+" => Opcode::Add,
+ "-" => Opcode::Sub,
+};
+
+Factor: Box<Expr> = {
+ Factor FactorOp Term => Box::new(Expr::Op(<>)),
+ Term,
+};
+
+FactorOp: Opcode = {
+ "*" => Opcode::Mul,
+ "/" => Opcode::Div,
+};
+
+Term: Box<Expr> = {
+ Num => Box::new(Expr::Number(<>)),
+ "(" <Expr> ")"
+};
+
+Num: f64 = {
+ r"([0-9]*[.])?[0-9]+" => f64::from_str(<>).unwrap()
+};
\ No newline at end of file
diff --git a/core/src/widgets/calculator.rs b/core/src/widgets/calculator.rs
new file mode 100644
index 00000000..3a3a90a7
--- /dev/null
+++ b/core/src/widgets/calculator.rs
@@ -0,0 +1,145 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::widgets::{Error, Result};
+use lalrpop_util::lalrpop_mod;
+use serde::Serialize;
+use std::fmt::{Debug, Display, Formatter};
+
+lalrpop_mod!(pub parser, "/widgets/calculator.rs");
+pub static PARSER: once_cell::sync::Lazy<parser::ExprParser> =
+ once_cell::sync::Lazy::new(parser::ExprParser::new);
+
+#[derive(Serialize)]
+pub enum Expr {
+ Number(f64),
+ Op(Box<Expr>, Opcode, Box<Expr>),
+}
+impl Expr {
+ fn calculate(&self) -> f64 {
+ match self {
+ Expr::Number(n) => *n,
+ Expr::Op(lhs, op, rhs) => {
+ let lhs = lhs.calculate();
+ let rhs = rhs.calculate();
+
+ match op {
+ Opcode::Mul => lhs * rhs,
+ Opcode::Div => lhs / rhs,
+ Opcode::Add => lhs + rhs,
+ Opcode::Sub => lhs - rhs,
+ }
+ }
+ }
+ }
+}
+
+#[derive(Copy, Clone, Serialize)]
+pub enum Opcode {
+ Mul,
+ Div,
+ Add,
+ Sub,
+}
+
+impl Debug for Expr {
+ fn fmt(&self, fmt: &mut Formatter) -> std::result::Result<(), std::fmt::Error> {
+ use self::Expr::*;
+ match *self {
+ Number(n) => write!(fmt, "{:?}", n),
+ Op(ref l, op, ref r) => write!(fmt, "({:?} {:?} {:?})", l, op, r),
+ }
+ }
+}
+
+impl Display for Expr {
+ fn fmt(&self, fmt: &mut Formatter<'_>) -> std::fmt::Result {
+ use self::Expr::*;
+ match *self {
+ Number(n) => write!(fmt, "{}", n),
+ Op(ref l, op, ref r) => write!(fmt, "{} {} {}", l, op, r),
+ }
+ }
+}
+
+impl Debug for Opcode {
+ fn fmt(&self, fmt: &mut Formatter) -> std::result::Result<(), std::fmt::Error> {
+ use self::Opcode::*;
+ match *self {
+ Mul => write!(fmt, "*"),
+ Div => write!(fmt, "/"),
+ Add => write!(fmt, "+"),
+ Sub => write!(fmt, "-"),
+ }
+ }
+}
+
+impl Display for Opcode {
+ fn fmt(&self, fmt: &mut Formatter) -> std::result::Result<(), std::fmt::Error> {
+ use self::Opcode::*;
+ match *self {
+ Mul => write!(fmt, "*"),
+ Div => write!(fmt, "/"),
+ Add => write!(fmt, "+"),
+ Sub => write!(fmt, "-"),
+ }
+ }
+}
+
+fn parse(expr: &str) -> Result<Box<Expr>> {
+ match PARSER.parse(expr) {
+ Ok(expr) => Ok(expr),
+ Err(_) => Err(Error::CalculatorParse),
+ }
+}
+
+#[derive(Debug, Serialize)]
+pub struct Calculation {
+ pub input: String,
+ pub expr: Box<Expr>,
+ pub result: f64,
+}
+
+pub fn try_calculate(expr: &str) -> Result<Calculation> {
+ let input = expr.to_string();
+ let expr = parse(expr)?;
+
+ Ok(Calculation {
+ input,
+ result: expr.calculate(),
+ expr,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn it_calculates_simple_expressions() {
+ assert_eq!(try_calculate("2+2").unwrap().result, 4.0);
+ assert_eq!(try_calculate("2*2").unwrap().result, 4.0);
+ assert_eq!(try_calculate("2*3").unwrap().result, 6.0);
+ assert!(try_calculate("2.1-3").unwrap().result + 0.9 < 0.0001);
+ assert_eq!(try_calculate("6/2").unwrap().result, 3.0);
+ }
+
+ #[test]
+ fn it_respects_paranthesis() {
+ assert_eq!(try_calculate("2+2*6").unwrap().result, 14.0);
+ assert_eq!(try_calculate("(2+2)*6").unwrap().result, 24.0);
+ }
+}
diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rs
new file mode 100644
index 00000000..8c7417cc
--- /dev/null
+++ b/core/src/widgets/mod.rs
@@ -0,0 +1,45 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::Serialize;
+use thiserror::Error;
+
+use self::calculator::{try_calculate, Calculation};
+
+mod calculator;
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Error, Debug)]
+pub enum Error {
+ #[error("Calculator parse")]
+ CalculatorParse,
+}
+
+#[derive(Debug, Serialize)]
+pub enum Widget {
+ Calculator(Calculation),
+}
+
+impl Widget {
+ pub fn try_new(query: &str) -> Option<Widget> {
+ Self::calculator(query)
+ }
+
+ fn calculator(query: &str) -> Option<Widget> {
+ try_calculate(query).ok().map(Widget::Calculator)
+ }
+}
diff --git a/frontend/src/components/CalculatorWidget.astro b/frontend/src/components/CalculatorWidget.astro
new file mode 100644
index 00000000..be5ab4a3
--- /dev/null
+++ b/frontend/src/components/CalculatorWidget.astro
@@ -0,0 +1,16 @@
+---
+import { askama } from "../askama";
+
+export interface Props {
+ calculation: string;
+}
+---
+
+<div class="border rounded-xl p-5 flex flex-col items-end">
+ <div class="text-xs text-gray-500 flex w-fit">
+ {askama`calculation.input`} =
+ </div>
+ <div class="font-bold text-3xl flex w-fit">
+ {askama`calculation.result`}
+ </div>
+</div>
\ No newline at end of file
diff --git a/frontend/src/components/Widget.astro b/frontend/src/components/Widget.astro
new file mode 100644
index 00000000..2251bac7
--- /dev/null
+++ b/frontend/src/components/Widget.astro
@@ -0,0 +1,22 @@
+---
+import { askama } from "../askama";
+import CalculatorWidget from "./CalculatorWidget.astro";
+
+export interface Props {
+ widget: string;
+}
+---
+
+{
+askama.if_("let Some(widget) = widget", (
+ <div>
+ {
+ askama.match("widget", {
+ "Widget::Calculator(calculation)": (
+ <CalculatorWidget calculation="calculation" />
+ )
+ })
+ }
+ </div>
+))
+}
\ No newline at end of file
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index 3e7b4a55..0009bd8d 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -8,6 +8,7 @@ import OpticSelector from "../components/OpticSelector.astro";
import Header from "../components/Header.astro";
import RankingModal from "../components/RankingModal.astro";
import Discussions from "../components/Discussions.astro";
+import Widget from "../components/Widget.astro";
import { askama } from "../askama";
---
@@ -78,6 +79,8 @@ import { askama } from "../askama";
))
}
+ <Widget widget="widget" />
+
{
askama.for_("item in search_result", () => (
<>
diff --git a/scripts/export_crossencoder b/scripts/export_crossencoder
index 8b02b183..50475818 100755
--- a/scripts/export_crossencoder
+++ b/scripts/export_crossencoder
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
import os
from optimum.onnxruntime import ORTModelForSequenceClassification, ORTQuantizer
-from optimum.onnxruntime.configuration import AutoQuantizationConfig, QuantizationMode
+from optimum.onnxruntime.configuration import AutoQuantizationConfig
import transformers
from functools import partial
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index 1991a5aa..5200fc7e 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -1,4 +1,5 @@
transformers[onnx]
+tensorboard>1.15.0
torch
optimum
datasets
\ No newline at end of file