web search done right
Thesaurus widget and minor bugfixes to calculator.
Mikkel Denker1eea731d26c1
diff --git a/.gitignore b/.gitignoreindex bc55d0f5..a76ab465 100644--- a/.gitignore+++ b/.gitignore@@ -13,3 +13,4 @@ notesltr_scores.json.venvlibtorch+proptest-regressionsdiff --git a/Cargo.lock b/Cargo.lockindex 83203a97..4dd9d8ec 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2871,6 +2871,18 @@ dependencies = ["stable_deref_trait",]+[[package]]+name = "oxilangtag"+version = "0.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8d91edf4fbb970279443471345a4e8c491bf05bb283b3e6c88e4e606fd8c181b"++[[package]]+name = "oxiri"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bb175ec8981211357b7b379869c2f8d555881c55ea62311428ec0de46d89bd5c"+[[package]]name = "parking_lot"version = "0.12.1"@@ -3605,6 +3617,23 @@ dependencies = ["thiserror",]+[[package]]+name = "rio_api"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1924fa1f0e6d851f9b73b3c569e607c368a0d92995d99d563ad7bf1414696603"++[[package]]+name = "rio_turtle"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5cec59971eafd99b9c7e3544bfcabafea81a7072ac51c9f46985ca0bd7ba6016"+dependencies = [+ "oxilangtag",+ "oxiri",+ "rio_api",+]+[[package]]name = "rkyv"version = "0.7.42"@@ -4280,6 +4309,8 @@ dependencies = ["regex","reqwest","reqwest-eventsource",+ "rio_api",+ "rio_turtle","rkyv","robotstxt-with-cache","rocksdb",diff --git a/Cargo.toml b/Cargo.tomlindex 6d67b4ed..53464fac 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -43,6 +43,8 @@ chrono = { version = "0.4.23", features = ["serde"] }uuid = "1.1.2"futures = "0.3.21"quick-xml = "0.23.0"+rio_turtle = "0.8.4"+rio_api = "0.8.4"bzip2 = "0.4.4"parse_wiki_text = "0.1.5"md5 = "0.7.0"diff --git a/configs/api.toml b/configs/api.tomlindex 610e979a..b3113812 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -1,16 +1,19 @@cluster_id = "dev_api"-queries_csv_path = "data/queries_us.csv"-host = "0.0.0.0:3000"-prometheus_host = "0.0.0.0:3001"-gossip_addr = "0.0.0.0:3005"-gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]crossencoder_model_path = "data/cross_encoder"fact_check_model_path = "data/fact_model"+gossip_addr = "0.0.0.0:3005"+gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]+host = "0.0.0.0:3000"+prometheus_host = "0.0.0.0:3001"+queries_csv_path = "data/queries_us.csv"# lambda_model_path = "data/lambdamart.txt"# qa_model_path = "data/qa_model"bangs_path = "data/bangs.json"summarizer_path = "data/summarizer"[thresholds]-stackoverflow = 0.0entity_sidebar = 0.0+stackoverflow = 0.0++[widgets]+thesaurus_paths = ["data/english-wordnet-2022-subset.ttl"]diff --git a/core/Cargo.toml b/core/Cargo.tomlindex f8c84f93..cf610dc0 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -100,6 +100,8 @@ rkyv = { workspace = true }hashbrown = { workspace = true }mime = { workspace = true }fend-core = { workspace = true }+rio_turtle = { workspace = true }+rio_api = { workspace = true }[target.'cfg(not(target_env = "msvc"))'.dependencies]tikv-jemallocator = { workspace = true }diff --git a/core/src/api/docs.rs b/core/src/api/docs.rsindex 324f0659..35f0a3c2 100644--- a/core/src/api/docs.rs+++ b/core/src/api/docs.rs@@ -63,6 +63,14 @@ use utoipa_swagger_ui::SwaggerUi;crate::widgets::Widget,crate::widgets::calculator::Calculation,+ crate::thesaurus::ThesaurusWidget,+ crate::thesaurus::Lemma,+ crate::thesaurus::WordMeaning,+ crate::thesaurus::Definition,+ crate::thesaurus::Example,+ crate::thesaurus::PartOfSpeech,+ crate::thesaurus::PartOfSpeechMeaning,+crate::ranking::signal::SignalScore,crate::bangs::BangHit,crate::bangs::Bang,diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 6bf2f241..c5f6c008 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -133,8 +133,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {lambda_model,qa_model,bangs,- config.collector.clone(),- config.thresholds.clone(),+ config.clone(),);let state = Arc::new(State {diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex caaa7be2..92f7785e 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -206,6 +206,8 @@ pub struct ApiConfig {#[serde(default)]pub thresholds: ApiThresholds,++ pub widgets: WidgetsConfig,}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -354,3 +356,8 @@ pub struct WebgraphServerConfig {#[serde(default = "defaults::WebgraphServer::max_similar_sites")]pub max_similar_sites: usize,}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct WidgetsConfig {+ pub thesaurus_paths: Vec<String>,+}diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 102dfda4..ae3dd8f9 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -41,6 +41,7 @@ fn download_files() {"sample.warc.gz","bangs.json","content.rdf.u8.gz",+ "english-wordnet-2022-subset.ttl",] {info!("Downloading {}", name);let body = reqwest::get(format!("http://s3.trystract.com/{BUCKET_NAME}/{name}"))diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex f680c98d..2243177c 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -405,8 +405,8 @@ impl Indexer {index = index.merge(other);std::fs::remove_dir_all(other_path).unwrap();- index.inverted_index.merge_into_max_segments(1).unwrap();}+ index.inverted_index.merge_into_max_segments(1).unwrap();index}));diff --git a/core/src/lib.rs b/core/src/lib.rsindex b2e94909..493cecd7 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -73,6 +73,7 @@ mod snippet;pub mod spell;mod subdomain_count;pub mod summarizer;+mod thesaurus;mod tokenizer;#[allow(unused)]mod ttl_cache;diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex 47f0ace7..d5811bb3 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -417,7 +417,9 @@ pub fn parse(query: &str) -> Vec<Box<Term>> {}if cur_term_begin < query.len() {- res.push(parse_term(&query[cur_term_begin..query.len()]));+ res.push(parse_term(+ &query[floor_char_boundary(&query, cur_term_begin)..query.len()],+ ));}res@@ -426,6 +428,7 @@ pub fn parse(query: &str) -> Vec<Box<Term>> {#[cfg(test)]mod tests {use super::*;+ use proptest::prelude::*;#[test]fn parse_not() {@@ -551,4 +554,18 @@ mod tests {]);}++ #[test]+ fn unicode() {+ let query = "\u{a0}";+ assert_eq!(parse(query).len(), 1);+ }++ proptest! {+ #[test]+ fn prop(query: String) {+ let res = parse(&query);+ prop_assert_eq!(query.is_empty() || query.chars().all(|c| c.is_whitespace()), res.is_empty());+ }+ }}diff --git a/core/src/schema.rs b/core/src/schema.rsindex 6998960f..82027f54 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -590,7 +590,6 @@ impl Field {| Field::Text(TextField::TitleIfHomepage)| Field::Text(TextField::SiteWithout) // will match url| Field::Text(TextField::Domain) // will match url- | Field::Text(TextField::CleanBody) // will match all_body) && !self.is_fast()}diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex b16afa4e..9320f6bc 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -25,7 +25,7 @@ use optics::Optic;use url::Url;use crate::bangs::{Bang, BangHit};-use crate::config::{ApiThresholds, CollectorConfig};+use crate::config::{ApiConfig, ApiThresholds, CollectorConfig};use crate::inverted_index::RetrievedWebpage;use crate::ranking::ALL_SIGNALS;use crate::search_prettifier::{@@ -68,8 +68,7 @@ impl ApiSearcher {lambda_model: Option<LambdaMART>,qa_model: Option<QaModel>,bangs: Bangs,- collector_config: CollectorConfig,- thresholds: ApiThresholds,+ config: ApiConfig,) -> Self {Self {distributed_searcher: DistributedSearcher::new(cluster),@@ -77,9 +76,9 @@ impl ApiSearcher {lambda_model: lambda_model.map(Arc::new),qa_model: qa_model.map(Arc::new),bangs,- collector_config,- thresholds,- widgets: Widgets::new(),+ collector_config: config.collector,+ thresholds: config.thresholds,+ widgets: Widgets::new(config.widgets).unwrap(),}}diff --git a/core/src/thesaurus.rs b/core/src/thesaurus.rsnew file mode 100644index 00000000..51f467fa--- /dev/null+++ b/core/src/thesaurus.rs@@ -0,0 +1,487 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use itertools::Itertools;+use regex::Regex;+use serde::{Deserialize, Serialize};+use std::{fs::File, io::BufReader, path::Path};+use utoipa::ToSchema;++use anyhow::{anyhow, Result};+use rio_api::parser::TriplesParser;+use rio_turtle::TurtleParser;+use std::collections::HashMap;++#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase", transparent)]+pub struct Lemma(String);++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+struct NormalizedLemma(String);++impl Lemma {+ fn normalize(&self) -> NormalizedLemma {+ NormalizedLemma(self.0.to_lowercase())+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+struct Id(String);++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase", transparent)]+pub struct Definition(String);++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase", transparent)]+pub struct Example(String);++#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]+pub enum PartOfSpeech {+ Noun,+ Verb,+ Adjective,+ AdjectiveSatellite,+ Adverb,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct Info {+ id: Id,+ definition: Definition,+ examples: Vec<Example>,+ similar: Vec<Id>,+ pos: PartOfSpeech,+}++pub struct Dictionary {+ map: HashMap<Id, Info>,+ lemmas: HashMap<NormalizedLemma, Vec<Id>>,+ reverse_lemmas: HashMap<Id, Vec<NormalizedLemma>>,+ spellings: HashMap<NormalizedLemma, Lemma>,++ matchers: Vec<Regex>,+}++#[derive(Debug, Clone, PartialEq, Eq, Hash)]+struct Node(String);++#[derive(Debug)]+struct Edge {+ source: Node,+ label: String,+ target: Node,+}++#[derive(Debug)]+struct Graph {+ map: HashMap<Node, Vec<Edge>>,+}++impl Graph {+ pub fn new() -> Self {+ Self {+ map: HashMap::new(),+ }+ }++ pub fn insert(&mut self, edge: Edge) {+ self.map.entry(edge.source.clone()).or_default().push(edge);+ }++ pub fn get(&self, node: &Node) -> Option<&Vec<Edge>> {+ self.map.get(node)+ }+}++struct EdgeQuery<'a> {+ to: NodeQuery<'a>,+ label: &'a str,+}++struct NodeQuery<'a> {+ node: &'a Node,+ graph: &'a Graph,+}++impl<'a> NodeQuery<'a> {+ pub fn new(node: &'a Node, graph: &'a Graph) -> Self {+ Self { node, graph }+ }++ pub fn edges(&self) -> Vec<EdgeQuery<'a>> {+ self.graph+ .get(self.node)+ .map(|edges| {+ edges+ .iter()+ .map(|e| EdgeQuery {+ to: NodeQuery::new(&e.target, self.graph),+ label: &e.label,+ })+ .collect()+ })+ .unwrap_or_default()+ }++ pub fn filtered_edges<F>(&self, f: F) -> Vec<EdgeQuery<'a>>+ where+ F: Fn(&EdgeQuery<'a>) -> bool,+ {+ self.edges().into_iter().filter(f).collect()+ }++ pub fn single_edge<F>(&self, f: F) -> Option<EdgeQuery<'a>>+ where+ F: Fn(&EdgeQuery<'a>) -> bool,+ {+ self.filtered_edges(f).into_iter().next()+ }+}++impl<'a> Clone for NodeQuery<'a> {+ fn clone(&self) -> Self {+ Self {+ node: self.node,+ graph: self.graph,+ }+ }+}++impl Dictionary {+ fn empty() -> Result<Self> {+ Ok(Self {+ map: HashMap::new(),+ lemmas: HashMap::new(),+ reverse_lemmas: HashMap::new(),+ spellings: HashMap::new(),+ matchers: vec![+ Regex::new(r"^definition of ([\w| ]+)$")?,+ Regex::new(r"^meaning of ([\w| ]+)$")?,+ Regex::new(r"^synonym of ([\w| ]+)$")?,+ Regex::new(r"^synonyms of ([\w| ]+)$")?,+ //+ Regex::new(r"^definition ([\w| ]+)$")?,+ Regex::new(r"^meaning ([\w| ]+)$")?,+ Regex::new(r"^synonym ([\w| ]+)$")?,+ Regex::new(r"^synonyms ([\w| ]+)$")?,+ //+ Regex::new(r"^([\w| ]+) definition$")?,+ Regex::new(r"^([\w| ]+) meaning$")?,+ Regex::new(r"^([\w| ]+) synonym$")?,+ Regex::new(r"^([\w| ]+) synonyms$")?,+ ],+ })+ }++ fn get_by_id(&self, id: Id) -> Option<&Info> {+ self.map.get(&id)+ }++ fn get(&self, lemma: Lemma) -> Vec<Info> {+ match self.lemmas.get(&lemma.normalize()) {+ Some(ids) => ids+ .iter()+ .filter_map(|id| self.get_by_id(id.clone()))+ .cloned()+ .collect(),+ None => Vec::new(),+ }+ }++ fn query_lemma(&self, query: &str) -> Option<String> {+ for matcher in &self.matchers {+ if let Some(captures) = matcher.captures(query) {+ return captures.get(1).map(|m| m.as_str().to_string());+ }+ }++ None+ }++ pub fn lookup(&self, query: &str) -> Option<ThesaurusWidget> {+ let query = self.query_lemma(query)?;+ let lemma = Lemma(query);++ let infos = self.get(lemma.clone());++ if infos.is_empty() {+ return None;+ }++ let mut meanings: HashMap<PartOfSpeech, Vec<WordMeaning>> = HashMap::new();++ for info in infos {+ meanings+ .entry(info.pos.clone())+ .or_default()+ .push(WordMeaning {+ definition: info.definition.clone(),+ examples: info.examples.clone(),+ similar: info+ .similar+ .iter()+ .filter_map(|id| self.get_by_id(id.clone()))+ .filter_map(|i| self.reverse_lemmas.get(&i.id))+ .flatten()+ .filter_map(|l| self.spellings.get(l))+ .filter(|&l| l != &lemma)+ .cloned()+ .dedup_by(|a, b| a == b)+ .collect(),+ });+ }++ Some(ThesaurusWidget {+ term: lemma,+ meanings: meanings+ .into_iter()+ .map(|(pos, meanings)| PartOfSpeechMeaning { pos, meanings })+ .sorted()+ .collect(),+ })+ }++ pub fn insert(&mut self, lemma: Lemma, info: Info) {+ let normalized = lemma.clone().normalize();++ self.map.insert(info.id.clone(), info.clone());++ self.lemmas+ .entry(normalized.clone())+ .or_default()+ .push(info.id.clone());++ self.reverse_lemmas+ .entry(info.id.clone())+ .or_default()+ .push(normalized.clone());++ self.spellings.insert(normalized, lemma);+ }++ pub fn build<P: AsRef<Path>>(path: P) -> Result<Self> {+ let reader = BufReader::new(File::open(path)?);++ let mut parser = TurtleParser::new(reader, None);+ let mut graph = Graph::new();++ parser.parse_all(&mut |t| {+ let edge = Edge {+ source: Node(t.subject.to_string()),+ label: t.predicate.to_string(),+ target: Node(t.object.to_string()),+ };++ graph.insert(edge);++ Ok::<_, anyhow::Error>(())+ })?;++ let mut lexical_entries = Vec::new();++ for (node, edges) in &graph.map {+ if let Some(e) = edges.iter().find(|e| e.label.contains("#type")) {+ if e.target.0.contains("#LexicalEntry") {+ lexical_entries.push(node);+ }+ }+ }++ let mut dict = Self::empty()?;+ let value_str_regex = regex::Regex::new(r#""(.*)"@"#).unwrap();++ for entry in lexical_entries {+ let entry = NodeQuery::new(entry, &graph);++ let written_rep = entry+ .single_edge(|e| e.label.contains("#canonicalForm"))+ .unwrap()+ .to+ .single_edge(|e| e.label.contains("#writtenRep"))+ .unwrap()+ .to+ .node+ .0+ .clone();++ let written_rep = value_str_regex+ .captures(&written_rep)+ .unwrap()+ .get(1)+ .unwrap()+ .as_str()+ .to_string();++ let lemma = Lemma(written_rep);++ for sense in entry.filtered_edges(|e| e.label.contains("#sense")) {+ let concept = sense+ .to+ .single_edge(|e| e.label.contains("#isLexicalizedSenseOf"))+ .unwrap()+ .to;+ let id = Id(concept.node.0.clone());++ let definition = concept+ .single_edge(|e| e.label.contains("#definition"))+ .unwrap()+ .to+ .single_edge(|e| e.label.contains("#value"))+ .unwrap()+ .to+ .node+ .0+ .clone();++ let definition = value_str_regex+ .captures(&definition)+ .unwrap()+ .get(1)+ .unwrap()+ .as_str()+ .to_string();+ let definition = Definition(definition);++ let examples: Vec<_> = concept+ .filtered_edges(|e| e.label.contains("#example"))+ .into_iter()+ .map(|e| {+ e.to.single_edge(|e| e.label.contains("#value"))+ .unwrap()+ .to+ .node+ .0+ .clone()+ })+ .map(|e| {+ value_str_regex+ .captures(&e)+ .unwrap()+ .get(1)+ .unwrap()+ .as_str()+ .to_string()+ })+ .map(Example)+ .collect();++ let similar: Vec<_> = concept+ .filtered_edges(|e| e.label.contains("#similar"))+ .into_iter()+ .map(|e| Id(e.to.node.0.clone()))+ .collect();++ let pos = concept+ .single_edge(|e| e.label.contains("#partOfSpeech"))+ .unwrap()+ .to+ .node+ .0+ .split_once('#')+ .unwrap()+ .1+ .to_string()+ .replace('>', "");++ let pos = match pos.as_str() {+ "noun" => PartOfSpeech::Noun,+ "adjective_satellite" => PartOfSpeech::AdjectiveSatellite,+ "verb" => PartOfSpeech::Verb,+ "adverb" => PartOfSpeech::Adverb,+ "adjective" => PartOfSpeech::Adjective,+ _ => return Err(anyhow!("Unknown part of speech: {}", pos)),+ };++ let info = Info {+ id,+ definition,+ examples,+ similar,+ pos,+ };++ dict.insert(lemma.clone(), info);+ }+ }++ Ok(dict)+ }+}++#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]+pub struct WordMeaning {+ pub definition: Definition,+ pub examples: Vec<Example>,+ pub similar: Vec<Lemma>,+}++#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]+pub struct PartOfSpeechMeaning {+ pub pos: PartOfSpeech,+ pub meanings: Vec<WordMeaning>,+}++impl PartialEq for PartOfSpeechMeaning {+ fn eq(&self, other: &Self) -> bool {+ self.pos == other.pos+ }+}++impl Eq for PartOfSpeechMeaning {}++impl Ord for PartOfSpeechMeaning {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.pos.cmp(&other.pos)+ }+}++impl PartialOrd for PartOfSpeechMeaning {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}++#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]+pub struct ThesaurusWidget {+ pub term: Lemma,+ pub meanings: Vec<PartOfSpeechMeaning>,+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn build_dict() {+ let dict = Dictionary::build("../data/english-wordnet-2022-subset.ttl").unwrap();++ let infos = dict.get(Lemma("barely".to_string()));++ let definitions = infos+ .iter()+ .map(|i| i.definition.0.clone())+ .collect::<Vec<_>>();++ assert!(definitions.contains(&String::from("only a very short time before")));+ assert!(definitions.contains(&String::from("by a little")));+ assert!(definitions.contains(&String::from("almost not")));+ }+}diff --git a/core/src/widgets/calculator.lalrpop b/core/src/widgets/calculator.lalrpopdeleted file mode 100644index 6eef0641..00000000--- a/core/src/widgets/calculator.lalrpop+++ /dev/null@@ -1,33 +0,0 @@-use std::str::FromStr;-use crate::widgets::calculator::{Expr, Opcode};--grammar;--pub Expr: Box<Expr> = {- Expr ExprOp Factor => Box::new(Expr::Op(<>)),- Factor,-};--ExprOp: Opcode = {- "+" => Opcode::Add,- "-" => Opcode::Sub,-};--Factor: Box<Expr> = {- Factor FactorOp Term => Box::new(Expr::Op(<>)),- Term,-};--FactorOp: Opcode = {- "*" => Opcode::Mul,- "/" => Opcode::Div,-};--Term: Box<Expr> = {- Num => Box::new(Expr::Number(<>)),- "(" <Expr> ")"-};--Num: f64 = {- r"([0-9]*[.])?[0-9]+" => f64::from_str(<>).unwrap()-};\ No newline at end of filediff --git a/core/src/widgets/calculator.rs b/core/src/widgets/calculator.rsindex d1b4890d..216c1ea6 100644--- a/core/src/widgets/calculator.rs+++ b/core/src/widgets/calculator.rs@@ -21,7 +21,7 @@ use rand::Rng;use serde::{Deserialize, Serialize};use std::{fmt::Debug,- sync::{Arc, Mutex},+ sync::{atomic::AtomicUsize, Arc, Mutex},};use utoipa::ToSchema;@@ -63,6 +63,7 @@ async fn get_rates() -> Result<CurrencyExchange> {.find(|a| a.as_ref().unwrap().key == b"rate").map(|a| a.unwrap().value.to_vec()).unwrap();+rates.insert(String::from_utf8(currency)?,String::from_utf8(rate)?.parse::<f64>()?,@@ -105,6 +106,27 @@ pub enum ExchangeUpdate {AsyncTokio,}+struct MaxIterations {+ max_iterations: usize,+ iterations: AtomicUsize,+}+impl MaxIterations {+ fn new(max_iterations: usize) -> Self {+ Self {+ max_iterations,+ iterations: AtomicUsize::new(0),+ }+ }+}++impl fend_core::Interrupt for MaxIterations {+ fn should_interrupt(&self) -> bool {+ self.iterations+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed)+ > self.max_iterations+ }+}+pub struct Calculator {exchange: Arc<Mutex<Arc<CurrencyExchange>>>,}@@ -136,8 +158,9 @@ impl Calculator {}pub fn try_calculate(&self, expr: &str) -> Result<Calculation, Error> {+ let expr = expr.replace(['"', '\''], "");// if expr starts with "d[0-9]+", wrap it in "roll(...)"- let expr = if DICE_REGEX.is_match(expr) {+ let expr = if DICE_REGEX.is_match(&expr) {format!("roll({})", expr)} else {expr.to_string()@@ -154,7 +177,10 @@ impl Calculator {rng.gen()});- let res = fend_core::evaluate(&expr, &mut context).map_err(|_| Error::CalculatorParse)?;+ let interrupt = MaxIterations::new(32);++ let res = fend_core::evaluate_with_interrupt(&expr, &mut context, &interrupt)+ .map_err(|_| Error::CalculatorParse)?;if res.get_main_result() == expr {return Err(Error::CalculatorParse);diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rsindex 99d3852f..86328133 100644--- a/core/src/widgets/mod.rs+++ b/core/src/widgets/mod.rs@@ -18,7 +18,13 @@ use serde::{Deserialize, Serialize};use thiserror::Error;use utoipa::ToSchema;+use crate::{+ config::WidgetsConfig,+ thesaurus::{self, ThesaurusWidget},+};+use self::calculator::{Calculation, Calculator};+use anyhow::{anyhow, Result};pub mod calculator;@@ -30,13 +36,25 @@ pub enum Error {pub struct Widgets {calculator: Calculator,+ thesaurus: Option<thesaurus::Dictionary>,}impl Widgets {- pub fn new() -> Self {- Self {- calculator: Calculator::new(calculator::ExchangeUpdate::AsyncTokio),+ pub fn new(config: WidgetsConfig) -> Result<Self> {+ if config.thesaurus_paths.len() > 1 {+ return Err(anyhow!("Only one thesaurus path is supported for now"));}++ let thesaurus = if let Some(path) = config.thesaurus_paths.get(0) {+ Some(thesaurus::Dictionary::build(path)?)+ } else {+ None+ };++ Ok(Self {+ calculator: Calculator::new(calculator::ExchangeUpdate::AsyncTokio),+ thesaurus,+ })}pub fn widget(&self, query: &str) -> Option<Widget> {@@ -44,6 +62,12 @@ impl Widgets {.try_calculate(query).ok().map(Widget::Calculator)+ .or_else(|| {+ self.thesaurus+ .as_ref()+ .and_then(|thesaurus| thesaurus.lookup(query))+ .map(Widget::Thesaurus)+ })}}@@ -51,4 +75,5 @@ impl Widgets {#[serde(tag = "type", content = "value", rename_all = "camelCase")]pub enum Widget {Calculator(Calculation),+ Thesaurus(ThesaurusWidget),}diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex 4373f702..1de87444 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -146,6 +146,7 @@ export type Sidebar = components['schemas']['DisplayedSidebar'];export type Entity = components['schemas']['DisplayedEntity'];export type EntitySnippet = components['schemas']['EntitySnippet'];export type Widget = components['schemas']['Widget'];+export type ThesaurusWidget = components['schemas']['ThesaurusWidget'];export type DisplayedAnswer = components['schemas']['DisplayedAnswer'];export type ScoredSite = components['schemas']['ScoredSite'];export type Suggestion = components['schemas']['Suggestion'];diff --git a/frontend/src/lib/api/schema.d.ts b/frontend/src/lib/api/schema.d.tsindex 95a7d309..85735d40 100644--- a/frontend/src/lib/api/schema.d.ts+++ b/frontend/src/lib/api/schema.d.ts@@ -101,6 +101,7 @@ export interface components {type: "text";value: string;};+ Definition: string;DisplayedAnswer: {answer: string;prettyUrl: string;@@ -156,6 +157,7 @@ export interface components {kind: "link";text: string;};+ Example: string;ExecutionState: {query: string;/** @enum {string} */@@ -203,10 +205,17 @@ export interface components {/** @enum {string} */type: "unknown";};+ Lemma: string;Node: {name: string;};/** @enum {string} */+ PartOfSpeech: "noun" | "verb" | "adjective" | "adjectiveSatellite" | "adverb";+ PartOfSpeechMeaning: {+ meanings: components["schemas"]["WordMeaning"][];+ pos: components["schemas"]["PartOfSpeech"];+ };+ /** @enum {string} */Region: "All" | "Denmark" | "France" | "Germany" | "Spain" | "US";ScoredSite: {description?: string | null;@@ -271,6 +280,10 @@ export interface components {kind: components["schemas"]["TextSnippetFragmentKind"];text: string;};+ ThesaurusWidget: {+ meanings: components["schemas"]["PartOfSpeechMeaning"][];+ term: components["schemas"]["Lemma"];+ };/*** Url* @description Wrapper around `Url` that implements `ToSchema` for `Url`.@@ -291,6 +304,15 @@ export interface components {/** @enum {string} */type: "calculator";value: components["schemas"]["Calculation"];+ } | {+ /** @enum {string} */+ type: "thesaurus";+ value: components["schemas"]["ThesaurusWidget"];+ };+ WordMeaning: {+ definition: components["schemas"]["Definition"];+ examples: components["schemas"]["Example"][];+ similar: components["schemas"]["Lemma"][];};};responses: never;diff --git a/frontend/src/routes/explore/+page.svelte b/frontend/src/routes/explore/+page.svelteindex 37222ed8..f6000223 100644--- a/frontend/src/routes/explore/+page.svelte+++ b/frontend/src/routes/explore/+page.svelte@@ -16,7 +16,7 @@let inputWebsite = '';let limit = LIMIT_OPTIONS[0];- let chosenSites: string[] = ['facebook.com'];+ let chosenSites: string[] = [];let similarSites: ScoredSite[] = [];let errorMessage = false;diff --git a/frontend/src/routes/search/ThesaurusWidget.svelte b/frontend/src/routes/search/ThesaurusWidget.sveltenew file mode 100644index 00000000..bd789846--- /dev/null+++ b/frontend/src/routes/search/ThesaurusWidget.svelte@@ -0,0 +1,63 @@+<script lang="ts">+ import type { ThesaurusWidget } from '$lib/api';++ export let widget: ThesaurusWidget;++ const posName = (pos: "noun" | "verb" | "adjective" | "adjectiveSatellite" | "adverb"): String => {+ switch (pos) {+ case "noun":+ return "Noun";+ case "verb":+ return "Verb";+ case "adjective":+ return "Adjective";+ case "adjectiveSatellite":+ return "Adjective Satellite";+ case "adverb":+ return "Adverb";+ }+ }+</script>++<div class="rounded-xl border pl-5 pr-3 pt-5 pb-1">+ <div class="text-neutral-focus">+ <h2 class="font-bold text-2xl">{widget.term}</h2>+ <div class="flex flex-col space-y-2">+ {#each widget.meanings as {pos, meanings}}+ <div>+ <div class="italic text-sm">+ {posName(pos)}+ </div>+ <ol class="list-decimal space-y-2 pl-5">+ {#each meanings as meaning}+ <li>+ <div>+ {meaning.definition}+ </div>+ {#if meaning.similar.length > 0}+ <div class="flex space-x-1 text-sm">+ <div class="text-primary-focus font-medium">+ Similar:+ </div>+ <div class="inline-block space-x-1">+ {#each meaning.similar as similar}+ <div class="float-left inline [&:not(:last-child)]:after:content-[',']">+ <a class="float-left hover:underline" href="/search?q={encodeURIComponent('definition of ' + similar)}">+ {similar}+ </a>+ </div>+ {/each}+ </div>+ </div>+ {/if}+ </li>+ {/each}+ </ol>+ </div>+ {/each}+ </div>+ <div class="text-[0.75rem] italic text-neutral float-right mt-1">+ Data from <a href="https://en-word.net/" class="hover:underline">Open English WordNet</a> and <a href="https://wordnet.princeton.edu/" class="hover:underline">Princeton WordNet</a>+ </div>+ </div>+</div>\ No newline at end of filediff --git a/frontend/src/routes/search/Widget.svelte b/frontend/src/routes/search/Widget.svelteindex 656c8780..97986684 100644--- a/frontend/src/routes/search/Widget.svelte+++ b/frontend/src/routes/search/Widget.svelte@@ -1,11 +1,14 @@<script lang="ts">import type { Widget } from '$lib/api';+ import ThesaurusWidget from './ThesaurusWidget.svelte';export let widget: Widget;</script>+{#if widget.type == 'calculator'}- <div class="flex flex-col items-end rounded-xl border p-5">+<div class="rounded-xl border p-5">+ <div class="flex flex-col items-end"><div class="flex w-fit text-xs text-neutral">{widget.value.input} =</div>@@ -13,4 +16,7 @@{widget.value.result}</div></div>+</div>+{:else if widget.type == "thesaurus"}+<ThesaurusWidget widget={widget.value} />{/if}