Skip to content

web search done right

Thesaurus widget and minor bugfixes to calculator.

Mikkel Denker1eea731d26c1
diff --git a/.gitignore b/.gitignore
index bc55d0f5..a76ab465 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ notes
ltr_scores.json
.venv
libtorch
+proptest-regressions
diff --git a/Cargo.lock b/Cargo.lock
index 83203a97..4dd9d8ec 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2871,6 +2871,18 @@ dependencies = [
"stable_deref_trait",
]
+[[package]]
+name = "oxilangtag"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d91edf4fbb970279443471345a4e8c491bf05bb283b3e6c88e4e606fd8c181b"
+
+[[package]]
+name = "oxiri"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb175ec8981211357b7b379869c2f8d555881c55ea62311428ec0de46d89bd5c"
+
[[package]]
name = "parking_lot"
version = "0.12.1"
@@ -3605,6 +3617,23 @@ dependencies = [
"thiserror",
]
+[[package]]
+name = "rio_api"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1924fa1f0e6d851f9b73b3c569e607c368a0d92995d99d563ad7bf1414696603"
+
+[[package]]
+name = "rio_turtle"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cec59971eafd99b9c7e3544bfcabafea81a7072ac51c9f46985ca0bd7ba6016"
+dependencies = [
+ "oxilangtag",
+ "oxiri",
+ "rio_api",
+]
+
[[package]]
name = "rkyv"
version = "0.7.42"
@@ -4280,6 +4309,8 @@ dependencies = [
"regex",
"reqwest",
"reqwest-eventsource",
+ "rio_api",
+ "rio_turtle",
"rkyv",
"robotstxt-with-cache",
"rocksdb",
diff --git a/Cargo.toml b/Cargo.toml
index 6d67b4ed..53464fac 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -43,6 +43,8 @@ chrono = { version = "0.4.23", features = ["serde"] }
uuid = "1.1.2"
futures = "0.3.21"
quick-xml = "0.23.0"
+rio_turtle = "0.8.4"
+rio_api = "0.8.4"
bzip2 = "0.4.4"
parse_wiki_text = "0.1.5"
md5 = "0.7.0"
diff --git a/configs/api.toml b/configs/api.toml
index 610e979a..b3113812 100644
--- a/configs/api.toml
+++ b/configs/api.toml
@@ -1,16 +1,19 @@
cluster_id = "dev_api"
-queries_csv_path = "data/queries_us.csv"
-host = "0.0.0.0:3000"
-prometheus_host = "0.0.0.0:3001"
-gossip_addr = "0.0.0.0:3005"
-gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]
crossencoder_model_path = "data/cross_encoder"
fact_check_model_path = "data/fact_model"
+gossip_addr = "0.0.0.0:3005"
+gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]
+host = "0.0.0.0:3000"
+prometheus_host = "0.0.0.0:3001"
+queries_csv_path = "data/queries_us.csv"
# lambda_model_path = "data/lambdamart.txt"
# qa_model_path = "data/qa_model"
bangs_path = "data/bangs.json"
summarizer_path = "data/summarizer"
[thresholds]
-stackoverflow = 0.0
entity_sidebar = 0.0
+stackoverflow = 0.0
+
+[widgets]
+thesaurus_paths = ["data/english-wordnet-2022-subset.ttl"]
diff --git a/core/Cargo.toml b/core/Cargo.toml
index f8c84f93..cf610dc0 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -100,6 +100,8 @@ rkyv = { workspace = true }
hashbrown = { workspace = true }
mime = { workspace = true }
fend-core = { workspace = true }
+rio_turtle = { workspace = true }
+rio_api = { workspace = true }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { workspace = true }
diff --git a/core/src/api/docs.rs b/core/src/api/docs.rs
index 324f0659..35f0a3c2 100644
--- a/core/src/api/docs.rs
+++ b/core/src/api/docs.rs
@@ -63,6 +63,14 @@ use utoipa_swagger_ui::SwaggerUi;
crate::widgets::Widget,
crate::widgets::calculator::Calculation,
+ crate::thesaurus::ThesaurusWidget,
+ crate::thesaurus::Lemma,
+ crate::thesaurus::WordMeaning,
+ crate::thesaurus::Definition,
+ crate::thesaurus::Example,
+ crate::thesaurus::PartOfSpeech,
+ crate::thesaurus::PartOfSpeechMeaning,
+
crate::ranking::signal::SignalScore,
crate::bangs::BangHit,
crate::bangs::Bang,
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 6bf2f241..c5f6c008 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -133,8 +133,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
lambda_model,
qa_model,
bangs,
- config.collector.clone(),
- config.thresholds.clone(),
+ config.clone(),
);
let state = Arc::new(State {
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index caaa7be2..92f7785e 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -206,6 +206,8 @@ pub struct ApiConfig {
#[serde(default)]
pub thresholds: ApiThresholds,
+
+ pub widgets: WidgetsConfig,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -354,3 +356,8 @@ pub struct WebgraphServerConfig {
#[serde(default = "defaults::WebgraphServer::max_similar_sites")]
pub max_similar_sites: usize,
}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct WidgetsConfig {
+ pub thesaurus_paths: Vec<String>,
+}
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 102dfda4..ae3dd8f9 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -41,6 +41,7 @@ fn download_files() {
"sample.warc.gz",
"bangs.json",
"content.rdf.u8.gz",
+ "english-wordnet-2022-subset.ttl",
] {
info!("Downloading {}", name);
let body = reqwest::get(format!("http://s3.trystract.com/{BUCKET_NAME}/{name}"))
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index f680c98d..2243177c 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -405,8 +405,8 @@ impl Indexer {
index = index.merge(other);
std::fs::remove_dir_all(other_path).unwrap();
- index.inverted_index.merge_into_max_segments(1).unwrap();
}
+ index.inverted_index.merge_into_max_segments(1).unwrap();
index
}));
diff --git a/core/src/lib.rs b/core/src/lib.rs
index b2e94909..493cecd7 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -73,6 +73,7 @@ mod snippet;
pub mod spell;
mod subdomain_count;
pub mod summarizer;
+mod thesaurus;
mod tokenizer;
#[allow(unused)]
mod ttl_cache;
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 47f0ace7..d5811bb3 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -417,7 +417,9 @@ pub fn parse(query: &str) -> Vec<Box<Term>> {
}
if cur_term_begin < query.len() {
- res.push(parse_term(&query[cur_term_begin..query.len()]));
+ res.push(parse_term(
+ &query[floor_char_boundary(&query, cur_term_begin)..query.len()],
+ ));
}
res
@@ -426,6 +428,7 @@ pub fn parse(query: &str) -> Vec<Box<Term>> {
#[cfg(test)]
mod tests {
use super::*;
+ use proptest::prelude::*;
#[test]
fn parse_not() {
@@ -551,4 +554,18 @@ mod tests {
]
);
}
+
+ #[test]
+ fn unicode() {
+ let query = "\u{a0}";
+ assert_eq!(parse(query).len(), 1);
+ }
+
+ proptest! {
+ #[test]
+ fn prop(query: String) {
+ let res = parse(&query);
+ prop_assert_eq!(query.is_empty() || query.chars().all(|c| c.is_whitespace()), res.is_empty());
+ }
+ }
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 6998960f..82027f54 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -590,7 +590,6 @@ impl Field {
| Field::Text(TextField::TitleIfHomepage)
| Field::Text(TextField::SiteWithout) // will match url
| Field::Text(TextField::Domain) // will match url
- | Field::Text(TextField::CleanBody) // will match all_body
) && !self.is_fast()
}
diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rs
index b16afa4e..9320f6bc 100644
--- a/core/src/searcher/api.rs
+++ b/core/src/searcher/api.rs
@@ -25,7 +25,7 @@ use optics::Optic;
use url::Url;
use crate::bangs::{Bang, BangHit};
-use crate::config::{ApiThresholds, CollectorConfig};
+use crate::config::{ApiConfig, ApiThresholds, CollectorConfig};
use crate::inverted_index::RetrievedWebpage;
use crate::ranking::ALL_SIGNALS;
use crate::search_prettifier::{
@@ -68,8 +68,7 @@ impl ApiSearcher {
lambda_model: Option<LambdaMART>,
qa_model: Option<QaModel>,
bangs: Bangs,
- collector_config: CollectorConfig,
- thresholds: ApiThresholds,
+ config: ApiConfig,
) -> Self {
Self {
distributed_searcher: DistributedSearcher::new(cluster),
@@ -77,9 +76,9 @@ impl ApiSearcher {
lambda_model: lambda_model.map(Arc::new),
qa_model: qa_model.map(Arc::new),
bangs,
- collector_config,
- thresholds,
- widgets: Widgets::new(),
+ collector_config: config.collector,
+ thresholds: config.thresholds,
+ widgets: Widgets::new(config.widgets).unwrap(),
}
}
diff --git a/core/src/thesaurus.rs b/core/src/thesaurus.rs
new file mode 100644
index 00000000..51f467fa
--- /dev/null
+++ b/core/src/thesaurus.rs
@@ -0,0 +1,487 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use itertools::Itertools;
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+use std::{fs::File, io::BufReader, path::Path};
+use utoipa::ToSchema;
+
+use anyhow::{anyhow, Result};
+use rio_api::parser::TriplesParser;
+use rio_turtle::TurtleParser;
+use std::collections::HashMap;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase", transparent)]
+pub struct Lemma(String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct NormalizedLemma(String);
+
+impl Lemma {
+ fn normalize(&self) -> NormalizedLemma {
+ NormalizedLemma(self.0.to_lowercase())
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct Id(String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase", transparent)]
+pub struct Definition(String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase", transparent)]
+pub struct Example(String);
+
+#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub enum PartOfSpeech {
+ Noun,
+ Verb,
+ Adjective,
+ AdjectiveSatellite,
+ Adverb,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Info {
+ id: Id,
+ definition: Definition,
+ examples: Vec<Example>,
+ similar: Vec<Id>,
+ pos: PartOfSpeech,
+}
+
+pub struct Dictionary {
+ map: HashMap<Id, Info>,
+ lemmas: HashMap<NormalizedLemma, Vec<Id>>,
+ reverse_lemmas: HashMap<Id, Vec<NormalizedLemma>>,
+ spellings: HashMap<NormalizedLemma, Lemma>,
+
+ matchers: Vec<Regex>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct Node(String);
+
+#[derive(Debug)]
+struct Edge {
+ source: Node,
+ label: String,
+ target: Node,
+}
+
+#[derive(Debug)]
+struct Graph {
+ map: HashMap<Node, Vec<Edge>>,
+}
+
+impl Graph {
+ pub fn new() -> Self {
+ Self {
+ map: HashMap::new(),
+ }
+ }
+
+ pub fn insert(&mut self, edge: Edge) {
+ self.map.entry(edge.source.clone()).or_default().push(edge);
+ }
+
+ pub fn get(&self, node: &Node) -> Option<&Vec<Edge>> {
+ self.map.get(node)
+ }
+}
+
+struct EdgeQuery<'a> {
+ to: NodeQuery<'a>,
+ label: &'a str,
+}
+
+struct NodeQuery<'a> {
+ node: &'a Node,
+ graph: &'a Graph,
+}
+
+impl<'a> NodeQuery<'a> {
+ pub fn new(node: &'a Node, graph: &'a Graph) -> Self {
+ Self { node, graph }
+ }
+
+ pub fn edges(&self) -> Vec<EdgeQuery<'a>> {
+ self.graph
+ .get(self.node)
+ .map(|edges| {
+ edges
+ .iter()
+ .map(|e| EdgeQuery {
+ to: NodeQuery::new(&e.target, self.graph),
+ label: &e.label,
+ })
+ .collect()
+ })
+ .unwrap_or_default()
+ }
+
+ pub fn filtered_edges<F>(&self, f: F) -> Vec<EdgeQuery<'a>>
+ where
+ F: Fn(&EdgeQuery<'a>) -> bool,
+ {
+ self.edges().into_iter().filter(f).collect()
+ }
+
+ pub fn single_edge<F>(&self, f: F) -> Option<EdgeQuery<'a>>
+ where
+ F: Fn(&EdgeQuery<'a>) -> bool,
+ {
+ self.filtered_edges(f).into_iter().next()
+ }
+}
+
+impl<'a> Clone for NodeQuery<'a> {
+ fn clone(&self) -> Self {
+ Self {
+ node: self.node,
+ graph: self.graph,
+ }
+ }
+}
+
+impl Dictionary {
+ fn empty() -> Result<Self> {
+ Ok(Self {
+ map: HashMap::new(),
+ lemmas: HashMap::new(),
+ reverse_lemmas: HashMap::new(),
+ spellings: HashMap::new(),
+ matchers: vec![
+ Regex::new(r"^definition of ([\w| ]+)$")?,
+ Regex::new(r"^meaning of ([\w| ]+)$")?,
+ Regex::new(r"^synonym of ([\w| ]+)$")?,
+ Regex::new(r"^synonyms of ([\w| ]+)$")?,
+ //
+ Regex::new(r"^definition ([\w| ]+)$")?,
+ Regex::new(r"^meaning ([\w| ]+)$")?,
+ Regex::new(r"^synonym ([\w| ]+)$")?,
+ Regex::new(r"^synonyms ([\w| ]+)$")?,
+ //
+ Regex::new(r"^([\w| ]+) definition$")?,
+ Regex::new(r"^([\w| ]+) meaning$")?,
+ Regex::new(r"^([\w| ]+) synonym$")?,
+ Regex::new(r"^([\w| ]+) synonyms$")?,
+ ],
+ })
+ }
+
+ fn get_by_id(&self, id: Id) -> Option<&Info> {
+ self.map.get(&id)
+ }
+
+ fn get(&self, lemma: Lemma) -> Vec<Info> {
+ match self.lemmas.get(&lemma.normalize()) {
+ Some(ids) => ids
+ .iter()
+ .filter_map(|id| self.get_by_id(id.clone()))
+ .cloned()
+ .collect(),
+ None => Vec::new(),
+ }
+ }
+
+ fn query_lemma(&self, query: &str) -> Option<String> {
+ for matcher in &self.matchers {
+ if let Some(captures) = matcher.captures(query) {
+ return captures.get(1).map(|m| m.as_str().to_string());
+ }
+ }
+
+ None
+ }
+
+ pub fn lookup(&self, query: &str) -> Option<ThesaurusWidget> {
+ let query = self.query_lemma(query)?;
+ let lemma = Lemma(query);
+
+ let infos = self.get(lemma.clone());
+
+ if infos.is_empty() {
+ return None;
+ }
+
+ let mut meanings: HashMap<PartOfSpeech, Vec<WordMeaning>> = HashMap::new();
+
+ for info in infos {
+ meanings
+ .entry(info.pos.clone())
+ .or_default()
+ .push(WordMeaning {
+ definition: info.definition.clone(),
+ examples: info.examples.clone(),
+ similar: info
+ .similar
+ .iter()
+ .filter_map(|id| self.get_by_id(id.clone()))
+ .filter_map(|i| self.reverse_lemmas.get(&i.id))
+ .flatten()
+ .filter_map(|l| self.spellings.get(l))
+ .filter(|&l| l != &lemma)
+ .cloned()
+ .dedup_by(|a, b| a == b)
+ .collect(),
+ });
+ }
+
+ Some(ThesaurusWidget {
+ term: lemma,
+ meanings: meanings
+ .into_iter()
+ .map(|(pos, meanings)| PartOfSpeechMeaning { pos, meanings })
+ .sorted()
+ .collect(),
+ })
+ }
+
+ pub fn insert(&mut self, lemma: Lemma, info: Info) {
+ let normalized = lemma.clone().normalize();
+
+ self.map.insert(info.id.clone(), info.clone());
+
+ self.lemmas
+ .entry(normalized.clone())
+ .or_default()
+ .push(info.id.clone());
+
+ self.reverse_lemmas
+ .entry(info.id.clone())
+ .or_default()
+ .push(normalized.clone());
+
+ self.spellings.insert(normalized, lemma);
+ }
+
+ pub fn build<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let reader = BufReader::new(File::open(path)?);
+
+ let mut parser = TurtleParser::new(reader, None);
+ let mut graph = Graph::new();
+
+ parser.parse_all(&mut |t| {
+ let edge = Edge {
+ source: Node(t.subject.to_string()),
+ label: t.predicate.to_string(),
+ target: Node(t.object.to_string()),
+ };
+
+ graph.insert(edge);
+
+ Ok::<_, anyhow::Error>(())
+ })?;
+
+ let mut lexical_entries = Vec::new();
+
+ for (node, edges) in &graph.map {
+ if let Some(e) = edges.iter().find(|e| e.label.contains("#type")) {
+ if e.target.0.contains("#LexicalEntry") {
+ lexical_entries.push(node);
+ }
+ }
+ }
+
+ let mut dict = Self::empty()?;
+ let value_str_regex = regex::Regex::new(r#""(.*)"@"#).unwrap();
+
+ for entry in lexical_entries {
+ let entry = NodeQuery::new(entry, &graph);
+
+ let written_rep = entry
+ .single_edge(|e| e.label.contains("#canonicalForm"))
+ .unwrap()
+ .to
+ .single_edge(|e| e.label.contains("#writtenRep"))
+ .unwrap()
+ .to
+ .node
+ .0
+ .clone();
+
+ let written_rep = value_str_regex
+ .captures(&written_rep)
+ .unwrap()
+ .get(1)
+ .unwrap()
+ .as_str()
+ .to_string();
+
+ let lemma = Lemma(written_rep);
+
+ for sense in entry.filtered_edges(|e| e.label.contains("#sense")) {
+ let concept = sense
+ .to
+ .single_edge(|e| e.label.contains("#isLexicalizedSenseOf"))
+ .unwrap()
+ .to;
+ let id = Id(concept.node.0.clone());
+
+ let definition = concept
+ .single_edge(|e| e.label.contains("#definition"))
+ .unwrap()
+ .to
+ .single_edge(|e| e.label.contains("#value"))
+ .unwrap()
+ .to
+ .node
+ .0
+ .clone();
+
+ let definition = value_str_regex
+ .captures(&definition)
+ .unwrap()
+ .get(1)
+ .unwrap()
+ .as_str()
+ .to_string();
+ let definition = Definition(definition);
+
+ let examples: Vec<_> = concept
+ .filtered_edges(|e| e.label.contains("#example"))
+ .into_iter()
+ .map(|e| {
+ e.to.single_edge(|e| e.label.contains("#value"))
+ .unwrap()
+ .to
+ .node
+ .0
+ .clone()
+ })
+ .map(|e| {
+ value_str_regex
+ .captures(&e)
+ .unwrap()
+ .get(1)
+ .unwrap()
+ .as_str()
+ .to_string()
+ })
+ .map(Example)
+ .collect();
+
+ let similar: Vec<_> = concept
+ .filtered_edges(|e| e.label.contains("#similar"))
+ .into_iter()
+ .map(|e| Id(e.to.node.0.clone()))
+ .collect();
+
+ let pos = concept
+ .single_edge(|e| e.label.contains("#partOfSpeech"))
+ .unwrap()
+ .to
+ .node
+ .0
+ .split_once('#')
+ .unwrap()
+ .1
+ .to_string()
+ .replace('>', "");
+
+ let pos = match pos.as_str() {
+ "noun" => PartOfSpeech::Noun,
+ "adjective_satellite" => PartOfSpeech::AdjectiveSatellite,
+ "verb" => PartOfSpeech::Verb,
+ "adverb" => PartOfSpeech::Adverb,
+ "adjective" => PartOfSpeech::Adjective,
+ _ => return Err(anyhow!("Unknown part of speech: {}", pos)),
+ };
+
+ let info = Info {
+ id,
+ definition,
+ examples,
+ similar,
+ pos,
+ };
+
+ dict.insert(lemma.clone(), info);
+ }
+ }
+
+ Ok(dict)
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct WordMeaning {
+ pub definition: Definition,
+ pub examples: Vec<Example>,
+ pub similar: Vec<Lemma>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct PartOfSpeechMeaning {
+ pub pos: PartOfSpeech,
+ pub meanings: Vec<WordMeaning>,
+}
+
+impl PartialEq for PartOfSpeechMeaning {
+ fn eq(&self, other: &Self) -> bool {
+ self.pos == other.pos
+ }
+}
+
+impl Eq for PartOfSpeechMeaning {}
+
+impl Ord for PartOfSpeechMeaning {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.pos.cmp(&other.pos)
+ }
+}
+
+impl PartialOrd for PartOfSpeechMeaning {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ThesaurusWidget {
+ pub term: Lemma,
+ pub meanings: Vec<PartOfSpeechMeaning>,
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn build_dict() {
+ let dict = Dictionary::build("../data/english-wordnet-2022-subset.ttl").unwrap();
+
+ let infos = dict.get(Lemma("barely".to_string()));
+
+ let definitions = infos
+ .iter()
+ .map(|i| i.definition.0.clone())
+ .collect::<Vec<_>>();
+
+ assert!(definitions.contains(&String::from("only a very short time before")));
+ assert!(definitions.contains(&String::from("by a little")));
+ assert!(definitions.contains(&String::from("almost not")));
+ }
+}
diff --git a/core/src/widgets/calculator.lalrpop b/core/src/widgets/calculator.lalrpop
deleted file mode 100644
index 6eef0641..00000000
--- a/core/src/widgets/calculator.lalrpop
+++ /dev/null
@@ -1,33 +0,0 @@
-use std::str::FromStr;
-use crate::widgets::calculator::{Expr, Opcode};
-
-grammar;
-
-pub Expr: Box<Expr> = {
- Expr ExprOp Factor => Box::new(Expr::Op(<>)),
- Factor,
-};
-
-ExprOp: Opcode = {
- "+" => Opcode::Add,
- "-" => Opcode::Sub,
-};
-
-Factor: Box<Expr> = {
- Factor FactorOp Term => Box::new(Expr::Op(<>)),
- Term,
-};
-
-FactorOp: Opcode = {
- "*" => Opcode::Mul,
- "/" => Opcode::Div,
-};
-
-Term: Box<Expr> = {
- Num => Box::new(Expr::Number(<>)),
- "(" <Expr> ")"
-};
-
-Num: f64 = {
- r"([0-9]*[.])?[0-9]+" => f64::from_str(<>).unwrap()
-};
\ No newline at end of file
diff --git a/core/src/widgets/calculator.rs b/core/src/widgets/calculator.rs
index d1b4890d..216c1ea6 100644
--- a/core/src/widgets/calculator.rs
+++ b/core/src/widgets/calculator.rs
@@ -21,7 +21,7 @@ use rand::Rng;
use serde::{Deserialize, Serialize};
use std::{
fmt::Debug,
- sync::{Arc, Mutex},
+ sync::{atomic::AtomicUsize, Arc, Mutex},
};
use utoipa::ToSchema;
@@ -63,6 +63,7 @@ async fn get_rates() -> Result<CurrencyExchange> {
.find(|a| a.as_ref().unwrap().key == b"rate")
.map(|a| a.unwrap().value.to_vec())
.unwrap();
+
rates.insert(
String::from_utf8(currency)?,
String::from_utf8(rate)?.parse::<f64>()?,
@@ -105,6 +106,27 @@ pub enum ExchangeUpdate {
AsyncTokio,
}
+struct MaxIterations {
+ max_iterations: usize,
+ iterations: AtomicUsize,
+}
+impl MaxIterations {
+ fn new(max_iterations: usize) -> Self {
+ Self {
+ max_iterations,
+ iterations: AtomicUsize::new(0),
+ }
+ }
+}
+
+impl fend_core::Interrupt for MaxIterations {
+ fn should_interrupt(&self) -> bool {
+ self.iterations
+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
+ > self.max_iterations
+ }
+}
+
pub struct Calculator {
exchange: Arc<Mutex<Arc<CurrencyExchange>>>,
}
@@ -136,8 +158,9 @@ impl Calculator {
}
pub fn try_calculate(&self, expr: &str) -> Result<Calculation, Error> {
+ let expr = expr.replace(['"', '\''], "");
// if expr starts with "d[0-9]+", wrap it in "roll(...)"
- let expr = if DICE_REGEX.is_match(expr) {
+ let expr = if DICE_REGEX.is_match(&expr) {
format!("roll({})", expr)
} else {
expr.to_string()
@@ -154,7 +177,10 @@ impl Calculator {
rng.gen()
});
- let res = fend_core::evaluate(&expr, &mut context).map_err(|_| Error::CalculatorParse)?;
+ let interrupt = MaxIterations::new(32);
+
+ let res = fend_core::evaluate_with_interrupt(&expr, &mut context, &interrupt)
+ .map_err(|_| Error::CalculatorParse)?;
if res.get_main_result() == expr {
return Err(Error::CalculatorParse);
diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rs
index 99d3852f..86328133 100644
--- a/core/src/widgets/mod.rs
+++ b/core/src/widgets/mod.rs
@@ -18,7 +18,13 @@ use serde::{Deserialize, Serialize};
use thiserror::Error;
use utoipa::ToSchema;
+use crate::{
+ config::WidgetsConfig,
+ thesaurus::{self, ThesaurusWidget},
+};
+
use self::calculator::{Calculation, Calculator};
+use anyhow::{anyhow, Result};
pub mod calculator;
@@ -30,13 +36,25 @@ pub enum Error {
pub struct Widgets {
calculator: Calculator,
+ thesaurus: Option<thesaurus::Dictionary>,
}
impl Widgets {
- pub fn new() -> Self {
- Self {
- calculator: Calculator::new(calculator::ExchangeUpdate::AsyncTokio),
+ pub fn new(config: WidgetsConfig) -> Result<Self> {
+ if config.thesaurus_paths.len() > 1 {
+ return Err(anyhow!("Only one thesaurus path is supported for now"));
}
+
+ let thesaurus = if let Some(path) = config.thesaurus_paths.get(0) {
+ Some(thesaurus::Dictionary::build(path)?)
+ } else {
+ None
+ };
+
+ Ok(Self {
+ calculator: Calculator::new(calculator::ExchangeUpdate::AsyncTokio),
+ thesaurus,
+ })
}
pub fn widget(&self, query: &str) -> Option<Widget> {
@@ -44,6 +62,12 @@ impl Widgets {
.try_calculate(query)
.ok()
.map(Widget::Calculator)
+ .or_else(|| {
+ self.thesaurus
+ .as_ref()
+ .and_then(|thesaurus| thesaurus.lookup(query))
+ .map(Widget::Thesaurus)
+ })
}
}
@@ -51,4 +75,5 @@ impl Widgets {
#[serde(tag = "type", content = "value", rename_all = "camelCase")]
pub enum Widget {
Calculator(Calculation),
+ Thesaurus(ThesaurusWidget),
}
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index 4373f702..1de87444 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -146,6 +146,7 @@ export type Sidebar = components['schemas']['DisplayedSidebar'];
export type Entity = components['schemas']['DisplayedEntity'];
export type EntitySnippet = components['schemas']['EntitySnippet'];
export type Widget = components['schemas']['Widget'];
+export type ThesaurusWidget = components['schemas']['ThesaurusWidget'];
export type DisplayedAnswer = components['schemas']['DisplayedAnswer'];
export type ScoredSite = components['schemas']['ScoredSite'];
export type Suggestion = components['schemas']['Suggestion'];
diff --git a/frontend/src/lib/api/schema.d.ts b/frontend/src/lib/api/schema.d.ts
index 95a7d309..85735d40 100644
--- a/frontend/src/lib/api/schema.d.ts
+++ b/frontend/src/lib/api/schema.d.ts
@@ -101,6 +101,7 @@ export interface components {
type: "text";
value: string;
};
+ Definition: string;
DisplayedAnswer: {
answer: string;
prettyUrl: string;
@@ -156,6 +157,7 @@ export interface components {
kind: "link";
text: string;
};
+ Example: string;
ExecutionState: {
query: string;
/** @enum {string} */
@@ -203,10 +205,17 @@ export interface components {
/** @enum {string} */
type: "unknown";
};
+ Lemma: string;
Node: {
name: string;
};
/** @enum {string} */
+ PartOfSpeech: "noun" | "verb" | "adjective" | "adjectiveSatellite" | "adverb";
+ PartOfSpeechMeaning: {
+ meanings: components["schemas"]["WordMeaning"][];
+ pos: components["schemas"]["PartOfSpeech"];
+ };
+ /** @enum {string} */
Region: "All" | "Denmark" | "France" | "Germany" | "Spain" | "US";
ScoredSite: {
description?: string | null;
@@ -271,6 +280,10 @@ export interface components {
kind: components["schemas"]["TextSnippetFragmentKind"];
text: string;
};
+ ThesaurusWidget: {
+ meanings: components["schemas"]["PartOfSpeechMeaning"][];
+ term: components["schemas"]["Lemma"];
+ };
/**
* Url
* @description Wrapper around `Url` that implements `ToSchema` for `Url`.
@@ -291,6 +304,15 @@ export interface components {
/** @enum {string} */
type: "calculator";
value: components["schemas"]["Calculation"];
+ } | {
+ /** @enum {string} */
+ type: "thesaurus";
+ value: components["schemas"]["ThesaurusWidget"];
+ };
+ WordMeaning: {
+ definition: components["schemas"]["Definition"];
+ examples: components["schemas"]["Example"][];
+ similar: components["schemas"]["Lemma"][];
};
};
responses: never;
diff --git a/frontend/src/routes/explore/+page.svelte b/frontend/src/routes/explore/+page.svelte
index 37222ed8..f6000223 100644
--- a/frontend/src/routes/explore/+page.svelte
+++ b/frontend/src/routes/explore/+page.svelte
@@ -16,7 +16,7 @@
let inputWebsite = '';
let limit = LIMIT_OPTIONS[0];
- let chosenSites: string[] = ['facebook.com'];
+ let chosenSites: string[] = [];
let similarSites: ScoredSite[] = [];
let errorMessage = false;
diff --git a/frontend/src/routes/search/ThesaurusWidget.svelte b/frontend/src/routes/search/ThesaurusWidget.svelte
new file mode 100644
index 00000000..bd789846
--- /dev/null
+++ b/frontend/src/routes/search/ThesaurusWidget.svelte
@@ -0,0 +1,63 @@
+<script lang="ts">
+ import type { ThesaurusWidget } from '$lib/api';
+
+ export let widget: ThesaurusWidget;
+
+ const posName = (pos: "noun" | "verb" | "adjective" | "adjectiveSatellite" | "adverb"): String => {
+ switch (pos) {
+ case "noun":
+ return "Noun";
+ case "verb":
+ return "Verb";
+ case "adjective":
+ return "Adjective";
+ case "adjectiveSatellite":
+ return "Adjective Satellite";
+ case "adverb":
+ return "Adverb";
+ }
+ }
+</script>
+
+<div class="rounded-xl border pl-5 pr-3 pt-5 pb-1">
+ <div class="text-neutral-focus">
+ <h2 class="font-bold text-2xl">{widget.term}</h2>
+ <div class="flex flex-col space-y-2">
+ {#each widget.meanings as {pos, meanings}}
+ <div>
+ <div class="italic text-sm">
+ {posName(pos)}
+ </div>
+ <ol class="list-decimal space-y-2 pl-5">
+ {#each meanings as meaning}
+ <li>
+ <div>
+ {meaning.definition}
+ </div>
+ {#if meaning.similar.length > 0}
+ <div class="flex space-x-1 text-sm">
+ <div class="text-primary-focus font-medium">
+ Similar:
+ </div>
+ <div class="inline-block space-x-1">
+ {#each meaning.similar as similar}
+ <div class="float-left inline [&:not(:last-child)]:after:content-[',']">
+ <a class="float-left hover:underline" href="/search?q={encodeURIComponent('definition of ' + similar)}">
+ {similar}
+ </a>
+ </div>
+ {/each}
+ </div>
+ </div>
+ {/if}
+ </li>
+ {/each}
+ </ol>
+ </div>
+ {/each}
+ </div>
+ <div class="text-[0.75rem] italic text-neutral float-right mt-1">
+ Data from <a href="https://en-word.net/" class="hover:underline">Open English WordNet</a> and <a href="https://wordnet.princeton.edu/" class="hover:underline">Princeton WordNet</a>
+ </div>
+ </div>
+</div>
\ No newline at end of file
diff --git a/frontend/src/routes/search/Widget.svelte b/frontend/src/routes/search/Widget.svelte
index 656c8780..97986684 100644
--- a/frontend/src/routes/search/Widget.svelte
+++ b/frontend/src/routes/search/Widget.svelte
@@ -1,11 +1,14 @@
<script lang="ts">
import type { Widget } from '$lib/api';
+ import ThesaurusWidget from './ThesaurusWidget.svelte';
export let widget: Widget;
</script>
+
{#if widget.type == 'calculator'}
- <div class="flex flex-col items-end rounded-xl border p-5">
+<div class="rounded-xl border p-5">
+ <div class="flex flex-col items-end">
<div class="flex w-fit text-xs text-neutral">
{widget.value.input} =
</div>
@@ -13,4 +16,7 @@
{widget.value.result}
</div>
</div>
+</div>
+{:else if widget.type == "thesaurus"}
+<ThesaurusWidget widget={widget.value} />
{/if}