web search done right
Ftr/lalrpop query parser (#16)
Mikkel Denker990dba7f86a7
diff --git a/Cargo.lock b/Cargo.lockindex 2ac2347d..234e1450 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -25,6 +25,15 @@ dependencies = ["version_check",]+[[package]]+name = "aho-corasick"+version = "0.7.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"+dependencies = [+ "memchr",+]+[[package]]name = "ansi_term"version = "0.12.1"@@ -40,6 +49,15 @@ version = "1.0.58"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704"+[[package]]+name = "ascii-canvas"+version = "3.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"+dependencies = [+ "term",+]+[[package]]name = "askama"version = "0.11.1"@@ -219,6 +237,21 @@ dependencies = ["shlex",]+[[package]]+name = "bit-set"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"+dependencies = [+ "bit-vec",+]++[[package]]+name = "bit-vec"+version = "0.6.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"+[[package]]name = "bit_field"version = "0.10.1"@@ -577,13 +610,15 @@ dependencies = ["image","indicatif","itertools",+ "lalrpop",+ "lalrpop-util","lazy_static","logos","lru","maplit","md5","memmap",- "nom",+ "once_cell","parse_wiki_text","quick-xml","rand",@@ -662,6 +697,15 @@ version = "1.6.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"+[[package]]+name = "ena"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"+dependencies = [+ "log",+]+[[package]]name = "encode_unicode"version = "0.3.6"@@ -729,6 +773,12 @@ dependencies = ["instant",]+[[package]]+name = "fixedbitset"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"+[[package]]name = "flate2"version = "1.0.24"@@ -1156,12 +1206,12 @@ dependencies = [[[package]]name = "indexmap"-version = "1.8.2"+version = "1.9.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a"+checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"dependencies = ["autocfg",- "hashbrown 0.11.2",+ "hashbrown 0.12.1",][[package]]@@ -1252,6 +1302,38 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "lalrpop"+version = "0.19.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823"+dependencies = [+ "ascii-canvas",+ "atty",+ "bit-set",+ "diff",+ "ena",+ "itertools",+ "lalrpop-util",+ "petgraph",+ "pico-args",+ "regex",+ "regex-syntax 0.6.26",+ "string_cache",+ "term",+ "tiny-keccak",+ "unicode-xid",+]++[[package]]+name = "lalrpop-util"+version = "0.19.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4"+dependencies = [+ "regex",+]+[[package]]name = "lazy_static"version = "1.4.0"@@ -1555,6 +1637,12 @@ dependencies = ["tempfile",]+[[package]]+name = "new_debug_unreachable"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"+[[package]]name = "nom"version = "7.1.1"@@ -1622,9 +1710,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"[[package]]name = "once_cell"-version = "1.12.0"+version = "1.13.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"+checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e"[[package]]name = "oneshot"@@ -1776,6 +1864,31 @@ version = "2.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"+[[package]]+name = "petgraph"+version = "0.6.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"+dependencies = [+ "fixedbitset",+ "indexmap",+]++[[package]]+name = "phf_shared"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"+dependencies = [+ "siphasher",+]++[[package]]+name = "pico-args"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"+[[package]]name = "pin-project"version = "1.0.10"@@ -1832,6 +1945,12 @@ version = "0.2.16"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"+[[package]]+name = "precomputed-hash"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"+[[package]]name = "pretty_assertions"version = "1.2.1"@@ -1975,6 +2094,8 @@ version = "1.5.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"dependencies = [+ "aho-corasick",+ "memchr","regex-syntax 0.6.26",]@@ -2310,6 +2431,12 @@ dependencies = ["libc",]+[[package]]+name = "siphasher"+version = "0.3.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"+[[package]]name = "slab"version = "0.4.6"@@ -2364,6 +2491,19 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"+[[package]]+name = "string_cache"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"+dependencies = [+ "new_debug_unreachable",+ "once_cell",+ "parking_lot 0.12.1",+ "phf_shared",+ "precomputed-hash",+]+[[package]]name = "strsim"version = "0.10.0"@@ -2496,6 +2636,17 @@ dependencies = ["winapi",]+[[package]]+name = "term"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"+dependencies = [+ "dirs-next",+ "rustversion",+ "winapi",+]+[[package]]name = "termcolor"version = "1.1.3"@@ -2593,6 +2744,15 @@ dependencies = ["serde",]+[[package]]+name = "tiny-keccak"+version = "2.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"+dependencies = [+ "crunchy",+]+[[package]]name = "tinyvec"version = "1.6.0"@@ -2830,6 +2990,12 @@ dependencies = ["tinyvec",]+[[package]]+name = "unicode-xid"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"+[[package]]name = "url"version = "2.2.2"diff --git a/Cargo.toml b/Cargo.tomlindex ae349940..278420a6 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -18,7 +18,6 @@ flate2 = "1.0.23"indicatif = { version = "0.16.2", features = ["rayon"] }itertools = "0.10.3"lru = "0.7.6"-nom = "7.1.1"rand = "0.8.5"rayon = "1.5.3"reqwest = { version = "0.11.10", features = ["blocking"] }@@ -53,9 +52,14 @@ bzip2 = "0.4.3"parse_wiki_text = "0.1.5"md5 = "0.7.0"memmap = "0.7.0"+lalrpop-util = { version = "0.19.8", features = ["lexer"] }+once_cell = "1.13.1"[dev-dependencies]maplit = "1.0.2"[profile.release]debug = true++[build-dependencies]+lalrpop = "0.19.8"diff --git a/build.rs b/build.rsindex 9f383a77..98dcc47d 100644--- a/build.rs+++ b/build.rs@@ -10,4 +10,6 @@ fn main() {println!("cargo:rerun-if-changed=yarn.lock");println!("cargo:rerun-if-changed=tailwind.config.js");println!("cargo:rerun-if-changed=postcss.config.js");++ lalrpop::process_root().unwrap();}diff --git a/data.tar.gz b/data.tar.gzindex 970d26d3..9a1e6ea4 100644--- a/data.tar.gz+++ b/data.tar.gz@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:2cabcd57754add6d968fb719d9c4e8a6d259a758995163027f7714ffec1e35d4-size 941862585+oid sha256:1cadc250585c97a68bdfcfda2e341c30235f3369b0c72f2ff898c3473f0417ea+size 968695137diff --git a/justfile b/justfileindex 18e4485c..800ca07e 100644--- a/justfile+++ b/justfile@@ -1,4 +1,5 @@@unpack-data:+ rm -rf datatar -zxvf data.tar.gz@flamegraph:diff --git a/src/entity_index/mod.rs b/src/entity_index/mod.rsindex 42481c0b..5d64e4cd 100644--- a/src/entity_index/mod.rs+++ b/src/entity_index/mod.rs@@ -26,6 +26,7 @@ use tantivy::{collector::TopDocs,query::{BooleanQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},schema::{BytesOptions, IndexRecordOption, Schema, TextFieldIndexing, TextOptions},+ tokenizer::Tokenizer,DocAddress, IndexReader, IndexWriter, LeasedItem, Searcher, Term,};use tracing::info;@@ -263,31 +264,31 @@ impl EntityIndex {let title = self.schema.get_field("title").unwrap();let entity_abstract = self.schema.get_field("abstract").unwrap();- let term_queries: Vec<(Occur, Box<dyn tantivy::query::Query>)> = query- .split(' ')- .map(|term| term.to_ascii_lowercase())- .filter(|term| !self.stopwords.contains(term))- .flat_map(|term| {- vec![- (- Occur::Must,- TermQuery::new(- Term::from_field_text(title, &term),- IndexRecordOption::WithFreqsAndPositions,- )- .box_clone(),- ),- (- Occur::Should,- TermQuery::new(- Term::from_field_text(entity_abstract, &term),- IndexRecordOption::WithFreqsAndPositions,- )- .box_clone(),- ),- ]- })- .collect();+ let mut term_queries = Vec::new();+ let mut stream = NormalTokenizer::default().token_stream(query.as_str());+ while let Some(token) = stream.next() {+ if self.stopwords.contains(&token.text) {+ continue;+ }++ term_queries.push((+ Occur::Must,+ TermQuery::new(+ Term::from_field_text(title, &token.text),+ IndexRecordOption::WithFreqsAndPositions,+ )+ .box_clone(),+ ));++ term_queries.push((+ Occur::Should,+ TermQuery::new(+ Term::from_field_text(entity_abstract, &token.text),+ IndexRecordOption::WithFreqsAndPositions,+ )+ .box_clone(),+ ));+ }let query = BooleanQuery::from(term_queries);diff --git a/src/kv/sled_store.rs b/src/kv/sled_store.rsindex 25e12aa3..b9d3c641 100644--- a/src/kv/sled_store.rs+++ b/src/kv/sled_store.rs@@ -16,7 +16,6 @@use std::marker::PhantomData;-use nom::AsBytes;use serde::{de::DeserializeOwned, Serialize};use crate::kv::Kv;@@ -31,7 +30,7 @@ wherefn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {self.get(key).expect("failed to retrieve key")- .map(|v| v.as_bytes().to_vec())+ .map(|v| v.to_vec())}fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {diff --git a/src/lib.rs b/src/lib.rsindex 97838062..86b90356 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -185,6 +185,15 @@ pub enum Error {#[error("Spell dictionary error")]Spell(#[from] crate::spell::dictionary::DictionaryError),++ #[error("Parser error")]+ Parse,+}++impl<L, T, E> From<lalrpop_util::ParseError<L, T, E>> for Error {+ fn from(_: lalrpop_util::ParseError<L, T, E>) -> Self {+ Error::Parse+ }}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/src/query/mod.rs b/src/query/mod.rsindex 34ea52df..03d58753 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -14,22 +14,19 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{query::weight::Weight, schema::ALL_FIELDS, Error, Result};-use nom::{- bytes::complete::{tag, take_while},- character::complete::multispace0,- error::ParseError,- multi::separated_list0,- sequence::preceded,- IResult,+use crate::{query::weight::Weight, schema::ALL_FIELDS, tokenizer::NormalTokenizer, Result};+use std::{+ collections::{BTreeMap, HashMap},+ sync::Arc,};-use std::{collections::BTreeMap, sync::Arc};use tantivy::{schema::{FieldType, IndexRecordOption, Schema},- tokenizer::TextAnalyzer,+ tokenizer::{TextAnalyzer, Tokenizer},};use self::bm25::Bm25Weight;+use lalrpop_util::lalrpop_mod;+lalrpop_mod!(pub parser, "/query/parser.rs"); // synthesized by LALRPOPmod bm25;mod field_union;@@ -37,22 +34,11 @@ mod term_intersection;mod term_scorer;mod vec_docset;mod weight;-use itertools::Itertools;--/// A combinator that takes a parser `inner` and produces a parser that also consumes leading-/// whitespace, returning the output of `inner`.-fn ws<'a, F: 'a, O, E: ParseError<&'a str>>(- inner: F,-) -> impl FnMut(&'a str) -> IResult<&'a str, O, E>-where- F: Fn(&'a str) -> IResult<&'a str, O, E>,-{- preceded(multispace0, inner)-}-fn term(term: &str) -> IResult<&str, &str> {- ws(take_while(|c: char| !c.is_whitespace()))(term)-}+static PARSER: once_cell::sync::Lazy<parser::TermsParser> =+ once_cell::sync::Lazy::new(parser::TermsParser::new);++const MAX_SIMILAR_TERMS: usize = 10;pub struct FieldData {tantivy: tantivy::schema::Field,@@ -69,6 +55,11 @@ impl std::fmt::Debug for FieldData {}}+#[derive(Debug)]+pub enum Term<'a> {+ Simple(&'a str),+}+#[derive(Clone, Debug)]pub struct Query {terms: Vec<String>,@@ -77,20 +68,34 @@ pub struct Query {impl Query {pub fn parse(query: &str, schema: Arc<Schema>) -> Result<Query> {- match separated_list0(tag(" "), term)(query) {- Ok((remaining, terms)) => {- debug_assert!(remaining.is_empty());-- let terms = terms- .into_iter()- .filter(|term| !term.is_empty())- .map(|term| term.to_string())- .unique()- .collect();- Ok(Query { terms, schema })+ let mut raw_terms = Vec::new();++ let terms = PARSER.parse(query)?;+ for term in terms {+ match term {+ Term::Simple(term) => {+ let mut stream = NormalTokenizer::default().token_stream(term);++ while let Some(token) = stream.next() {+ raw_terms.push(token.text.to_string());+ }+ }}- Err(error) => Err(Error::ParsingError(error.to_string())),}++ let mut term_count = HashMap::new();+ let mut terms = Vec::new();+ for term in raw_terms {+ let count = term_count.entry(term.clone()).or_insert(0);++ if *count < MAX_SIMILAR_TERMS {+ terms.push(term);+ }++ *count += 1;+ }++ Ok(Query { terms, schema })}pub fn simple_terms(&self) -> &[String] {@@ -175,18 +180,30 @@ mod tests {fn simple_parse() {let schema = Arc::new(create_schema());- let query = Query::parse("This is a simple query the the query the the", Arc::clone(&schema))- .expect("Failed to parse query");+ let query = Query::parse(+ "This is a simple query the the the the the the the the the the the the the",+ Arc::clone(&schema),+ )+ .expect("Failed to parse query");assert_eq!(query.terms,vec![- "This".to_string(),+ "this".to_string(),"is".to_string(),"a".to_string(),"simple".to_string(),"query".to_string(),"the".to_string(),+ "the".to_string(),+ "the".to_string(),+ "the".to_string(),+ "the".to_string(),+ "the".to_string(),+ "the".to_string(),+ "the".to_string(),+ "the".to_string(),+ "the".to_string(),]);}@@ -201,7 +218,7 @@ mod tests {assert_eq!(query.terms,vec![- "This".to_string(),+ "this".to_string(),"is".to_string(),"a".to_string(),"simple".to_string(),@@ -229,7 +246,13 @@ mod tests {.terms;assert_eq!(terms,- vec!["term!".to_string(), "term#".to_string(), "$".to_string()]+ vec![+ "term".to_string(),+ "!".to_string(),+ "term".to_string(),+ "#".to_string(),+ "$".to_string()+ ]);}}diff --git a/src/query/parser.lalrpop b/src/query/parser.lalrpopnew file mode 100644index 00000000..9ec70847--- /dev/null+++ b/src/query/parser.lalrpop@@ -0,0 +1,12 @@+use crate::query::Term;++grammar;+++pub Terms: Vec<Term<'input>> = <Term*> => <>;++Term: Term<'input> = {+ <BasicTerm> => Term::Simple(<>),+}++BasicTerm: &'input str = r"[\S]*" => <>;\ No newline at end of filediff --git a/src/query/term_intersection.rs b/src/query/term_intersection.rsindex 2f2a90e5..c9f96f8e 100644--- a/src/query/term_intersection.rs+++ b/src/query/term_intersection.rs@@ -20,8 +20,8 @@ use tantivy::{DocId, DocSet, Postings, Score, TERMINATED};use super::field_union::FieldUnion;-const MAX_WIDTH: u32 = 45;-const LAMBDA: f32 = 0.55;+const MAX_WIDTH: u32 = 25;+const LAMBDA: f32 = 0.75;const GAMMA: f32 = 0.25;pub fn intersect_terms(mut scorers: Vec<FieldUnion>) -> Box<dyn Scorer> {diff --git a/src/spell/dictionary.rs b/src/spell/dictionary.rsindex 74660a96..9ab1c6f2 100644--- a/src/spell/dictionary.rs+++ b/src/spell/dictionary.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::spell::distance::LevenshteinDistance;+use crate::tokenizer::NormalTokenizer;use crate::webpage::Webpage;use fst::map::Union;use fst::{Automaton, IntoStreamer, Map, MapBuilder, Streamer};@@ -24,6 +25,7 @@ use std::hash::{Hash, Hasher};use std::ops::AddAssign;use std::path::Path;use std::{cmp, io, mem};+use tantivy::tokenizer::Tokenizer;use thiserror::Error;pub trait EditStrategy: Send + Sync {@@ -332,8 +334,12 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {}pub fn insert_page(&mut self, webpage: &Webpage) {- for text in webpage.html.clean_text().unwrap_or_default().split(' ') {- self.insert(text);+ let text = webpage.html.clean_text().unwrap_or_default();++ let mut stream = NormalTokenizer::default().token_stream(text.as_str());++ while let Some(token) = stream.next() {+ self.insert(&token.text);}}@@ -445,7 +451,7 @@ mod tests {}#[test]- fn only_stop_top_n() {+ fn only_store_top_n() {let mut dict: Dictionary<2> = Dictionary::open::<&str>(None).unwrap();dict.insert("test");diff --git a/src/tokenizer.rs b/src/tokenizer.rsindex 7efd8991..092cdd23 100644--- a/src/tokenizer.rs+++ b/src/tokenizer.rs@@ -14,9 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use logos::{Lexer, Logos};use tantivy::tokenizer::{- Language, LowerCaser, SimpleTokenizer, Stemmer, StopWordFilter, TextAnalyzer,+ BoxTokenStream, Language, LowerCaser, Stemmer, StopWordFilter, TextAnalyzer,};+use whatlang::Lang;struct MyStemmer(Stemmer);@@ -140,3 +142,191 @@ impl tantivy::tokenizer::Tokenizer for StemmedTokenizer {}}}++#[derive(Logos, Debug, PartialEq)]+enum Token {+ #[regex("[\\w|\\p{Han}|\\p{Hiragana}|\\p{Katakana}|\\p{Cyrillic}|\\p{Arabic}]+")]+ Text,++ #[error]+ #[regex(r"[ \t\n\f]+", logos::skip)]+ Error,+}++#[derive(Clone)]+pub struct SimpleTokenizer;++pub struct SimpleTokenStream<'a> {+ lexer: Lexer<'a, Token>,+ token: Option<tantivy::tokenizer::Token>,+ next_position: usize,+}++impl tantivy::tokenizer::Tokenizer for SimpleTokenizer {+ fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {+ let lexer = Token::lexer(text);+ BoxTokenStream::from(SimpleTokenStream {+ lexer,+ token: None,+ next_position: 0,+ })+ }+}++impl<'a> tantivy::tokenizer::TokenStream for SimpleTokenStream<'a> {+ fn advance(&mut self) -> bool {+ self.token = self.lexer.next().map(|_| {+ let span = self.lexer.span();+ let pos = self.next_position;+ self.next_position += 1;+ tantivy::tokenizer::Token {+ offset_from: span.start,+ offset_to: span.end,+ position: pos,+ text: self.lexer.slice().to_string(),+ ..Default::default()+ }+ });++ self.token.is_some()+ }++ fn token(&self) -> &tantivy::tokenizer::Token {+ self.token.as_ref().unwrap()+ }++ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {+ self.token.as_mut().unwrap()+ }+}++#[cfg(test)]+mod tests {+ use tantivy::tokenizer::Tokenizer;++ use super::*;++ fn tokenize(s: &str) -> Vec<String> {+ let mut res = Vec::new();+ let mut stream = NormalTokenizer::default().token_stream(s);++ while let Some(token) = stream.next() {+ res.push(token.text.clone());+ }++ res+ }++ #[test]+ fn simple_tokenization() {+ assert_eq!(+ tokenize("this is a relatively simple123 test string"),+ vec![+ "this".to_string(),+ "is".to_string(),+ "a".to_string(),+ "relatively".to_string(),+ "simple123".to_string(),+ "test".to_string(),+ "string".to_string()+ ]+ );+ }++ #[test]+ fn special_character_tokenization() {+ assert_eq!(+ tokenize("example.com"),+ vec!["example".to_string(), ".".to_string(), "com".to_string(),]+ );+ assert_eq!(+ tokenize("example. com"),+ vec!["example".to_string(), ".".to_string(), "com".to_string(),]+ );+ assert_eq!(+ tokenize("example . com"),+ vec!["example".to_string(), ".".to_string(), "com".to_string(),]+ );++ assert_eq!(+ tokenize("a c++ blog post"),+ vec![+ "a".to_string(),+ "c".to_string(),+ "+".to_string(),+ "+".to_string(),+ "blog".to_string(),+ "post".to_string()+ ]+ );+ assert_eq!(+ tokenize("path/test"),+ vec!["path".to_string(), "/".to_string(), "test".to_string(),]+ );+ }++ #[test]+ fn han() {+ assert_eq!(+ tokenize("test 漢.com"),+ vec![+ "test".to_string(),+ "漢".to_string(),+ ".".to_string(),+ "com".to_string()+ ]+ );+ }++ #[test]+ fn hiragana() {+ assert_eq!(+ tokenize("test あ.com"),+ vec![+ "test".to_string(),+ "あ".to_string(),+ ".".to_string(),+ "com".to_string()+ ]+ );+ }++ #[test]+ fn katakana() {+ assert_eq!(+ tokenize("test ダ.com"),+ vec![+ "test".to_string(),+ "ダ".to_string(),+ ".".to_string(),+ "com".to_string()+ ]+ );+ }++ #[test]+ fn cyrillic() {+ assert_eq!(+ tokenize("test б.com"),+ vec![+ "test".to_string(),+ "б".to_string(),+ ".".to_string(),+ "com".to_string()+ ]+ );+ }++ #[test]+ fn arabic() {+ assert_eq!(+ tokenize("test ب.com"),+ vec![+ "test".to_string(),+ "ب".to_string(),+ ".".to_string(),+ "com".to_string()+ ]+ );+ }+}diff --git a/templates/entity.html b/templates/entity.htmlindex 068b3bd5..d4670703 100644--- a/templates/entity.html+++ b/templates/entity.html@@ -4,8 +4,8 @@{% match entity.image %}{% when Some(image) %}<div class="w-lg mb-3">- <a href="https://en.wikipedia.org/wiki/{{ entity.title }}">- <img class="h-full w-full rounded-full" src="/entity/image/{{ image }}" />+ <a href="https://en.wikipedia.org/wiki/{{ entity.title|urlencode }}">+ <img class="h-full w-full rounded-full" src="/entity/image/{{ image|urlencode }}" /></a></div>{% when None %}@@ -16,7 +16,8 @@<div class="text-sm">{{ entity.small_abstract }}<span class="italic">- source: <a class="text-blue-600" href="https://en.wikipedia.org/wiki/{{ entity.title }}">wikipedia</a>+ source: <a class="text-blue-600"+ href="https://en.wikipedia.org/wiki/{{ entity.title|urlencode }}">wikipedia</a></span></div>{% if !entity.info.is_empty() %}@@ -46,14 +47,15 @@{% match entity.image %}{% when Some(image) %}<div class="h-20 w-20 mb-3">- <a href="/search?q={{ entity.title }}">- <img class="h-full object-cover w-full rounded-full" src="/entity/image/{{ image }}" />+ <a href="/search?q={{ entity.title|urlencode }}">+ <img class="h-full object-cover w-full rounded-full"+ src="/entity/image/{{ image|urlencode }}" /></a></div>{% when None %}{% endmatch %}<div class="text-center">- <a href="/search?q={{ entity.title }}">+ <a href="/search?q={{ entity.title|urlencode }}">{{ entity.title }}</a></div>diff --git a/templates/search.html b/templates/search.htmlindex 9144010e..fa76e7f9 100644--- a/templates/search.html+++ b/templates/search.html@@ -19,7 +19,8 @@{% match spell_correction %}{% when Some with (correction) %}<div class="pl-3 pb-3 ">- Did you mean: <a class="italic title-link font-bold" href="/search?q={{ correction }}">{{ correction+ Did you mean: <a class="italic title-link font-bold" href="/search?q={{ correction|urlencode }}">{{+ correction}}</a></div>{% when None %}@@ -31,10 +32,11 @@<div class="flex flex-col w-full"><div class="text-sm mb-1 flex"><div class="flex items-center mr-2">- <img src="/favicons/{{ item.domain }}" width="13px" />+ <img src="/favicons/{{ item.domain|urlencode }}" width="13px" /></div><div>- <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url+ <a class="hover:no-underline text-gray-600" href="{{ item.url|urlencode }}">{{+ item.pretty_url}}</a></div></div>@@ -56,8 +58,8 @@<div class="h-20 w-20 pl-2 pt-1 pb-1">{% match item.primary_image_uuid %}{% when Some with (val) %}- <a href="{{ item.url }}">- <img class="h-full w-full object-cover rounded-full" src="/image/{{ val }}" />+ <a href="{{ item.url|urlencode }}">+ <img class="h-full w-full object-cover rounded-full" src="/image/{{ val|urlencode }}" /></a>{% when None %}{% endmatch %}