Skip to content

web search done right

Ftr/lalrpop query parser (#16)

Mikkel Denker990dba7f86a7
diff --git a/Cargo.lock b/Cargo.lock
index 2ac2347d..234e1450 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -25,6 +25,15 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "aho-corasick"
+version = "0.7.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "ansi_term"
version = "0.12.1"
@@ -40,6 +49,15 @@ version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704"
+[[package]]
+name = "ascii-canvas"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"
+dependencies = [
+ "term",
+]
+
[[package]]
name = "askama"
version = "0.11.1"
@@ -219,6 +237,21 @@ dependencies = [
"shlex",
]
+[[package]]
+name = "bit-set"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+dependencies = [
+ "bit-vec",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+
[[package]]
name = "bit_field"
version = "0.10.1"
@@ -577,13 +610,15 @@ dependencies = [
"image",
"indicatif",
"itertools",
+ "lalrpop",
+ "lalrpop-util",
"lazy_static",
"logos",
"lru",
"maplit",
"md5",
"memmap",
- "nom",
+ "once_cell",
"parse_wiki_text",
"quick-xml",
"rand",
@@ -662,6 +697,15 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+[[package]]
+name = "ena"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"
+dependencies = [
+ "log",
+]
+
[[package]]
name = "encode_unicode"
version = "0.3.6"
@@ -729,6 +773,12 @@ dependencies = [
"instant",
]
+[[package]]
+name = "fixedbitset"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
+
[[package]]
name = "flate2"
version = "1.0.24"
@@ -1156,12 +1206,12 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "1.8.2"
+version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a"
+checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
dependencies = [
"autocfg",
- "hashbrown 0.11.2",
+ "hashbrown 0.12.1",
]
[[package]]
@@ -1252,6 +1302,38 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "lalrpop"
+version = "0.19.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823"
+dependencies = [
+ "ascii-canvas",
+ "atty",
+ "bit-set",
+ "diff",
+ "ena",
+ "itertools",
+ "lalrpop-util",
+ "petgraph",
+ "pico-args",
+ "regex",
+ "regex-syntax 0.6.26",
+ "string_cache",
+ "term",
+ "tiny-keccak",
+ "unicode-xid",
+]
+
+[[package]]
+name = "lalrpop-util"
+version = "0.19.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4"
+dependencies = [
+ "regex",
+]
+
[[package]]
name = "lazy_static"
version = "1.4.0"
@@ -1555,6 +1637,12 @@ dependencies = [
"tempfile",
]
+[[package]]
+name = "new_debug_unreachable"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
+
[[package]]
name = "nom"
version = "7.1.1"
@@ -1622,9 +1710,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
-version = "1.12.0"
+version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"
+checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e"
[[package]]
name = "oneshot"
@@ -1776,6 +1864,31 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
+[[package]]
+name = "petgraph"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"
+dependencies = [
+ "fixedbitset",
+ "indexmap",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "pico-args"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"
+
[[package]]
name = "pin-project"
version = "1.0.10"
@@ -1832,6 +1945,12 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+
[[package]]
name = "pretty_assertions"
version = "1.2.1"
@@ -1975,6 +2094,8 @@ version = "1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
dependencies = [
+ "aho-corasick",
+ "memchr",
"regex-syntax 0.6.26",
]
@@ -2310,6 +2431,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "siphasher"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
+
[[package]]
name = "slab"
version = "0.4.6"
@@ -2364,6 +2491,19 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+[[package]]
+name = "string_cache"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"
+dependencies = [
+ "new_debug_unreachable",
+ "once_cell",
+ "parking_lot 0.12.1",
+ "phf_shared",
+ "precomputed-hash",
+]
+
[[package]]
name = "strsim"
version = "0.10.0"
@@ -2496,6 +2636,17 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "term"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
+dependencies = [
+ "dirs-next",
+ "rustversion",
+ "winapi",
+]
+
[[package]]
name = "termcolor"
version = "1.1.3"
@@ -2593,6 +2744,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "tiny-keccak"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
+dependencies = [
+ "crunchy",
+]
+
[[package]]
name = "tinyvec"
version = "1.6.0"
@@ -2830,6 +2990,12 @@ dependencies = [
"tinyvec",
]
+[[package]]
+name = "unicode-xid"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
+
[[package]]
name = "url"
version = "2.2.2"
diff --git a/Cargo.toml b/Cargo.toml
index ae349940..278420a6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,7 +18,6 @@ flate2 = "1.0.23"
indicatif = { version = "0.16.2", features = ["rayon"] }
itertools = "0.10.3"
lru = "0.7.6"
-nom = "7.1.1"
rand = "0.8.5"
rayon = "1.5.3"
reqwest = { version = "0.11.10", features = ["blocking"] }
@@ -53,9 +52,14 @@ bzip2 = "0.4.3"
parse_wiki_text = "0.1.5"
md5 = "0.7.0"
memmap = "0.7.0"
+lalrpop-util = { version = "0.19.8", features = ["lexer"] }
+once_cell = "1.13.1"
[dev-dependencies]
maplit = "1.0.2"
[profile.release]
debug = true
+
+[build-dependencies]
+lalrpop = "0.19.8"
diff --git a/build.rs b/build.rs
index 9f383a77..98dcc47d 100644
--- a/build.rs
+++ b/build.rs
@@ -10,4 +10,6 @@ fn main() {
println!("cargo:rerun-if-changed=yarn.lock");
println!("cargo:rerun-if-changed=tailwind.config.js");
println!("cargo:rerun-if-changed=postcss.config.js");
+
+ lalrpop::process_root().unwrap();
}
diff --git a/data.tar.gz b/data.tar.gz
index 970d26d3..9a1e6ea4 100644
--- a/data.tar.gz
+++ b/data.tar.gz
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:2cabcd57754add6d968fb719d9c4e8a6d259a758995163027f7714ffec1e35d4
-size 941862585
+oid sha256:1cadc250585c97a68bdfcfda2e341c30235f3369b0c72f2ff898c3473f0417ea
+size 968695137
diff --git a/justfile b/justfile
index 18e4485c..800ca07e 100644
--- a/justfile
+++ b/justfile
@@ -1,4 +1,5 @@
@unpack-data:
+ rm -rf data
tar -zxvf data.tar.gz
@flamegraph:
diff --git a/src/entity_index/mod.rs b/src/entity_index/mod.rs
index 42481c0b..5d64e4cd 100644
--- a/src/entity_index/mod.rs
+++ b/src/entity_index/mod.rs
@@ -26,6 +26,7 @@ use tantivy::{
collector::TopDocs,
query::{BooleanQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},
schema::{BytesOptions, IndexRecordOption, Schema, TextFieldIndexing, TextOptions},
+ tokenizer::Tokenizer,
DocAddress, IndexReader, IndexWriter, LeasedItem, Searcher, Term,
};
use tracing::info;
@@ -263,31 +264,31 @@ impl EntityIndex {
let title = self.schema.get_field("title").unwrap();
let entity_abstract = self.schema.get_field("abstract").unwrap();
- let term_queries: Vec<(Occur, Box<dyn tantivy::query::Query>)> = query
- .split(' ')
- .map(|term| term.to_ascii_lowercase())
- .filter(|term| !self.stopwords.contains(term))
- .flat_map(|term| {
- vec![
- (
- Occur::Must,
- TermQuery::new(
- Term::from_field_text(title, &term),
- IndexRecordOption::WithFreqsAndPositions,
- )
- .box_clone(),
- ),
- (
- Occur::Should,
- TermQuery::new(
- Term::from_field_text(entity_abstract, &term),
- IndexRecordOption::WithFreqsAndPositions,
- )
- .box_clone(),
- ),
- ]
- })
- .collect();
+ let mut term_queries = Vec::new();
+ let mut stream = NormalTokenizer::default().token_stream(query.as_str());
+ while let Some(token) = stream.next() {
+ if self.stopwords.contains(&token.text) {
+ continue;
+ }
+
+ term_queries.push((
+ Occur::Must,
+ TermQuery::new(
+ Term::from_field_text(title, &token.text),
+ IndexRecordOption::WithFreqsAndPositions,
+ )
+ .box_clone(),
+ ));
+
+ term_queries.push((
+ Occur::Should,
+ TermQuery::new(
+ Term::from_field_text(entity_abstract, &token.text),
+ IndexRecordOption::WithFreqsAndPositions,
+ )
+ .box_clone(),
+ ));
+ }
let query = BooleanQuery::from(term_queries);
diff --git a/src/kv/sled_store.rs b/src/kv/sled_store.rs
index 25e12aa3..b9d3c641 100644
--- a/src/kv/sled_store.rs
+++ b/src/kv/sled_store.rs
@@ -16,7 +16,6 @@
use std::marker::PhantomData;
-use nom::AsBytes;
use serde::{de::DeserializeOwned, Serialize};
use crate::kv::Kv;
@@ -31,7 +30,7 @@ where
fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {
self.get(key)
.expect("failed to retrieve key")
- .map(|v| v.as_bytes().to_vec())
+ .map(|v| v.to_vec())
}
fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {
diff --git a/src/lib.rs b/src/lib.rs
index 97838062..86b90356 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -185,6 +185,15 @@ pub enum Error {
#[error("Spell dictionary error")]
Spell(#[from] crate::spell::dictionary::DictionaryError),
+
+ #[error("Parser error")]
+ Parse,
+}
+
+impl<L, T, E> From<lalrpop_util::ParseError<L, T, E>> for Error {
+ fn from(_: lalrpop_util::ParseError<L, T, E>) -> Self {
+ Error::Parse
+ }
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
diff --git a/src/query/mod.rs b/src/query/mod.rs
index 34ea52df..03d58753 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -14,22 +14,19 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{query::weight::Weight, schema::ALL_FIELDS, Error, Result};
-use nom::{
- bytes::complete::{tag, take_while},
- character::complete::multispace0,
- error::ParseError,
- multi::separated_list0,
- sequence::preceded,
- IResult,
+use crate::{query::weight::Weight, schema::ALL_FIELDS, tokenizer::NormalTokenizer, Result};
+use std::{
+ collections::{BTreeMap, HashMap},
+ sync::Arc,
};
-use std::{collections::BTreeMap, sync::Arc};
use tantivy::{
schema::{FieldType, IndexRecordOption, Schema},
- tokenizer::TextAnalyzer,
+ tokenizer::{TextAnalyzer, Tokenizer},
};
use self::bm25::Bm25Weight;
+use lalrpop_util::lalrpop_mod;
+lalrpop_mod!(pub parser, "/query/parser.rs"); // synthesized by LALRPOP
mod bm25;
mod field_union;
@@ -37,22 +34,11 @@ mod term_intersection;
mod term_scorer;
mod vec_docset;
mod weight;
-use itertools::Itertools;
-
-/// A combinator that takes a parser `inner` and produces a parser that also consumes leading
-/// whitespace, returning the output of `inner`.
-fn ws<'a, F: 'a, O, E: ParseError<&'a str>>(
- inner: F,
-) -> impl FnMut(&'a str) -> IResult<&'a str, O, E>
-where
- F: Fn(&'a str) -> IResult<&'a str, O, E>,
-{
- preceded(multispace0, inner)
-}
-fn term(term: &str) -> IResult<&str, &str> {
- ws(take_while(|c: char| !c.is_whitespace()))(term)
-}
+static PARSER: once_cell::sync::Lazy<parser::TermsParser> =
+ once_cell::sync::Lazy::new(parser::TermsParser::new);
+
+const MAX_SIMILAR_TERMS: usize = 10;
pub struct FieldData {
tantivy: tantivy::schema::Field,
@@ -69,6 +55,11 @@ impl std::fmt::Debug for FieldData {
}
}
+#[derive(Debug)]
+pub enum Term<'a> {
+ Simple(&'a str),
+}
+
#[derive(Clone, Debug)]
pub struct Query {
terms: Vec<String>,
@@ -77,20 +68,34 @@ pub struct Query {
impl Query {
pub fn parse(query: &str, schema: Arc<Schema>) -> Result<Query> {
- match separated_list0(tag(" "), term)(query) {
- Ok((remaining, terms)) => {
- debug_assert!(remaining.is_empty());
-
- let terms = terms
- .into_iter()
- .filter(|term| !term.is_empty())
- .map(|term| term.to_string())
- .unique()
- .collect();
- Ok(Query { terms, schema })
+ let mut raw_terms = Vec::new();
+
+ let terms = PARSER.parse(query)?;
+ for term in terms {
+ match term {
+ Term::Simple(term) => {
+ let mut stream = NormalTokenizer::default().token_stream(term);
+
+ while let Some(token) = stream.next() {
+ raw_terms.push(token.text.to_string());
+ }
+ }
}
- Err(error) => Err(Error::ParsingError(error.to_string())),
}
+
+ let mut term_count = HashMap::new();
+ let mut terms = Vec::new();
+ for term in raw_terms {
+ let count = term_count.entry(term.clone()).or_insert(0);
+
+ if *count < MAX_SIMILAR_TERMS {
+ terms.push(term);
+ }
+
+ *count += 1;
+ }
+
+ Ok(Query { terms, schema })
}
pub fn simple_terms(&self) -> &[String] {
@@ -175,18 +180,30 @@ mod tests {
fn simple_parse() {
let schema = Arc::new(create_schema());
- let query = Query::parse("This is a simple query the the query the the", Arc::clone(&schema))
- .expect("Failed to parse query");
+ let query = Query::parse(
+ "This is a simple query the the the the the the the the the the the the the",
+ Arc::clone(&schema),
+ )
+ .expect("Failed to parse query");
assert_eq!(
query.terms,
vec![
- "This".to_string(),
+ "this".to_string(),
"is".to_string(),
"a".to_string(),
"simple".to_string(),
"query".to_string(),
"the".to_string(),
+ "the".to_string(),
+ "the".to_string(),
+ "the".to_string(),
+ "the".to_string(),
+ "the".to_string(),
+ "the".to_string(),
+ "the".to_string(),
+ "the".to_string(),
+ "the".to_string(),
]
);
}
@@ -201,7 +218,7 @@ mod tests {
assert_eq!(
query.terms,
vec![
- "This".to_string(),
+ "this".to_string(),
"is".to_string(),
"a".to_string(),
"simple".to_string(),
@@ -229,7 +246,13 @@ mod tests {
.terms;
assert_eq!(
terms,
- vec!["term!".to_string(), "term#".to_string(), "$".to_string()]
+ vec![
+ "term".to_string(),
+ "!".to_string(),
+ "term".to_string(),
+ "#".to_string(),
+ "$".to_string()
+ ]
);
}
}
diff --git a/src/query/parser.lalrpop b/src/query/parser.lalrpop
new file mode 100644
index 00000000..9ec70847
--- /dev/null
+++ b/src/query/parser.lalrpop
@@ -0,0 +1,12 @@
+use crate::query::Term;
+
+grammar;
+
+
+pub Terms: Vec<Term<'input>> = <Term*> => <>;
+
+Term: Term<'input> = {
+ <BasicTerm> => Term::Simple(<>),
+}
+
+BasicTerm: &'input str = r"[\S]*" => <>;
\ No newline at end of file
diff --git a/src/query/term_intersection.rs b/src/query/term_intersection.rs
index 2f2a90e5..c9f96f8e 100644
--- a/src/query/term_intersection.rs
+++ b/src/query/term_intersection.rs
@@ -20,8 +20,8 @@ use tantivy::{DocId, DocSet, Postings, Score, TERMINATED};
use super::field_union::FieldUnion;
-const MAX_WIDTH: u32 = 45;
-const LAMBDA: f32 = 0.55;
+const MAX_WIDTH: u32 = 25;
+const LAMBDA: f32 = 0.75;
const GAMMA: f32 = 0.25;
pub fn intersect_terms(mut scorers: Vec<FieldUnion>) -> Box<dyn Scorer> {
diff --git a/src/spell/dictionary.rs b/src/spell/dictionary.rs
index 74660a96..9ab1c6f2 100644
--- a/src/spell/dictionary.rs
+++ b/src/spell/dictionary.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::spell::distance::LevenshteinDistance;
+use crate::tokenizer::NormalTokenizer;
use crate::webpage::Webpage;
use fst::map::Union;
use fst::{Automaton, IntoStreamer, Map, MapBuilder, Streamer};
@@ -24,6 +25,7 @@ use std::hash::{Hash, Hasher};
use std::ops::AddAssign;
use std::path::Path;
use std::{cmp, io, mem};
+use tantivy::tokenizer::Tokenizer;
use thiserror::Error;
pub trait EditStrategy: Send + Sync {
@@ -332,8 +334,12 @@ impl<const TOP_N: usize> Dictionary<TOP_N> {
}
pub fn insert_page(&mut self, webpage: &Webpage) {
- for text in webpage.html.clean_text().unwrap_or_default().split(' ') {
- self.insert(text);
+ let text = webpage.html.clean_text().unwrap_or_default();
+
+ let mut stream = NormalTokenizer::default().token_stream(text.as_str());
+
+ while let Some(token) = stream.next() {
+ self.insert(&token.text);
}
}
@@ -445,7 +451,7 @@ mod tests {
}
#[test]
- fn only_stop_top_n() {
+ fn only_store_top_n() {
let mut dict: Dictionary<2> = Dictionary::open::<&str>(None).unwrap();
dict.insert("test");
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 7efd8991..092cdd23 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -14,9 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use logos::{Lexer, Logos};
use tantivy::tokenizer::{
- Language, LowerCaser, SimpleTokenizer, Stemmer, StopWordFilter, TextAnalyzer,
+ BoxTokenStream, Language, LowerCaser, Stemmer, StopWordFilter, TextAnalyzer,
};
+
use whatlang::Lang;
struct MyStemmer(Stemmer);
@@ -140,3 +142,191 @@ impl tantivy::tokenizer::Tokenizer for StemmedTokenizer {
}
}
}
+
+#[derive(Logos, Debug, PartialEq)]
+enum Token {
+ #[regex("[\\w|\\p{Han}|\\p{Hiragana}|\\p{Katakana}|\\p{Cyrillic}|\\p{Arabic}]+")]
+ Text,
+
+ #[error]
+ #[regex(r"[ \t\n\f]+", logos::skip)]
+ Error,
+}
+
+#[derive(Clone)]
+pub struct SimpleTokenizer;
+
+pub struct SimpleTokenStream<'a> {
+ lexer: Lexer<'a, Token>,
+ token: Option<tantivy::tokenizer::Token>,
+ next_position: usize,
+}
+
+impl tantivy::tokenizer::Tokenizer for SimpleTokenizer {
+ fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> {
+ let lexer = Token::lexer(text);
+ BoxTokenStream::from(SimpleTokenStream {
+ lexer,
+ token: None,
+ next_position: 0,
+ })
+ }
+}
+
+impl<'a> tantivy::tokenizer::TokenStream for SimpleTokenStream<'a> {
+ fn advance(&mut self) -> bool {
+ self.token = self.lexer.next().map(|_| {
+ let span = self.lexer.span();
+ let pos = self.next_position;
+ self.next_position += 1;
+ tantivy::tokenizer::Token {
+ offset_from: span.start,
+ offset_to: span.end,
+ position: pos,
+ text: self.lexer.slice().to_string(),
+ ..Default::default()
+ }
+ });
+
+ self.token.is_some()
+ }
+
+ fn token(&self) -> &tantivy::tokenizer::Token {
+ self.token.as_ref().unwrap()
+ }
+
+ fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
+ self.token.as_mut().unwrap()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use tantivy::tokenizer::Tokenizer;
+
+ use super::*;
+
+ fn tokenize(s: &str) -> Vec<String> {
+ let mut res = Vec::new();
+ let mut stream = NormalTokenizer::default().token_stream(s);
+
+ while let Some(token) = stream.next() {
+ res.push(token.text.clone());
+ }
+
+ res
+ }
+
+ #[test]
+ fn simple_tokenization() {
+ assert_eq!(
+ tokenize("this is a relatively simple123 test string"),
+ vec![
+ "this".to_string(),
+ "is".to_string(),
+ "a".to_string(),
+ "relatively".to_string(),
+ "simple123".to_string(),
+ "test".to_string(),
+ "string".to_string()
+ ]
+ );
+ }
+
+ #[test]
+ fn special_character_tokenization() {
+ assert_eq!(
+ tokenize("example.com"),
+ vec!["example".to_string(), ".".to_string(), "com".to_string(),]
+ );
+ assert_eq!(
+ tokenize("example. com"),
+ vec!["example".to_string(), ".".to_string(), "com".to_string(),]
+ );
+ assert_eq!(
+ tokenize("example . com"),
+ vec!["example".to_string(), ".".to_string(), "com".to_string(),]
+ );
+
+ assert_eq!(
+ tokenize("a c++ blog post"),
+ vec![
+ "a".to_string(),
+ "c".to_string(),
+ "+".to_string(),
+ "+".to_string(),
+ "blog".to_string(),
+ "post".to_string()
+ ]
+ );
+ assert_eq!(
+ tokenize("path/test"),
+ vec!["path".to_string(), "/".to_string(), "test".to_string(),]
+ );
+ }
+
+ #[test]
+ fn han() {
+ assert_eq!(
+ tokenize("test 漢.com"),
+ vec![
+ "test".to_string(),
+ "漢".to_string(),
+ ".".to_string(),
+ "com".to_string()
+ ]
+ );
+ }
+
+ #[test]
+ fn hiragana() {
+ assert_eq!(
+ tokenize("test あ.com"),
+ vec![
+ "test".to_string(),
+ "あ".to_string(),
+ ".".to_string(),
+ "com".to_string()
+ ]
+ );
+ }
+
+ #[test]
+ fn katakana() {
+ assert_eq!(
+ tokenize("test ダ.com"),
+ vec![
+ "test".to_string(),
+ "ダ".to_string(),
+ ".".to_string(),
+ "com".to_string()
+ ]
+ );
+ }
+
+ #[test]
+ fn cyrillic() {
+ assert_eq!(
+ tokenize("test б.com"),
+ vec![
+ "test".to_string(),
+ "б".to_string(),
+ ".".to_string(),
+ "com".to_string()
+ ]
+ );
+ }
+
+ #[test]
+ fn arabic() {
+ assert_eq!(
+ tokenize("test ب.com"),
+ vec![
+ "test".to_string(),
+ "ب".to_string(),
+ ".".to_string(),
+ "com".to_string()
+ ]
+ );
+ }
+}
diff --git a/templates/entity.html b/templates/entity.html
index 068b3bd5..d4670703 100644
--- a/templates/entity.html
+++ b/templates/entity.html
@@ -4,8 +4,8 @@
{% match entity.image %}
{% when Some(image) %}
<div class="w-lg mb-3">
- <a href="https://en.wikipedia.org/wiki/{{ entity.title }}">
- <img class="h-full w-full rounded-full" src="/entity/image/{{ image }}" />
+ <a href="https://en.wikipedia.org/wiki/{{ entity.title|urlencode }}">
+ <img class="h-full w-full rounded-full" src="/entity/image/{{ image|urlencode }}" />
</a>
</div>
{% when None %}
@@ -16,7 +16,8 @@
<div class="text-sm">
{{ entity.small_abstract }}
<span class="italic">
- source: <a class="text-blue-600" href="https://en.wikipedia.org/wiki/{{ entity.title }}">wikipedia</a>
+ source: <a class="text-blue-600"
+ href="https://en.wikipedia.org/wiki/{{ entity.title|urlencode }}">wikipedia</a>
</span>
</div>
{% if !entity.info.is_empty() %}
@@ -46,14 +47,15 @@
{% match entity.image %}
{% when Some(image) %}
<div class="h-20 w-20 mb-3">
- <a href="/search?q={{ entity.title }}">
- <img class="h-full object-cover w-full rounded-full" src="/entity/image/{{ image }}" />
+ <a href="/search?q={{ entity.title|urlencode }}">
+ <img class="h-full object-cover w-full rounded-full"
+ src="/entity/image/{{ image|urlencode }}" />
</a>
</div>
{% when None %}
{% endmatch %}
<div class="text-center">
- <a href="/search?q={{ entity.title }}">
+ <a href="/search?q={{ entity.title|urlencode }}">
{{ entity.title }}
</a>
</div>
diff --git a/templates/search.html b/templates/search.html
index 9144010e..fa76e7f9 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -19,7 +19,8 @@
{% match spell_correction %}
{% when Some with (correction) %}
<div class="pl-3 pb-3 ">
- Did you mean: <a class="italic title-link font-bold" href="/search?q={{ correction }}">{{ correction
+ Did you mean: <a class="italic title-link font-bold" href="/search?q={{ correction|urlencode }}">{{
+ correction
}}</a>
</div>
{% when None %}
@@ -31,10 +32,11 @@
<div class="flex flex-col w-full">
<div class="text-sm mb-1 flex">
<div class="flex items-center mr-2">
- <img src="/favicons/{{ item.domain }}" width="13px" />
+ <img src="/favicons/{{ item.domain|urlencode }}" width="13px" />
</div>
<div>
- <a class="hover:no-underline text-gray-600" href="{{ item.url }}">{{ item.pretty_url
+ <a class="hover:no-underline text-gray-600" href="{{ item.url|urlencode }}">{{
+ item.pretty_url
}}</a>
</div>
</div>
@@ -56,8 +58,8 @@
<div class="h-20 w-20 pl-2 pt-1 pb-1">
{% match item.primary_image_uuid %}
{% when Some with (val) %}
- <a href="{{ item.url }}">
- <img class="h-full w-full object-cover rounded-full" src="/image/{{ val }}" />
+ <a href="{{ item.url|urlencode }}">
+ <img class="h-full w-full object-cover rounded-full" src="/image/{{ val|urlencode }}" />
</a>
{% when None %}
{% endmatch %}