Skip to content

web search done right

Tokenizer index out of bounds

Mikkel Denkera53fd6808517
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 1d100fe5..e951c7cc 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -304,3 +304,23 @@ fn rand_words(num_words: usize) -> String {
res.trim().to_string()
}
+
+fn ceil_char_boundary(str: &str, index: usize) -> usize {
+ let mut res = index;
+
+ while !str.is_char_boundary(res) && res < str.len() {
+ res += 1;
+ }
+
+ res
+}
+
+fn floor_char_boundary(str: &str, index: usize) -> usize {
+ let mut res = index;
+
+ while !str.is_char_boundary(res) && res > 0 {
+ res -= 1;
+ }
+
+ res
+}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index f6279db1..e46656f3 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -16,8 +16,10 @@
use crate::{
bangs::{Bang, BangHit, Bangs},
+ ceil_char_boundary,
collector::{self, BucketCollector},
exponential_backoff::ExponentialBackoff,
+ floor_char_boundary,
inverted_index::{self, RetrievedWebpage},
qa_model::QaModel,
query,
@@ -637,26 +639,6 @@ fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {
}
}
-fn ceil_char_boundary(str: &str, index: usize) -> usize {
- let mut res = index;
-
- while !str.is_char_boundary(res) && res < str.len() {
- res += 1;
- }
-
- res
-}
-
-fn floor_char_boundary(str: &str, index: usize) -> usize {
- let mut res = index;
-
- while !str.is_char_boundary(res) && res > 0 {
- res -= 1;
- }
-
- res
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
index 2d88a7d1..5521b3b6 100644
--- a/core/src/tokenizer.rs
+++ b/core/src/tokenizer.rs
@@ -23,6 +23,8 @@ use tantivy::tokenizer::{
use whatlang::Lang;
+use crate::{ceil_char_boundary, floor_char_boundary};
+
struct MyStemmer(Stemmer);
impl From<Lang> for MyStemmer {
@@ -393,6 +395,10 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {
if prev_was_quote {
self.token.offset_from -= 1;
self.token.offset_to += 1;
+
+ self.token.offset_from = floor_char_boundary(self.text, self.token.offset_from);
+ self.token.offset_to =
+ ceil_char_boundary(self.text, self.token.offset_to).min(self.text.len());
}
self.token
@@ -459,6 +465,18 @@ mod tests {
);
}
+ #[test]
+ fn out_of_bounds_crash() {
+ tokenize_json(
+ r#"
+Breadcrumb.title="Home"
+Breadcrumb.url="https://www.eurotecnicaservice.it/?lang=en"
+Breadcrumb.title="Fuser Pur"
+Breadcrumb.url="https://www.eurotecnicaservice.it/testing\"
+"#,
+ );
+ }
+
#[test]
fn special_character_tokenization() {
assert_eq!(