web search done right
Tokenizer index out of bounds
Mikkel Denkera53fd6808517
diff --git a/core/src/lib.rs b/core/src/lib.rsindex 1d100fe5..e951c7cc 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -304,3 +304,23 @@ fn rand_words(num_words: usize) -> String {res.trim().to_string()}++fn ceil_char_boundary(str: &str, index: usize) -> usize {+ let mut res = index;++ while !str.is_char_boundary(res) && res < str.len() {+ res += 1;+ }++ res+}++fn floor_char_boundary(str: &str, index: usize) -> usize {+ let mut res = index;++ while !str.is_char_boundary(res) && res > 0 {+ res -= 1;+ }++ res+}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex f6279db1..e46656f3 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -16,8 +16,10 @@use crate::{bangs::{Bang, BangHit, Bangs},+ ceil_char_boundary,collector::{self, BucketCollector},exponential_backoff::ExponentialBackoff,+ floor_char_boundary,inverted_index::{self, RetrievedWebpage},qa_model::QaModel,query,@@ -637,26 +639,6 @@ fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {}}-fn ceil_char_boundary(str: &str, index: usize) -> usize {- let mut res = index;-- while !str.is_char_boundary(res) && res < str.len() {- res += 1;- }-- res-}--fn floor_char_boundary(str: &str, index: usize) -> usize {- let mut res = index;-- while !str.is_char_boundary(res) && res > 0 {- res -= 1;- }-- res-}-#[cfg(test)]mod tests {use super::*;diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rsindex 2d88a7d1..5521b3b6 100644--- a/core/src/tokenizer.rs+++ b/core/src/tokenizer.rs@@ -23,6 +23,8 @@ use tantivy::tokenizer::{use whatlang::Lang;+use crate::{ceil_char_boundary, floor_char_boundary};+struct MyStemmer(Stemmer);impl From<Lang> for MyStemmer {@@ -393,6 +395,10 @@ impl<'a> tantivy::tokenizer::TokenStream for JsonFieldTokenStream<'a> {if prev_was_quote {self.token.offset_from -= 1;self.token.offset_to += 1;++ self.token.offset_from = floor_char_boundary(self.text, self.token.offset_from);+ self.token.offset_to =+ ceil_char_boundary(self.text, self.token.offset_to).min(self.text.len());}self.token@@ -459,6 +465,18 @@ mod tests {);}+ #[test]+ fn out_of_bounds_crash() {+ tokenize_json(+ r#"+Breadcrumb.title="Home"+Breadcrumb.url="https://www.eurotecnicaservice.it/?lang=en"+Breadcrumb.title="Fuser Pur"+Breadcrumb.url="https://www.eurotecnicaservice.it/testing\"+"#,+ );+ }+#[test]fn special_character_tokenization() {assert_eq!(