web search done right
Web spell as dedicated module (#240)
Mikkel Denker040d04413e0f
diff --git a/Cargo.lock b/Cargo.lockindex 46322d29..420cf83e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5072,6 +5072,7 @@ dependencies = ["utoipa","utoipa-swagger-ui","uuid",+ "web-spell","whatlang","zimba",]@@ -6186,6 +6187,28 @@ dependencies = ["riff",]+[[package]]+name = "web-spell"+version = "0.1.0"+dependencies = [+ "anyhow",+ "bincode",+ "file_store",+ "fnv",+ "fst",+ "indicatif 0.17.8",+ "itertools 0.13.0",+ "memmap2",+ "proptest",+ "rayon",+ "serde",+ "serde_json",+ "thiserror",+ "tracing",+ "uuid",+ "whatlang",+]+[[package]]name = "web-sys"version = "0.3.72"diff --git a/Cargo.toml b/Cargo.tomlindex d2be4e50..65d09ca9 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -16,6 +16,7 @@ members = ["crates/leechy","crates/common","crates/simple-wal",+ "crates/web-spell","fuzz",]resolver = "2"@@ -187,6 +188,7 @@ xxhash-rust = { version = "0.8.10", features = ["xxh3", "const_xxh3"] }zipf = "7.0.0"zstd = { version = "0.13", features = ["experimental"] }urlencoding = "2.1.3"+file_store = { path = "crates/file-store" }[profile.test.package]flate2.opt-level = 3diff --git a/assets/licenses.html b/assets/licenses.htmlindex 862235ab..ab81f847 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -46,7 +46,7 @@<ul class="licenses-overview"><li><a href="#Apache-2.0">Apache License 2.0</a> (411)</li><li><a href="#MIT">MIT License</a> (191)</li>- <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (9)</li>+ <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (10)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (9)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (8)</li><li><a href="#Unicode-3.0">Unicode License v3</a> (4)</li>@@ -76,6 +76,7 @@<li><a href=" https://crates.io/crates/robotstxt ">robotstxt 0.1.0</a></li><li><a href=" https://crates.io/crates/simple_wal ">simple_wal 0.1.0</a></li><li><a href=" https://crates.io/crates/speedy_kv ">speedy_kv 0.1.0</a></li>+ <li><a href=" https://crates.io/crates/web-spell ">web-spell 0.1.0</a></li></ul><pre class="license-text">GNU AFFERO GENERAL PUBLIC LICENSEVersion 3, 19 November 2007diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 6afe17f3..9b78f17c 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -117,6 +117,7 @@ uuid.workspace = truewhatlang.workspace = truezimba = { path = "../zimba" }urlencoding.workspace = true+web-spell = { path = "../web-spell" }[target.'cfg(not(target_env = "msvc"))'.dependencies]tikv-jemallocator.workspace = truediff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex c5b87df3..cc5b4e64 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -156,22 +156,6 @@ impl SearchQuery {}}-pub struct Correction;--impl Correction {- pub fn misspelled_prob() -> f64 {- 0.1- }-- pub fn correction_threshold() -> f64 {- 50.0 // logprob difference- }-- pub fn lm_prob_weight() -> f64 {- 5.77- }-}-pub struct Widgets;impl Widgets {diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 05f4955f..8de655f8 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -16,6 +16,8 @@pub mod defaults;+pub use web_spell::CorrectionConfig;+use super::Result;use crate::ampc::dht;use crate::distributed::member::ShardId;@@ -656,33 +658,6 @@ pub struct SiteStatsConfig {pub skip_warc_files: Option<usize>,}-#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize)]-pub struct CorrectionConfig {- /// The probability that a word is misspelled- #[serde(default = "defaults::Correction::misspelled_prob")]- pub misspelled_prob: f64,-- /// Lambda in eq. 2 (http://static.googleusercontent.com/media/research.google.com/en/us/pubs/archive/36180.pdf)- #[serde(default = "defaults::Correction::lm_prob_weight")]- pub lm_prob_weight: f64,-- /// The threshold that the difference between the log probability of the best- /// correction and the observed word must be above for the word to be- /// corrected- #[serde(default = "defaults::Correction::correction_threshold")]- pub correction_threshold: f64,-}--impl Default for CorrectionConfig {- fn default() -> Self {- Self {- misspelled_prob: defaults::Correction::misspelled_prob(),- lm_prob_weight: defaults::Correction::lm_prob_weight(),- correction_threshold: defaults::Correction::correction_threshold(),- }- }-}-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]pub struct GossipConfig {pub seed_nodes: Option<Vec<SocketAddr>>,diff --git a/crates/core/src/entrypoint/web_spell.rs b/crates/core/src/entrypoint/web_spell.rsindex 9c59aeaa..172b861a 100644--- a/crates/core/src/entrypoint/web_spell.rs+++ b/crates/core/src/entrypoint/web_spell.rs@@ -26,9 +26,9 @@ use whatlang::Lang;use crate::{config::{self, WebSpellConfig},entrypoint::download_all_warc_files,- web_spell::{FirstTrainer, FirstTrainerResult, SecondTrainer},webpage::Html,};+use web_spell::{FirstTrainer, FirstTrainerResult, SecondTrainer};pub struct SpellWorker {prev_trained: Vec<FnvHashMap<Lang, FirstTrainerResult>>,diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 6b75ffd4..cf56010e 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -92,7 +92,6 @@ pub mod tokenizer;#[allow(unused)]mod ttl_cache;pub mod warc;-pub mod web_spell;pub mod webgraph;pub mod webpage;mod widgets;diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rsindex 966fe333..2a51299d 100644--- a/crates/core/src/search_prettifier/mod.rs+++ b/crates/core/src/search_prettifier/mod.rs@@ -34,9 +34,9 @@ use crate::{ranking::{SignalEnumDiscriminants, SignalScore},searcher::SearchQuery,snippet::TextSnippet,- web_spell::{self, CorrectionTerm},webpage::url_ext::UrlExt,};+use web_spell::{self, CorrectionTerm};pub use self::stack_overflow::{create_stackoverflow_sidebar, CodeOrText};pub use entity::DisplayedEntity;diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 74fb491c..db304f85 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -40,7 +40,6 @@ use crate::ranking::{SignalScore,};use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};-use crate::web_spell::SpellChecker;use crate::webgraph::remote::RemoteWebgraph;use crate::webgraph::EdgeLimit;use crate::webpage::html::links::RelFlags;@@ -51,6 +50,7 @@ use crate::{ranking::{models::lambdamart::LambdaMART, pipeline::RankingPipeline},};use crate::{query, webgraph, Result};+use web_spell::SpellChecker;use self::sidebar::SidebarManager;use self::widget::WidgetManager;@@ -361,33 +361,29 @@ where.terms.into_iter().filter_map(|t| match t {- crate::web_spell::CorrectionTerm::Corrected { orig, correction } => {+ web_spell::CorrectionTerm::Corrected { orig, correction } => {Some((orig, correction))}- crate::web_spell::CorrectionTerm::NotCorrected(_) => None,+ web_spell::CorrectionTerm::NotCorrected(_) => None,}).collect();- let mut correction = crate::web_spell::Correction::empty(query);+ let mut correction = web_spell::Correction::empty(query);for term in terms {match term {query::parser::Term::SimpleOrPhrase(query::parser::SimpleOrPhrase::Simple(t)) => {if let Some(term_correction) = correction_map.get(t.as_str()) {- correction.push(crate::web_spell::CorrectionTerm::Corrected {+ correction.push(web_spell::CorrectionTerm::Corrected {orig: String::from(t),correction: term_correction.to_string(),});} else {- correction.push(crate::web_spell::CorrectionTerm::NotCorrected(- String::from(t),- ));+ correction.push(web_spell::CorrectionTerm::NotCorrected(String::from(t)));}}_ => {- correction.push(crate::web_spell::CorrectionTerm::NotCorrected(- term.to_string(),- ));+ correction.push(web_spell::CorrectionTerm::NotCorrected(term.to_string()));}}}diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex 99af9c15..a581332e 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -22,11 +22,11 @@ use crate::query::Query;use crate::tokenizer::fields::{BigramTokenizer, DefaultTokenizer, FieldTokenizer, Stemmed, TrigramTokenizer,};-use crate::web_spell::sentence_ranges;use crate::webpage::region::Region;use hashbrown::{HashMap, HashSet};use lending_iter::LendingIterator;use utoipa::ToSchema;+use web_spell::sentence_ranges;use itertools::Itertools;use whatlang::Lang;diff --git a/crates/web-spell/Cargo.toml b/crates/web-spell/Cargo.tomlnew file mode 100644index 00000000..c4b1f4ca--- /dev/null+++ b/crates/web-spell/Cargo.toml@@ -0,0 +1,27 @@+[package]+edition = "2021"+license = "AGPL-3.0"+name = "web-spell"+version = "0.1.0"++# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html++[dependencies]+anyhow.workspace = true+thiserror.workspace = true+serde.workspace = true+serde_json.workspace = true+bincode.workspace = true+fst.workspace = true+itertools.workspace = true+indicatif.workspace = true+rayon.workspace = true+uuid.workspace = true+tracing.workspace = true+memmap2.workspace = true+whatlang.workspace = true+fnv.workspace = true++[dev-dependencies]+file_store.workspace = true+proptest.workspace = truediff --git a/crates/web-spell/README.md b/crates/web-spell/README.mdnew file mode 100644index 00000000..be6b40a2--- /dev/null+++ b/crates/web-spell/README.md@@ -0,0 +1,13 @@+# Web Spell++Automatic spelling correction from web data. It is based on the paper+[Using the Web for Language Independent Spellchecking and+Autocorrection](http://static.googleusercontent.com/media/research.google.com/en/us/pubs/archive/36180.pdf)+from google.++## Usage+```rust+let checker = SpellChecker::open("<path-to-model>", CorrectionConfig::default()).unwrap();+let correction = checker.correct("hwllo", Lang::Eng);+assert_eq!(correction.unwrap().terms, vec![CorrectionTerm::Corrected { orig: "hwllo".to_string(), correction: "hello".to_string() }]);+```diff --git a/crates/web-spell/src/config.rs b/crates/web-spell/src/config.rsnew file mode 100644index 00000000..0a34217d--- /dev/null+++ b/crates/web-spell/src/config.rs@@ -0,0 +1,58 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++fn misspelled_prob() -> f64 {+ 0.1+}++fn correction_threshold() -> f64 {+ 50.0 // logprob difference+}++fn lm_prob_weight() -> f64 {+ 5.77+}++#[derive(Clone, Copy, Debug, serde::Deserialize, serde::Serialize)]+pub struct CorrectionConfig {+ /// The probability that a word is misspelled+ #[serde(default = "misspelled_prob")]+ pub misspelled_prob: f64,++ /// Lambda in eq. 2 (http://static.googleusercontent.com/media/research.google.com/en/us/pubs/archive/36180.pdf)+ #[serde(default = "lm_prob_weight")]+ pub lm_prob_weight: f64,++ /// The threshold that the difference between the log probability of the best+ /// correction and the observed word must be above for the word to be+ /// corrected+ #[serde(default = "correction_threshold")]+ pub correction_threshold: f64,+}++impl Default for CorrectionConfig {+ fn default() -> Self {+ Self {+ misspelled_prob: misspelled_prob(),+ lm_prob_weight: lm_prob_weight(),+ correction_threshold: correction_threshold(),+ }+ }+}++pub fn bincode_config() -> bincode::config::Configuration {+ bincode::config::standard()+}diff --git a/crates/core/src/web_spell/error_model.rs b/crates/web-spell/src/error_model.rssimilarity index 99%rename from crates/core/src/web_spell/error_model.rsrename to crates/web-spell/src/error_model.rsindex 2b118dca..5a7fae95 100644--- a/crates/core/src/web_spell/error_model.rs+++ b/crates/web-spell/src/error_model.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License asdiff --git a/crates/core/src/web_spell/mod.rs b/crates/web-spell/src/lib.rssimilarity index 91%rename from crates/core/src/web_spell/mod.rsrename to crates/web-spell/src/lib.rsindex 36384306..54d5bfcb 100644--- a/crates/core/src/web_spell/mod.rs+++ b/crates/web-spell/src/lib.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -17,12 +17,14 @@//! This module contains the spell checker. It is based on the paper//! http://static.googleusercontent.com/media/research.google.com/en/us/pubs/archive/36180.pdf//! from google.+mod config;mod error_model;pub mod spell_checker;mod stupid_backoff;mod term_freqs;mod trainer;+pub use config::CorrectionConfig;pub use error_model::ErrorModel;pub use spell_checker::SpellChecker;pub use stupid_backoff::StupidBackoff;@@ -34,7 +36,6 @@ pub use trainer::SecondTrainer;use fst::Streamer;use std::ops::Range;-use crate::ceil_char_boundary;use itertools::intersperse;#[derive(Debug, thiserror::Error)]@@ -233,9 +234,20 @@ impl PartialEq for MergePointer<'_> {impl Eq for MergePointer<'_> {}+fn ceil_char_boundary(str: &str, index: usize) -> usize {+ let mut res = index;++ while !str.is_char_boundary(res) && res < str.len() {+ res += 1;+ }++ res+}+#[cfg(test)]mod tests {use super::*;+ use proptest::prelude::*;#[test]fn test_sentence_ranges() {@@ -269,4 +281,18 @@ mod tests {assert_eq!(ranges.len(), 1);assert_eq!(&text[ranges[0].clone()], "site.com is the best");}++ proptest! {+ #[test]+ fn prop_ceil_char_boundary(s: String, index: usize) {+ let index = if s.is_empty() {+ 0+ } else {+ index % s.len()+ };++ let ceil = ceil_char_boundary(&s, index);+ prop_assert!(s.is_char_boundary(ceil));+ }+ }}diff --git a/crates/core/src/web_spell/spell_checker.rs b/crates/web-spell/src/spell_checker.rssimilarity index 98%rename from crates/core/src/web_spell/spell_checker.rsrename to crates/web-spell/src/spell_checker.rsindex e7ac0b32..47399d74 100644--- a/crates/core/src/web_spell/spell_checker.rs+++ b/crates/web-spell/src/spell_checker.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -20,10 +20,8 @@ use std::{path::Path, str::FromStr};use fnv::FnvHashMap;use whatlang::Lang;-use crate::{- config::CorrectionConfig,- web_spell::stupid_backoff::{IntoMiddle, LeftToRight, RightToLeft},-};+use crate::config::CorrectionConfig;+use crate::stupid_backoff::{IntoMiddle, LeftToRight, RightToLeft};use super::{error_model, Correction, CorrectionTerm, Error, ErrorModel, StupidBackoff, TermDict};diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/web-spell/src/stupid_backoff.rssimilarity index 98%rename from crates/core/src/web_spell/stupid_backoff.rsrename to crates/web-spell/src/stupid_backoff.rsindex 0d7ef596..2ebef6df 100644--- a/crates/core/src/web_spell/stupid_backoff.rs+++ b/crates/web-spell/src/stupid_backoff.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -23,6 +23,7 @@ use std::{path::{Path, PathBuf},};+use crate::config;use fst::{Automaton, IntoStreamer, Streamer};const DISCOUNT: f64 = 0.4;@@ -143,7 +144,7 @@ impl StupidBackoffTrainer {let f = File::create(path.as_ref().join("n_counts.bin"))?;let mut wrt = BufWriter::new(f);- bincode::encode_into_std_write(&self.n_counts, &mut wrt, common::bincode_config())?;+ bincode::encode_into_std_write(&self.n_counts, &mut wrt, config::bincode_config())?;wrt.flush()?;Ok(())@@ -226,7 +227,7 @@ impl StupidBackoff {let file = File::open(folder.as_ref().join("n_counts.bin"))?;let mut reader = std::io::BufReader::new(file);- let n_counts = bincode::decode_from_std_read(&mut reader, common::bincode_config())?;+ let n_counts = bincode::decode_from_std_read(&mut reader, config::bincode_config())?;Ok(Self {ngrams,@@ -257,7 +258,7 @@ impl StupidBackoff {.open(folder.as_ref().join("n_counts.bin"))?;let mut wrt = BufWriter::new(file);- bincode::encode_into_std_write(&n_counts, &mut wrt, common::bincode_config())?;+ bincode::encode_into_std_write(&n_counts, &mut wrt, config::bincode_config())?;wrt.flush()?;let file = OpenOptions::new()@@ -424,7 +425,7 @@ impl NextWordsStrategy for IntoMiddle {#[cfg(test)]mod tests {use super::*;- use crate::gen_temp_dir;+ use file_store::gen_temp_dir;#[test]fn test_contexts() {diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/web-spell/src/term_freqs.rssimilarity index 99%rename from crates/core/src/web_spell/term_freqs.rsrename to crates/web-spell/src/term_freqs.rsindex 4fbebd69..7cf1c421 100644--- a/crates/core/src/web_spell/term_freqs.rs+++ b/crates/web-spell/src/term_freqs.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License asdiff --git a/crates/core/src/web_spell/trainer.rs b/crates/web-spell/src/trainer.rssimilarity index 98%rename from crates/core/src/web_spell/trainer.rsrename to crates/web-spell/src/trainer.rsindex 4a41ab10..13f4b19f 100644--- a/crates/core/src/web_spell/trainer.rs+++ b/crates/web-spell/src/trainer.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -17,7 +17,7 @@use indicatif::ParallelProgressIterator;use rayon::prelude::*;-use crate::web_spell::error_model::ErrorModel;+use crate::error_model::ErrorModel;use super::{stupid_backoff::StupidBackoffTrainer, tokenize, Result};use std::{