web search done right
use binary heap for less cmp when merging spell correction dictionaries
Mikkel Denkerd023552391d3
diff --git a/Cargo.lock b/Cargo.lockindex 2a9ef968..f3dccdc9 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4236,9 +4236,9 @@ checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"[[package]]name = "rustls-webpki"-version = "0.102.3"+version = "0.102.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf"+checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"dependencies = ["ring","rustls-pki-types",@@ -5789,7 +5789,7 @@ dependencies = ["serde","serde_json","utoipa",- "zip 1.2.3",+ "zip 1.3.0",][[package]]@@ -6350,9 +6350,9 @@ dependencies = [[[package]]name = "zip"-version = "1.2.3"+version = "1.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c700ea425e148de30c29c580c1f9508b93ca57ad31c9f4e96b83c194c37a7a8f"+checksum = "f1f4a27345eb6f7aa7bd015ba7eb4175fa4e1b462a29874b779e0bbcf96c6ac7"dependencies = ["arbitrary","crc32fast",diff --git a/assets/licenses.html b/assets/licenses.htmlindex 5bfb6a3c..f2ff509b 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -11829,7 +11829,7 @@ express Statement of Purpose.<h3 id="ISC">ISC License</h3><h4>Used by:</h4><ul class="license-used-by">- <li><a href=" https://github.com/rustls/webpki ">rustls-webpki 0.102.3</a></li>+ <li><a href=" https://github.com/rustls/webpki ">rustls-webpki 0.102.4</a></li></ul><pre class="license-text">// Copyright 2022 Rafael Fernández López.//@@ -14137,7 +14137,7 @@ SOFTWARE.<ul class="license-used-by"><li><a href=" https://github.com/mvdnes/spin-rs.git ">spin 0.9.8</a></li><li><a href=" https://github.com/zip-rs/zip.git ">zip 0.6.6</a></li>- <li><a href=" https://github.com/zip-rs/zip2.git ">zip 1.2.3</a></li>+ <li><a href=" https://github.com/zip-rs/zip2.git ">zip 1.3.0</a></li></ul><pre class="license-text">The MIT License (MIT)diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rsindex ab8f7ceb..97959af6 100644--- a/crates/core/src/web_spell/mod.rs+++ b/crates/core/src/web_spell/mod.rs@@ -210,6 +210,30 @@ impl<'a> MergePointer<'a> {}}+impl<'a> PartialOrd for MergePointer<'a> {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}++impl<'a> Ord for MergePointer<'a> {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ match (self.is_finished, other.is_finished) {+ (true, true) | (false, false) => self.term.cmp(&other.term),+ (true, false) => std::cmp::Ordering::Less,+ (false, true) => std::cmp::Ordering::Greater,+ }+ }+}++impl<'a> PartialEq for MergePointer<'a> {+ fn eq(&self, other: &Self) -> bool {+ self.term == other.term && self.is_finished == other.is_finished+ }+}++impl<'a> Eq for MergePointer<'a> {}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rsindex f50bf459..a19c5a14 100644--- a/crates/core/src/web_spell/stupid_backoff.rs+++ b/crates/core/src/web_spell/stupid_backoff.rs@@ -16,9 +16,10 @@use super::{tokenize, MergePointer, Result};use std::{- collections::BTreeMap,+ cmp::Reverse,+ collections::{BTreeMap, BinaryHeap},fs::{File, OpenOptions},- io::BufWriter,+ io::{BufWriter, Write},path::Path,};@@ -143,6 +144,7 @@ impl StupidBackoffTrainer {let mut wrt = BufWriter::new(f);bincode::encode_into_std_write(&self.n_counts, &mut wrt, bincode::config::standard())?;+ wrt.flush()?;Ok(())}@@ -166,40 +168,38 @@ fn merge_streams(pointer.advance();}- while pointers.iter().any(|p| !p.is_finished) {- let mut min_pointer: Option<&MergePointer<'_>> = None;-- for pointer in pointers.iter() {- if pointer.is_finished {- continue;- }-- if let Some(min) = min_pointer {- if pointer.term < min.term {- min_pointer = Some(pointer);+ let mut pointers: BinaryHeap<_> = pointers.into_iter().map(Reverse).collect();++ loop {+ let (term, mut freq, is_finished) = {+ match pointers.peek_mut() {+ Some(mut pointer) => {+ let res = (+ pointer.0.term.clone(),+ pointer.0.value,+ pointer.0.is_finished,+ );+ pointer.0.advance();+ res}- } else {- min_pointer = Some(pointer);+ None => break,}- }-- if let Some(min_pointer) = min_pointer {- let term = min_pointer.term.clone();- let mut freq = 0;+ };- for pointer in pointers.iter_mut() {- if pointer.is_finished {- continue;- }+ if is_finished {+ break;+ }- if pointer.term == term {- freq += pointer.value;- pointer.advance();- }+ while let Some(mut other) = pointers.peek_mut() {+ if other.0.term != term || other.0.is_finished {+ break;}- builder.insert(term, freq)?;+ freq += other.0.value;+ other.0.advance();}++ builder.insert(term, freq)?;}builder.finish()?;@@ -255,6 +255,7 @@ impl StupidBackoff {let mut wrt = BufWriter::new(file);bincode::encode_into_std_write(&n_counts, &mut wrt, bincode::config::standard())?;+ wrt.flush()?;let file = OpenOptions::new().create(true)@@ -281,11 +282,12 @@ impl StupidBackoff {let wtr = BufWriter::new(file);let builder = fst::MapBuilder::new(wtr)?;- let streams: Vec<_> = models.iter().map(|d| d.ngrams.stream()).collect();+ let streams: Vec<_> = models.iter().map(|d| d.rotated_ngrams.stream()).collect();merge_streams(builder, streams)?;- let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ let mmap =+ unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };let rotated_ngrams = fst::Map::new(mmap)?;Ok(Self {