Skip to content

web search done right

use binary heap for less cmp when merging spell correction dictionaries

Mikkel Denkerd023552391d3
diff --git a/Cargo.lock b/Cargo.lock
index 2a9ef968..f3dccdc9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4236,9 +4236,9 @@ checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
[[package]]
name = "rustls-webpki"
-version = "0.102.3"
+version = "0.102.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf"
+checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"
dependencies = [
"ring",
"rustls-pki-types",
@@ -5789,7 +5789,7 @@ dependencies = [
"serde",
"serde_json",
"utoipa",
- "zip 1.2.3",
+ "zip 1.3.0",
]
[[package]]
@@ -6350,9 +6350,9 @@ dependencies = [
[[package]]
name = "zip"
-version = "1.2.3"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c700ea425e148de30c29c580c1f9508b93ca57ad31c9f4e96b83c194c37a7a8f"
+checksum = "f1f4a27345eb6f7aa7bd015ba7eb4175fa4e1b462a29874b779e0bbcf96c6ac7"
dependencies = [
"arbitrary",
"crc32fast",
diff --git a/assets/licenses.html b/assets/licenses.html
index 5bfb6a3c..f2ff509b 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -11829,7 +11829,7 @@ express Statement of Purpose.
<h3 id="ISC">ISC License</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
- <li><a href=" https://github.com/rustls/webpki ">rustls-webpki 0.102.3</a></li>
+ <li><a href=" https://github.com/rustls/webpki ">rustls-webpki 0.102.4</a></li>
</ul>
<pre class="license-text">// Copyright 2022 Rafael Fernández López.
//
@@ -14137,7 +14137,7 @@ SOFTWARE.
<ul class="license-used-by">
<li><a href=" https://github.com/mvdnes/spin-rs.git ">spin 0.9.8</a></li>
<li><a href=" https://github.com/zip-rs/zip.git ">zip 0.6.6</a></li>
- <li><a href=" https://github.com/zip-rs/zip2.git ">zip 1.2.3</a></li>
+ <li><a href=" https://github.com/zip-rs/zip2.git ">zip 1.3.0</a></li>
</ul>
<pre class="license-text">The MIT License (MIT)
diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rs
index ab8f7ceb..97959af6 100644
--- a/crates/core/src/web_spell/mod.rs
+++ b/crates/core/src/web_spell/mod.rs
@@ -210,6 +210,30 @@ impl<'a> MergePointer<'a> {
}
}
+impl<'a> PartialOrd for MergePointer<'a> {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl<'a> Ord for MergePointer<'a> {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ match (self.is_finished, other.is_finished) {
+ (true, true) | (false, false) => self.term.cmp(&other.term),
+ (true, false) => std::cmp::Ordering::Less,
+ (false, true) => std::cmp::Ordering::Greater,
+ }
+ }
+}
+
+impl<'a> PartialEq for MergePointer<'a> {
+ fn eq(&self, other: &Self) -> bool {
+ self.term == other.term && self.is_finished == other.is_finished
+ }
+}
+
+impl<'a> Eq for MergePointer<'a> {}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rs
index f50bf459..a19c5a14 100644
--- a/crates/core/src/web_spell/stupid_backoff.rs
+++ b/crates/core/src/web_spell/stupid_backoff.rs
@@ -16,9 +16,10 @@
use super::{tokenize, MergePointer, Result};
use std::{
- collections::BTreeMap,
+ cmp::Reverse,
+ collections::{BTreeMap, BinaryHeap},
fs::{File, OpenOptions},
- io::BufWriter,
+ io::{BufWriter, Write},
path::Path,
};
@@ -143,6 +144,7 @@ impl StupidBackoffTrainer {
let mut wrt = BufWriter::new(f);
bincode::encode_into_std_write(&self.n_counts, &mut wrt, bincode::config::standard())?;
+ wrt.flush()?;
Ok(())
}
@@ -166,40 +168,38 @@ fn merge_streams(
pointer.advance();
}
- while pointers.iter().any(|p| !p.is_finished) {
- let mut min_pointer: Option<&MergePointer<'_>> = None;
-
- for pointer in pointers.iter() {
- if pointer.is_finished {
- continue;
- }
-
- if let Some(min) = min_pointer {
- if pointer.term < min.term {
- min_pointer = Some(pointer);
+ let mut pointers: BinaryHeap<_> = pointers.into_iter().map(Reverse).collect();
+
+ loop {
+ let (term, mut freq, is_finished) = {
+ match pointers.peek_mut() {
+ Some(mut pointer) => {
+ let res = (
+ pointer.0.term.clone(),
+ pointer.0.value,
+ pointer.0.is_finished,
+ );
+ pointer.0.advance();
+ res
}
- } else {
- min_pointer = Some(pointer);
+ None => break,
}
- }
-
- if let Some(min_pointer) = min_pointer {
- let term = min_pointer.term.clone();
- let mut freq = 0;
+ };
- for pointer in pointers.iter_mut() {
- if pointer.is_finished {
- continue;
- }
+ if is_finished {
+ break;
+ }
- if pointer.term == term {
- freq += pointer.value;
- pointer.advance();
- }
+ while let Some(mut other) = pointers.peek_mut() {
+ if other.0.term != term || other.0.is_finished {
+ break;
}
- builder.insert(term, freq)?;
+ freq += other.0.value;
+ other.0.advance();
}
+
+ builder.insert(term, freq)?;
}
builder.finish()?;
@@ -255,6 +255,7 @@ impl StupidBackoff {
let mut wrt = BufWriter::new(file);
bincode::encode_into_std_write(&n_counts, &mut wrt, bincode::config::standard())?;
+ wrt.flush()?;
let file = OpenOptions::new()
.create(true)
@@ -281,11 +282,12 @@ impl StupidBackoff {
let wtr = BufWriter::new(file);
let builder = fst::MapBuilder::new(wtr)?;
- let streams: Vec<_> = models.iter().map(|d| d.ngrams.stream()).collect();
+ let streams: Vec<_> = models.iter().map(|d| d.rotated_ngrams.stream()).collect();
merge_streams(builder, streams)?;
- let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
+ let mmap =
+ unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };
let rotated_ngrams = fst::Map::new(mmap)?;
Ok(Self {