web search done right
web-spell cleanup old dicts after merge
Mikkel Denker38416c607071
diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rsindex 97959af6..9b047c9b 100644--- a/crates/core/src/web_spell/mod.rs+++ b/crates/core/src/web_spell/mod.rs@@ -220,8 +220,8 @@ impl<'a> Ord for MergePointer<'a> {fn cmp(&self, other: &Self) -> std::cmp::Ordering {match (self.is_finished, other.is_finished) {(true, true) | (false, false) => self.term.cmp(&other.term),- (true, false) => std::cmp::Ordering::Less,- (false, true) => std::cmp::Ordering::Greater,+ (true, false) => std::cmp::Ordering::Greater,+ (false, true) => std::cmp::Ordering::Less,}}}diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rsindex a19c5a14..6981176c 100644--- a/crates/core/src/web_spell/stupid_backoff.rs+++ b/crates/core/src/web_spell/stupid_backoff.rs@@ -20,7 +20,7 @@ use std::{collections::{BTreeMap, BinaryHeap},fs::{File, OpenOptions},io::{BufWriter, Write},- path::Path,+ path::{Path, PathBuf},};use fst::{Automaton, IntoStreamer, Streamer};@@ -211,18 +211,20 @@ pub struct StupidBackoff {ngrams: fst::Map<memmap2::Mmap>,rotated_ngrams: fst::Map<memmap2::Mmap>,n_counts: Vec<u64>,+ folder: PathBuf,}impl StupidBackoff {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {+ let mmap = unsafe { memmap2::Mmap::map(&File::open(folder.as_ref().join("ngrams.bin"))?)? };let ngrams = fst::Map::new(mmap)?;- let mmap =- unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };+ let mmap = unsafe {+ memmap2::Mmap::map(&File::open(folder.as_ref().join("rotated_ngrams.bin"))?)?+ };let rotated_ngrams = fst::Map::new(mmap)?;- let file = File::open(path.as_ref().join("n_counts.bin"))?;+ let file = File::open(folder.as_ref().join("n_counts.bin"))?;let mut reader = std::io::BufReader::new(file);let n_counts = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;@@ -230,12 +232,13 @@ impl StupidBackoff {ngrams,rotated_ngrams,n_counts,+ folder: folder.as_ref().to_path_buf(),})}- pub fn merge<P: AsRef<Path>>(models: Vec<Self>, path: P) -> Result<Self> {- if !path.as_ref().exists() {- std::fs::create_dir_all(path.as_ref())?;+ pub fn merge<P: AsRef<Path>>(models: Vec<Self>, folder: P) -> Result<Self> {+ if !folder.as_ref().exists() {+ std::fs::create_dir_all(folder.as_ref())?;}let n_counts = models.iter()@@ -251,7 +254,7 @@ impl StupidBackoff {.create(true).truncate(true).write(true)- .open(path.as_ref().join("n_counts.bin"))?;+ .open(folder.as_ref().join("n_counts.bin"))?;let mut wrt = BufWriter::new(file);bincode::encode_into_std_write(&n_counts, &mut wrt, bincode::config::standard())?;@@ -261,7 +264,7 @@ impl StupidBackoff {.create(true).truncate(true).write(true)- .open(path.as_ref().join("ngrams.bin"))?;+ .open(folder.as_ref().join("ngrams.bin"))?;let wtr = BufWriter::new(file);let builder = fst::MapBuilder::new(wtr)?;@@ -270,14 +273,14 @@ impl StupidBackoff {merge_streams(builder, streams)?;- let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };+ let mmap = unsafe { memmap2::Mmap::map(&File::open(folder.as_ref().join("ngrams.bin"))?)? };let ngrams = fst::Map::new(mmap)?;let file = OpenOptions::new().create(true).truncate(true).write(true)- .open(path.as_ref().join("rotated_ngrams.bin"))?;+ .open(folder.as_ref().join("rotated_ngrams.bin"))?;let wtr = BufWriter::new(file);let builder = fst::MapBuilder::new(wtr)?;@@ -286,14 +289,20 @@ impl StupidBackoff {merge_streams(builder, streams)?;- let mmap =- unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };+ let mmap = unsafe {+ memmap2::Mmap::map(&File::open(folder.as_ref().join("rotated_ngrams.bin"))?)?+ };let rotated_ngrams = fst::Map::new(mmap)?;+ for model in models {+ std::fs::remove_dir_all(model.folder)?;+ }+Ok(Self {ngrams,rotated_ngrams,n_counts,+ folder: folder.as_ref().to_path_buf(),})}diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/core/src/web_spell/term_freqs.rsindex 0c663926..11be993c 100644--- a/crates/core/src/web_spell/term_freqs.rs+++ b/crates/core/src/web_spell/term_freqs.rs@@ -18,6 +18,7 @@ use super::{MergePointer, Result};use fst::{IntoStreamer, Streamer};use std::{+ cmp::Reverse,collections::{BTreeMap, BinaryHeap},fs::{File, OpenOptions},io::BufWriter,@@ -85,12 +86,12 @@ impl StoredDict {})}- fn merge<P: AsRef<Path>>(dicts: Vec<Self>, path: P) -> Result<Self> {+ fn merge<P: AsRef<Path>>(dicts: Vec<Self>, folder: P) -> Result<Self> {let file = OpenOptions::new().create(true).truncate(true).write(true)- .open(path.as_ref())?;+ .open(folder.as_ref())?;let wtr = BufWriter::new(file);let mut builder = fst::MapBuilder::new(wtr)?;@@ -109,49 +110,47 @@ impl StoredDict {pointer.advance();}- while pointers.iter().any(|p| !p.is_finished) {- let mut min_pointer: Option<&MergePointer<'_>> = None;-- for pointer in pointers.iter() {- if pointer.is_finished {- continue;- }-- if let Some(min) = min_pointer {- if pointer.term < min.term {- min_pointer = Some(pointer);+ let mut pointers: BinaryHeap<_> = pointers.into_iter().map(Reverse).collect();++ loop {+ let (term, mut freq, is_finished) = {+ match pointers.peek_mut() {+ Some(mut pointer) => {+ let res = (+ pointer.0.term.clone(),+ pointer.0.value,+ pointer.0.is_finished,+ );+ pointer.0.advance();+ res}- } else {- min_pointer = Some(pointer);+ None => break,}- }-- if let Some(min_pointer) = min_pointer {- let term = min_pointer.term.clone();- let mut freq = 0;+ };- for pointer in pointers.iter_mut() {- if pointer.is_finished {- continue;- }+ if is_finished {+ break;+ }- if pointer.term == term {- freq += pointer.value;- pointer.advance();- }+ while let Some(mut other) = pointers.peek_mut() {+ if other.0.term != term || other.0.is_finished {+ break;}- builder.insert(term, freq)?;+ freq += other.0.value;+ other.0.advance();}++ builder.insert(term, freq)?;}builder.finish()?;- let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref())?)? };+ let mmap = unsafe { memmap2::Mmap::map(&File::open(folder.as_ref())?)? };Ok(StoredDict {map: fst::Map::new(mmap)?,- path: path.as_ref().to_path_buf(),+ path: folder.as_ref().to_path_buf(),})}}@@ -305,6 +304,7 @@ impl TermDict {self.save_meta()?;self.stored.push(merged);+ self.gc()?;Ok(())}@@ -452,6 +452,7 @@ mod tests {dict.insert("foo");dict.insert("bar");dict.insert("foo");+ dict.insert("abc");dict.commit()?;@@ -459,9 +460,10 @@ mod tests {assert_eq!(dict.stored.len(), 1);- assert_eq!(dict.freq("foo"), Some(6));+ assert_eq!(dict.freq("abc"), Some(1));assert_eq!(dict.freq("bar"), Some(4));assert_eq!(dict.freq("baz"), Some(2));+ assert_eq!(dict.freq("foo"), Some(6));Ok(())}diff --git a/crates/core/src/webpage/safety_classifier.rs b/crates/core/src/webpage/safety_classifier.rsindex 8c230025..ceac0791 100644--- a/crates/core/src/webpage/safety_classifier.rs+++ b/crates/core/src/webpage/safety_classifier.rs@@ -16,6 +16,7 @@use std::fmt::Display;use std::fs::OpenOptions;+use std::io::Write;use std::path::Path;use itertools::Itertools;@@ -187,6 +188,7 @@ impl Model {.open(path)?;bincode::encode_into_std_write(&self, &mut file, bincode::config::standard())?;+ file.flush()?;Ok(())}diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rsindex e3c6e0e9..e6758ca0 100644--- a/crates/speedy-kv/src/segment.rs+++ b/crates/speedy-kv/src/segment.rs@@ -80,6 +80,7 @@ wherelet mut wrt = BufWriter::new(self.writers.bloom);bincode::encode_into_std_write(self.bloom, &mut wrt, bincode::config::standard())?;+ wrt.flush()?;Ok(())}