Skip to content

web search done right

web-spell cleanup old dicts after merge

Mikkel Denker38416c607071
diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rs
index 97959af6..9b047c9b 100644
--- a/crates/core/src/web_spell/mod.rs
+++ b/crates/core/src/web_spell/mod.rs
@@ -220,8 +220,8 @@ impl<'a> Ord for MergePointer<'a> {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
match (self.is_finished, other.is_finished) {
(true, true) | (false, false) => self.term.cmp(&other.term),
- (true, false) => std::cmp::Ordering::Less,
- (false, true) => std::cmp::Ordering::Greater,
+ (true, false) => std::cmp::Ordering::Greater,
+ (false, true) => std::cmp::Ordering::Less,
}
}
}
diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rs
index a19c5a14..6981176c 100644
--- a/crates/core/src/web_spell/stupid_backoff.rs
+++ b/crates/core/src/web_spell/stupid_backoff.rs
@@ -20,7 +20,7 @@ use std::{
collections::{BTreeMap, BinaryHeap},
fs::{File, OpenOptions},
io::{BufWriter, Write},
- path::Path,
+ path::{Path, PathBuf},
};
use fst::{Automaton, IntoStreamer, Streamer};
@@ -211,18 +211,20 @@ pub struct StupidBackoff {
ngrams: fst::Map<memmap2::Mmap>,
rotated_ngrams: fst::Map<memmap2::Mmap>,
n_counts: Vec<u64>,
+ folder: PathBuf,
}
impl StupidBackoff {
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
- let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
+ let mmap = unsafe { memmap2::Mmap::map(&File::open(folder.as_ref().join("ngrams.bin"))?)? };
let ngrams = fst::Map::new(mmap)?;
- let mmap =
- unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };
+ let mmap = unsafe {
+ memmap2::Mmap::map(&File::open(folder.as_ref().join("rotated_ngrams.bin"))?)?
+ };
let rotated_ngrams = fst::Map::new(mmap)?;
- let file = File::open(path.as_ref().join("n_counts.bin"))?;
+ let file = File::open(folder.as_ref().join("n_counts.bin"))?;
let mut reader = std::io::BufReader::new(file);
let n_counts = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;
@@ -230,12 +232,13 @@ impl StupidBackoff {
ngrams,
rotated_ngrams,
n_counts,
+ folder: folder.as_ref().to_path_buf(),
})
}
- pub fn merge<P: AsRef<Path>>(models: Vec<Self>, path: P) -> Result<Self> {
- if !path.as_ref().exists() {
- std::fs::create_dir_all(path.as_ref())?;
+ pub fn merge<P: AsRef<Path>>(models: Vec<Self>, folder: P) -> Result<Self> {
+ if !folder.as_ref().exists() {
+ std::fs::create_dir_all(folder.as_ref())?;
}
let n_counts = models
.iter()
@@ -251,7 +254,7 @@ impl StupidBackoff {
.create(true)
.truncate(true)
.write(true)
- .open(path.as_ref().join("n_counts.bin"))?;
+ .open(folder.as_ref().join("n_counts.bin"))?;
let mut wrt = BufWriter::new(file);
bincode::encode_into_std_write(&n_counts, &mut wrt, bincode::config::standard())?;
@@ -261,7 +264,7 @@ impl StupidBackoff {
.create(true)
.truncate(true)
.write(true)
- .open(path.as_ref().join("ngrams.bin"))?;
+ .open(folder.as_ref().join("ngrams.bin"))?;
let wtr = BufWriter::new(file);
let builder = fst::MapBuilder::new(wtr)?;
@@ -270,14 +273,14 @@ impl StupidBackoff {
merge_streams(builder, streams)?;
- let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("ngrams.bin"))?)? };
+ let mmap = unsafe { memmap2::Mmap::map(&File::open(folder.as_ref().join("ngrams.bin"))?)? };
let ngrams = fst::Map::new(mmap)?;
let file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
- .open(path.as_ref().join("rotated_ngrams.bin"))?;
+ .open(folder.as_ref().join("rotated_ngrams.bin"))?;
let wtr = BufWriter::new(file);
let builder = fst::MapBuilder::new(wtr)?;
@@ -286,14 +289,20 @@ impl StupidBackoff {
merge_streams(builder, streams)?;
- let mmap =
- unsafe { memmap2::Mmap::map(&File::open(path.as_ref().join("rotated_ngrams.bin"))?)? };
+ let mmap = unsafe {
+ memmap2::Mmap::map(&File::open(folder.as_ref().join("rotated_ngrams.bin"))?)?
+ };
let rotated_ngrams = fst::Map::new(mmap)?;
+ for model in models {
+ std::fs::remove_dir_all(model.folder)?;
+ }
+
Ok(Self {
ngrams,
rotated_ngrams,
n_counts,
+ folder: folder.as_ref().to_path_buf(),
})
}
diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/core/src/web_spell/term_freqs.rs
index 0c663926..11be993c 100644
--- a/crates/core/src/web_spell/term_freqs.rs
+++ b/crates/core/src/web_spell/term_freqs.rs
@@ -18,6 +18,7 @@ use super::{MergePointer, Result};
use fst::{IntoStreamer, Streamer};
use std::{
+ cmp::Reverse,
collections::{BTreeMap, BinaryHeap},
fs::{File, OpenOptions},
io::BufWriter,
@@ -85,12 +86,12 @@ impl StoredDict {
})
}
- fn merge<P: AsRef<Path>>(dicts: Vec<Self>, path: P) -> Result<Self> {
+ fn merge<P: AsRef<Path>>(dicts: Vec<Self>, folder: P) -> Result<Self> {
let file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
- .open(path.as_ref())?;
+ .open(folder.as_ref())?;
let wtr = BufWriter::new(file);
let mut builder = fst::MapBuilder::new(wtr)?;
@@ -109,49 +110,47 @@ impl StoredDict {
pointer.advance();
}
- while pointers.iter().any(|p| !p.is_finished) {
- let mut min_pointer: Option<&MergePointer<'_>> = None;
-
- for pointer in pointers.iter() {
- if pointer.is_finished {
- continue;
- }
-
- if let Some(min) = min_pointer {
- if pointer.term < min.term {
- min_pointer = Some(pointer);
+ let mut pointers: BinaryHeap<_> = pointers.into_iter().map(Reverse).collect();
+
+ loop {
+ let (term, mut freq, is_finished) = {
+ match pointers.peek_mut() {
+ Some(mut pointer) => {
+ let res = (
+ pointer.0.term.clone(),
+ pointer.0.value,
+ pointer.0.is_finished,
+ );
+ pointer.0.advance();
+ res
}
- } else {
- min_pointer = Some(pointer);
+ None => break,
}
- }
-
- if let Some(min_pointer) = min_pointer {
- let term = min_pointer.term.clone();
- let mut freq = 0;
+ };
- for pointer in pointers.iter_mut() {
- if pointer.is_finished {
- continue;
- }
+ if is_finished {
+ break;
+ }
- if pointer.term == term {
- freq += pointer.value;
- pointer.advance();
- }
+ while let Some(mut other) = pointers.peek_mut() {
+ if other.0.term != term || other.0.is_finished {
+ break;
}
- builder.insert(term, freq)?;
+ freq += other.0.value;
+ other.0.advance();
}
+
+ builder.insert(term, freq)?;
}
builder.finish()?;
- let mmap = unsafe { memmap2::Mmap::map(&File::open(path.as_ref())?)? };
+ let mmap = unsafe { memmap2::Mmap::map(&File::open(folder.as_ref())?)? };
Ok(StoredDict {
map: fst::Map::new(mmap)?,
- path: path.as_ref().to_path_buf(),
+ path: folder.as_ref().to_path_buf(),
})
}
}
@@ -305,6 +304,7 @@ impl TermDict {
self.save_meta()?;
self.stored.push(merged);
+ self.gc()?;
Ok(())
}
@@ -452,6 +452,7 @@ mod tests {
dict.insert("foo");
dict.insert("bar");
dict.insert("foo");
+ dict.insert("abc");
dict.commit()?;
@@ -459,9 +460,10 @@ mod tests {
assert_eq!(dict.stored.len(), 1);
- assert_eq!(dict.freq("foo"), Some(6));
+ assert_eq!(dict.freq("abc"), Some(1));
assert_eq!(dict.freq("bar"), Some(4));
assert_eq!(dict.freq("baz"), Some(2));
+ assert_eq!(dict.freq("foo"), Some(6));
Ok(())
}
diff --git a/crates/core/src/webpage/safety_classifier.rs b/crates/core/src/webpage/safety_classifier.rs
index 8c230025..ceac0791 100644
--- a/crates/core/src/webpage/safety_classifier.rs
+++ b/crates/core/src/webpage/safety_classifier.rs
@@ -16,6 +16,7 @@
use std::fmt::Display;
use std::fs::OpenOptions;
+use std::io::Write;
use std::path::Path;
use itertools::Itertools;
@@ -187,6 +188,7 @@ impl Model {
.open(path)?;
bincode::encode_into_std_write(&self, &mut file, bincode::config::standard())?;
+ file.flush()?;
Ok(())
}
diff --git a/crates/speedy-kv/src/segment.rs b/crates/speedy-kv/src/segment.rs
index e3c6e0e9..e6758ca0 100644
--- a/crates/speedy-kv/src/segment.rs
+++ b/crates/speedy-kv/src/segment.rs
@@ -80,6 +80,7 @@ where
let mut wrt = BufWriter::new(self.writers.bloom);
bincode::encode_into_std_write(self.bloom, &mut wrt, bincode::config::standard())?;
+ wrt.flush()?;
Ok(())
}