Skip to content

web search done right

fixed bug that caused error model in spell correction to always be empty

Mikkel Denkera1381d667b23
diff --git a/crates/core/src/web_spell/error_model.rs b/crates/core/src/web_spell/error_model.rs
index dc8e1834..79ae2d69 100644
--- a/crates/core/src/web_spell/error_model.rs
+++ b/crates/core/src/web_spell/error_model.rs
@@ -223,8 +223,8 @@ impl ErrorModel {
pub fn log_prob(&self, error: &ErrorSequence) -> f64 {
match self.errors.get(error) {
- Some(count) => (*count as f64).log2() - (self.total as f64).log2(),
- None => 0.0 - (self.total as f64).log2(),
+ Some(count) => (*count as f64).log2() - ((self.total + 1) as f64).log2(),
+ None => 0.0 - ((self.total + 1) as f64).log2(),
}
}
}
diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/core/src/web_spell/term_freqs.rs
index 11be993c..21fbf579 100644
--- a/crates/core/src/web_spell/term_freqs.rs
+++ b/crates/core/src/web_spell/term_freqs.rs
@@ -324,61 +324,6 @@ impl TermDict {
freqs
}
- pub fn prune(&mut self, top_n_terms: usize) -> Result<()> {
- let mut top_term_freqs: BinaryHeap<u64> = BinaryHeap::new();
-
- for stored in self.stored.iter() {
- let mut stream = stored.map.stream();
-
- while let Some((_, freq)) = stream.next() {
- if top_term_freqs.len() < top_n_terms {
- top_term_freqs.push(freq);
- } else if let Some(mut min) = top_term_freqs.peek_mut() {
- if freq > *min {
- *min = freq;
- }
- }
- }
- }
-
- if top_term_freqs.len() < top_n_terms {
- return Ok(());
- }
-
- let lowest = top_term_freqs.into_sorted_vec().pop().unwrap();
-
- self.metadata.dicts.clear();
- for stored in self.stored.iter_mut() {
- let uuid = uuid::Uuid::new_v4();
- let file = OpenOptions::new()
- .create(true)
- .truncate(true)
- .write(true)
- .open(self.path.join(format!("{}.dict", uuid)))?;
-
- let wtr = BufWriter::new(file);
-
- let mut builder = fst::MapBuilder::new(wtr)?;
-
- let mut stream = stored.map.stream();
- while let Some((term, freq)) = stream.next() {
- if freq >= lowest {
- builder.insert(term, freq)?;
- }
- }
-
- self.metadata.dicts.push(uuid);
-
- builder.finish()?;
-
- *stored = StoredDict::open(self.path.join(format!("{}.dict", uuid)))?;
- }
-
- self.save_meta()?;
-
- Ok(())
- }
-
pub fn terms(&self) -> Vec<String> {
let mut terms = Vec::new();
diff --git a/crates/core/src/web_spell/trainer.rs b/crates/core/src/web_spell/trainer.rs
index 7cc3c1db..4a41ab10 100644
--- a/crates/core/src/web_spell/trainer.rs
+++ b/crates/core/src/web_spell/trainer.rs
@@ -103,7 +103,6 @@ impl SecondTrainer {
let mut term_dict = TermDict::open(path.as_ref().join("term_dict"))?;
term_dict.merge_dicts()?;
- term_dict.prune(10_000_000)?;
Ok(Self {
term_dict,
@@ -159,7 +158,7 @@ impl SecondTrainer {
}
},
)
- .filter(|c| c.len() != 3)
+ .filter(|c| c.len() == 3)
.map(|context| {
let most_probable = possible_corrections
.iter()
diff --git a/justfile b/justfile
index 4ae55f46..829e6050 100644
--- a/justfile
+++ b/justfile
@@ -14,8 +14,6 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
cargo watch -i frontend -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"
@dev-webgraph:
cargo watch -i frontend -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/host_server.toml" && cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/page_server.toml"
-@dev-llm:
- ./.venv/bin/python3 -m llama_cpp.server --model data/mistral-7b-instruct-v0.2.Q4_K_M.gguf --port 4000
@dev-frontend:
cd frontend && npm run dev
diff --git a/scripts/run_dev.py b/scripts/run_dev.py
index 3a5aec0f..9cfb6236 100755
--- a/scripts/run_dev.py
+++ b/scripts/run_dev.py
@@ -19,7 +19,6 @@ processes.append(subprocess.Popen(["just", "dev-search-server"]))
processes.append(subprocess.Popen(["just", "dev-entity-search-server"]))
processes.append(subprocess.Popen(["just", "dev-webgraph"]))
processes.append(subprocess.Popen(["just", "dev-frontend"]))
-processes.append(subprocess.Popen(["just", "dev-llm"]))
# kill processes on ctrl-c
import time