web search done right
fixed bug that caused error model in spell correction to always be empty
Mikkel Denkera1381d667b23
diff --git a/crates/core/src/web_spell/error_model.rs b/crates/core/src/web_spell/error_model.rsindex dc8e1834..79ae2d69 100644--- a/crates/core/src/web_spell/error_model.rs+++ b/crates/core/src/web_spell/error_model.rs@@ -223,8 +223,8 @@ impl ErrorModel {pub fn log_prob(&self, error: &ErrorSequence) -> f64 {match self.errors.get(error) {- Some(count) => (*count as f64).log2() - (self.total as f64).log2(),- None => 0.0 - (self.total as f64).log2(),+ Some(count) => (*count as f64).log2() - ((self.total + 1) as f64).log2(),+ None => 0.0 - ((self.total + 1) as f64).log2(),}}}diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/core/src/web_spell/term_freqs.rsindex 11be993c..21fbf579 100644--- a/crates/core/src/web_spell/term_freqs.rs+++ b/crates/core/src/web_spell/term_freqs.rs@@ -324,61 +324,6 @@ impl TermDict {freqs}- pub fn prune(&mut self, top_n_terms: usize) -> Result<()> {- let mut top_term_freqs: BinaryHeap<u64> = BinaryHeap::new();-- for stored in self.stored.iter() {- let mut stream = stored.map.stream();-- while let Some((_, freq)) = stream.next() {- if top_term_freqs.len() < top_n_terms {- top_term_freqs.push(freq);- } else if let Some(mut min) = top_term_freqs.peek_mut() {- if freq > *min {- *min = freq;- }- }- }- }-- if top_term_freqs.len() < top_n_terms {- return Ok(());- }-- let lowest = top_term_freqs.into_sorted_vec().pop().unwrap();-- self.metadata.dicts.clear();- for stored in self.stored.iter_mut() {- let uuid = uuid::Uuid::new_v4();- let file = OpenOptions::new()- .create(true)- .truncate(true)- .write(true)- .open(self.path.join(format!("{}.dict", uuid)))?;-- let wtr = BufWriter::new(file);-- let mut builder = fst::MapBuilder::new(wtr)?;-- let mut stream = stored.map.stream();- while let Some((term, freq)) = stream.next() {- if freq >= lowest {- builder.insert(term, freq)?;- }- }-- self.metadata.dicts.push(uuid);-- builder.finish()?;-- *stored = StoredDict::open(self.path.join(format!("{}.dict", uuid)))?;- }-- self.save_meta()?;-- Ok(())- }-pub fn terms(&self) -> Vec<String> {let mut terms = Vec::new();diff --git a/crates/core/src/web_spell/trainer.rs b/crates/core/src/web_spell/trainer.rsindex 7cc3c1db..4a41ab10 100644--- a/crates/core/src/web_spell/trainer.rs+++ b/crates/core/src/web_spell/trainer.rs@@ -103,7 +103,6 @@ impl SecondTrainer {let mut term_dict = TermDict::open(path.as_ref().join("term_dict"))?;term_dict.merge_dicts()?;- term_dict.prune(10_000_000)?;Ok(Self {term_dict,@@ -159,7 +158,7 @@ impl SecondTrainer {}},)- .filter(|c| c.len() != 3)+ .filter(|c| c.len() == 3).map(|context| {let most_probable = possible_corrections.iter()diff --git a/justfile b/justfileindex 4ae55f46..829e6050 100644--- a/justfile+++ b/justfile@@ -14,8 +14,6 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")cargo watch -i frontend -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"@dev-webgraph:cargo watch -i frontend -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/host_server.toml" && cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/page_server.toml"-@dev-llm:- ./.venv/bin/python3 -m llama_cpp.server --model data/mistral-7b-instruct-v0.2.Q4_K_M.gguf --port 4000@dev-frontend:cd frontend && npm run devdiff --git a/scripts/run_dev.py b/scripts/run_dev.pyindex 3a5aec0f..9cfb6236 100755--- a/scripts/run_dev.py+++ b/scripts/run_dev.py@@ -19,7 +19,6 @@ processes.append(subprocess.Popen(["just", "dev-search-server"]))processes.append(subprocess.Popen(["just", "dev-entity-search-server"]))processes.append(subprocess.Popen(["just", "dev-webgraph"]))processes.append(subprocess.Popen(["just", "dev-frontend"]))-processes.append(subprocess.Popen(["just", "dev-llm"]))# kill processes on ctrl-cimport time