web search done right
move thesaurus into widgets module
Mikkel Denkerd5242a279991
diff --git a/core/src/api/docs.rs b/core/src/api/docs.rsindex 80c19975..d5de1841 100644--- a/core/src/api/docs.rs+++ b/core/src/api/docs.rs@@ -64,13 +64,13 @@ use utoipa_swagger_ui::SwaggerUi;crate::widgets::Widget,crate::widgets::calculator::Calculation,- crate::thesaurus::ThesaurusWidget,- crate::thesaurus::Lemma,- crate::thesaurus::WordMeaning,- crate::thesaurus::Definition,- crate::thesaurus::Example,- crate::thesaurus::PartOfSpeech,- crate::thesaurus::PartOfSpeechMeaning,+ crate::widgets::thesaurus::ThesaurusWidget,+ crate::widgets::thesaurus::Lemma,+ crate::widgets::thesaurus::WordMeaning,+ crate::widgets::thesaurus::Definition,+ crate::widgets::thesaurus::Example,+ crate::widgets::thesaurus::PartOfSpeech,+ crate::widgets::thesaurus::PartOfSpeechMeaning,crate::ranking::signal::SignalScore,crate::bangs::BangHit,diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex d871eb21..1c737533 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -594,7 +594,7 @@ impl CrawlDb {let domains: DashMap<Domain, Vec<UrlToInsert>> = DashMap::new();responses.par_iter().for_each(|res| {- let urls: Vec<(Domain, Url)> = res+ let mut urls: Vec<(Domain, Url)> = res.discovered_urls.iter().filter_map(|url| {@@ -614,11 +614,13 @@ impl CrawlDb {.iter().filter(|(domain, _)| res.domain != *domain).count() as f64;- let mut used_budget = 0.0;- for url in &res.discovered_urls {- let domain = Domain::from(url);+ urls.sort_unstable_by(|(_, a), (_, b)| a.as_str().cmp(b.as_str()));+ urls.dedup_by(|(_, a), (_, b)| a.as_str() == b.as_str());++ let mut used_budget = 0.0;+ for (domain, url) in urls {for tld in BLOCKED_TLD.iter() {if domain.as_str().ends_with(tld) {continue;@@ -635,10 +637,10 @@ impl CrawlDb {used_budget += weight;- domains.entry(domain).or_default().push(UrlToInsert {- url: url.clone(),- weight,- });+ domains+ .entry(domain)+ .or_default()+ .push(UrlToInsert { url, weight });}let mut domain_state = selfdiff --git a/core/src/lib.rs b/core/src/lib.rsindex 493cecd7..b2e94909 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -73,7 +73,6 @@ mod snippet;pub mod spell;mod subdomain_count;pub mod summarizer;-mod thesaurus;mod tokenizer;#[allow(unused)]mod ttl_cache;diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rsindex 86328133..325e0518 100644--- a/core/src/widgets/mod.rs+++ b/core/src/widgets/mod.rs@@ -18,15 +18,14 @@ use serde::{Deserialize, Serialize};use thiserror::Error;use utoipa::ToSchema;-use crate::{- config::WidgetsConfig,- thesaurus::{self, ThesaurusWidget},-};+use self::thesaurus::ThesaurusWidget;+use crate::config::WidgetsConfig;use self::calculator::{Calculation, Calculator};use anyhow::{anyhow, Result};pub mod calculator;+pub mod thesaurus;#[derive(Error, Debug)]pub enum Error {diff --git a/core/src/thesaurus.rs b/core/src/widgets/thesaurus.rssimilarity index 99%rename from core/src/thesaurus.rsrename to core/src/widgets/thesaurus.rsindex 51f467fa..791d27f4 100644--- a/core/src/thesaurus.rs+++ b/core/src/widgets/thesaurus.rs@@ -184,6 +184,8 @@ impl Dictionary {Regex::new(r"^([\w| ]+) meaning$")?,Regex::new(r"^([\w| ]+) synonym$")?,Regex::new(r"^([\w| ]+) synonyms$")?,+ //+ Regex::new(r"^define ([\w| ]+)$")?,],})}