Skip to content

web search done right

move thesaurus into widgets module

Mikkel Denkerd5242a279991
diff --git a/core/src/api/docs.rs b/core/src/api/docs.rs
index 80c19975..d5de1841 100644
--- a/core/src/api/docs.rs
+++ b/core/src/api/docs.rs
@@ -64,13 +64,13 @@ use utoipa_swagger_ui::SwaggerUi;
crate::widgets::Widget,
crate::widgets::calculator::Calculation,
- crate::thesaurus::ThesaurusWidget,
- crate::thesaurus::Lemma,
- crate::thesaurus::WordMeaning,
- crate::thesaurus::Definition,
- crate::thesaurus::Example,
- crate::thesaurus::PartOfSpeech,
- crate::thesaurus::PartOfSpeechMeaning,
+ crate::widgets::thesaurus::ThesaurusWidget,
+ crate::widgets::thesaurus::Lemma,
+ crate::widgets::thesaurus::WordMeaning,
+ crate::widgets::thesaurus::Definition,
+ crate::widgets::thesaurus::Example,
+ crate::widgets::thesaurus::PartOfSpeech,
+ crate::widgets::thesaurus::PartOfSpeechMeaning,
crate::ranking::signal::SignalScore,
crate::bangs::BangHit,
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index d871eb21..1c737533 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -594,7 +594,7 @@ impl CrawlDb {
let domains: DashMap<Domain, Vec<UrlToInsert>> = DashMap::new();
responses.par_iter().for_each(|res| {
- let urls: Vec<(Domain, Url)> = res
+ let mut urls: Vec<(Domain, Url)> = res
.discovered_urls
.iter()
.filter_map(|url| {
@@ -614,11 +614,13 @@ impl CrawlDb {
.iter()
.filter(|(domain, _)| res.domain != *domain)
.count() as f64;
- let mut used_budget = 0.0;
- for url in &res.discovered_urls {
- let domain = Domain::from(url);
+ urls.sort_unstable_by(|(_, a), (_, b)| a.as_str().cmp(b.as_str()));
+ urls.dedup_by(|(_, a), (_, b)| a.as_str() == b.as_str());
+
+ let mut used_budget = 0.0;
+ for (domain, url) in urls {
for tld in BLOCKED_TLD.iter() {
if domain.as_str().ends_with(tld) {
continue;
@@ -635,10 +637,10 @@ impl CrawlDb {
used_budget += weight;
- domains.entry(domain).or_default().push(UrlToInsert {
- url: url.clone(),
- weight,
- });
+ domains
+ .entry(domain)
+ .or_default()
+ .push(UrlToInsert { url, weight });
}
let mut domain_state = self
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 493cecd7..b2e94909 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -73,7 +73,6 @@ mod snippet;
pub mod spell;
mod subdomain_count;
pub mod summarizer;
-mod thesaurus;
mod tokenizer;
#[allow(unused)]
mod ttl_cache;
diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rs
index 86328133..325e0518 100644
--- a/core/src/widgets/mod.rs
+++ b/core/src/widgets/mod.rs
@@ -18,15 +18,14 @@ use serde::{Deserialize, Serialize};
use thiserror::Error;
use utoipa::ToSchema;
-use crate::{
- config::WidgetsConfig,
- thesaurus::{self, ThesaurusWidget},
-};
+use self::thesaurus::ThesaurusWidget;
+use crate::config::WidgetsConfig;
use self::calculator::{Calculation, Calculator};
use anyhow::{anyhow, Result};
pub mod calculator;
+pub mod thesaurus;
#[derive(Error, Debug)]
pub enum Error {
diff --git a/core/src/thesaurus.rs b/core/src/widgets/thesaurus.rs
similarity index 99%
rename from core/src/thesaurus.rs
rename to core/src/widgets/thesaurus.rs
index 51f467fa..791d27f4 100644
--- a/core/src/thesaurus.rs
+++ b/core/src/widgets/thesaurus.rs
@@ -184,6 +184,8 @@ impl Dictionary {
Regex::new(r"^([\w| ]+) meaning$")?,
Regex::new(r"^([\w| ]+) synonym$")?,
Regex::new(r"^([\w| ]+) synonyms$")?,
+ //
+ Regex::new(r"^define ([\w| ]+)$")?,
],
})
}