web search done right
more reliable region detection
Mikkel Denker70d3aa19e423
diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex e2c03061..088331bd 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,6 +1,6 @@warc_paths_file = "warc.paths"centrality_store_path = "./data/centrality"-limit_warc_files = 10+limit_warc_files = 1[warc_source]# type = "Local"diff --git a/data.tar.gz b/data.tar.gzindex 9a1e6ea4..4fc2f5a4 100644--- a/data.tar.gz+++ b/data.tar.gz@@ -1,3 +1,3 @@version https://git-lfs.github.com/spec/v1-oid sha256:1cadc250585c97a68bdfcfda2e341c30235f3369b0c72f2ff898c3473f0417ea-size 968695137+oid sha256:3e9a72086157a1ab2550265435838f7cd61cae8173f71da62aaaa225061859a1+size 1049395440diff --git a/src/webpage/region.rs b/src/webpage/region.rsindex b1953a4a..ac0ea32a 100644--- a/src/webpage/region.rs+++ b/src/webpage/region.rs@@ -92,14 +92,20 @@ impl Region {.or_else(|| webpage.html.all_text()).or_else(|| webpage.html.title()){- Some(text) => whatlang::detect_lang(&text)- .map(|lang| match lang {- whatlang::Lang::Eng => Ok(Region::US),- whatlang::Lang::Spa => Ok(Region::Spain),- whatlang::Lang::Fra => Ok(Region::France),- whatlang::Lang::Deu => Ok(Region::Germany),- whatlang::Lang::Dan => Ok(Region::Denmark),- _ => Err(Error::UnknownRegion),+ Some(text) => whatlang::detect(&text)+ .and_then(|info| {+ if info.is_reliable() && info.confidence() > 0.95 {+ match info.lang() {+ whatlang::Lang::Eng => Some(Ok(Region::US)),+ whatlang::Lang::Spa => Some(Ok(Region::Spain)),+ whatlang::Lang::Fra => Some(Ok(Region::France)),+ whatlang::Lang::Deu => Some(Ok(Region::Germany)),+ whatlang::Lang::Dan => Some(Ok(Region::Denmark)),+ _ => Some(Err(Error::UnknownRegion)),+ }+ } else {+ None+ }}).unwrap_or(Err(Error::UnknownRegion)),None => Err(Error::UnknownRegion),