Skip to content

web search done right

more reliable region detection

Mikkel Denker70d3aa19e423
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index e2c03061..088331bd 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,6 +1,6 @@
warc_paths_file = "warc.paths"
centrality_store_path = "./data/centrality"
-limit_warc_files = 10
+limit_warc_files = 1
[warc_source]
# type = "Local"
diff --git a/data.tar.gz b/data.tar.gz
index 9a1e6ea4..4fc2f5a4 100644
--- a/data.tar.gz
+++ b/data.tar.gz
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:1cadc250585c97a68bdfcfda2e341c30235f3369b0c72f2ff898c3473f0417ea
-size 968695137
+oid sha256:3e9a72086157a1ab2550265435838f7cd61cae8173f71da62aaaa225061859a1
+size 1049395440
diff --git a/src/webpage/region.rs b/src/webpage/region.rs
index b1953a4a..ac0ea32a 100644
--- a/src/webpage/region.rs
+++ b/src/webpage/region.rs
@@ -92,14 +92,20 @@ impl Region {
.or_else(|| webpage.html.all_text())
.or_else(|| webpage.html.title())
{
- Some(text) => whatlang::detect_lang(&text)
- .map(|lang| match lang {
- whatlang::Lang::Eng => Ok(Region::US),
- whatlang::Lang::Spa => Ok(Region::Spain),
- whatlang::Lang::Fra => Ok(Region::France),
- whatlang::Lang::Deu => Ok(Region::Germany),
- whatlang::Lang::Dan => Ok(Region::Denmark),
- _ => Err(Error::UnknownRegion),
+ Some(text) => whatlang::detect(&text)
+ .and_then(|info| {
+ if info.is_reliable() && info.confidence() > 0.95 {
+ match info.lang() {
+ whatlang::Lang::Eng => Some(Ok(Region::US)),
+ whatlang::Lang::Spa => Some(Ok(Region::Spain)),
+ whatlang::Lang::Fra => Some(Ok(Region::France)),
+ whatlang::Lang::Deu => Some(Ok(Region::Germany)),
+ whatlang::Lang::Dan => Some(Ok(Region::Denmark)),
+ _ => Some(Err(Error::UnknownRegion)),
+ }
+ } else {
+ None
+ }
})
.unwrap_or(Err(Error::UnknownRegion)),
None => Err(Error::UnknownRegion),