web search done right
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{collections::HashMap, fs::File, io::Write, path::Path};
use utoipa::ToSchema;
use crate::{Error, Result};
use super::Webpage;
#[derive(
Debug,
PartialEq,
Eq,
Clone,
Copy,
Hash,
serde::Serialize,
serde::Deserialize,
bincode::Encode,
bincode::Decode,
Default,
ToSchema,
)]
pub enum Region {
#[default]
All,
Denmark,
France,
Germany,
Spain,
US,
}
pub const ALL_REGIONS: [Region; 6] = [
Region::All,
Region::Denmark,
Region::France,
Region::Germany,
Region::Spain,
Region::US,
];
impl Region {
pub fn name(&self) -> String {
// TODO: Translate these strings.
// The names are used in the UI. In the future we should
// actually use region, but right now we only detect region
// from the language of the page, so it makes more sense to
// just use the language name.
match self {
Region::All => "All Languages".to_string(),
Region::Denmark => "Danish".to_string(),
Region::France => "French".to_string(),
Region::Germany => "German".to_string(),
Region::Spain => "Spainish".to_string(),
Region::US => "English".to_string(),
}
}
pub fn gl(&self) -> String {
match self {
Region::All => "all".to_string(),
Region::Denmark => "dk".to_string(),
Region::France => "fr".to_string(),
Region::Germany => "ger".to_string(),
Region::Spain => "spa".to_string(),
Region::US => "us".to_string(),
}
}
pub fn id(&self) -> u64 {
ALL_REGIONS
.iter()
.enumerate()
.find(|(_, region)| self == *region)
.map(|(id, _)| id as u64)
.unwrap()
}
pub fn from_gl(gl: &str) -> Result<Self> {
match gl {
"all" => Ok(Region::All),
"dk" => Ok(Region::Denmark),
"fr" => Ok(Region::France),
"ger" => Ok(Region::Germany),
"spa" => Ok(Region::Spain),
"us" => Ok(Region::US),
_ => Err(Error::UnknownRegion.into()),
}
}
pub fn guess_from(webpage: &Webpage) -> Result<Self> {
match webpage.html.lang() {
Some(lang) => match lang {
whatlang::Lang::Eng => Ok(Region::US),
whatlang::Lang::Spa => Ok(Region::Spain),
whatlang::Lang::Fra => Ok(Region::France),
whatlang::Lang::Deu => Ok(Region::Germany),
whatlang::Lang::Dan => Ok(Region::Denmark),
_ => Err(Error::UnknownRegion.into()),
},
None => Err(Error::UnknownRegion.into()),
}
}
pub fn from_id(doc: u64) -> Self {
ALL_REGIONS[doc as usize]
}
pub fn lang(&self) -> Option<whatlang::Lang> {
match self {
Region::Denmark => Some(whatlang::Lang::Dan),
Region::France => Some(whatlang::Lang::Fra),
Region::Germany => Some(whatlang::Lang::Deu),
Region::Spain => Some(whatlang::Lang::Spa),
Region::US => Some(whatlang::Lang::Eng),
Region::All => None,
}
}
}
#[derive(
serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default, Clone,
)]
pub struct RegionCount {
map: HashMap<Region, u64>,
fast_count: Vec<Option<u64>>,
total_counts: u64,
path: String,
}
impl RegionCount {
pub fn open<P: AsRef<Path>>(path: P) -> Self {
let map: HashMap<Region, u64> = if !path.as_ref().exists() {
if let Some(parent) = path.as_ref().parent() {
std::fs::create_dir_all(parent).unwrap();
}
File::create(path.as_ref()).unwrap();
HashMap::new()
} else {
let json = std::fs::read_to_string(path.as_ref()).unwrap_or_default();
serde_json::from_str(&json).unwrap_or_else(|_| HashMap::new())
};
let mut fast_count = Vec::new();
for (region, count) in &map {
let idx = region.id() as usize;
while idx >= fast_count.len() {
fast_count.push(None);
}
fast_count[idx] = Some(*count);
}
RegionCount {
total_counts: map.values().sum(),
map,
fast_count,
path: path.as_ref().to_str().unwrap().to_string(),
}
}
pub fn increment(&mut self, region: &Region) {
let entry = self.map.entry(*region).or_insert(0);
self.total_counts += 1;
*entry += 1;
}
pub fn commit(&mut self) {
let json = serde_json::to_string(&self.map).unwrap();
let mut file = File::options().write(true).open(&self.path).unwrap();
file.write_all(json.as_bytes()).unwrap();
self.total_counts = self.map.values().sum();
let mut fast_count = Vec::new();
for (region, count) in &self.map {
let idx = region.id() as usize;
while idx >= fast_count.len() {
fast_count.push(None);
}
fast_count[idx] = Some(*count);
}
self.fast_count = fast_count;
}
pub fn merge(&mut self, other: Self) {
for (region, count) in other.map {
*self.map.entry(region).or_insert(0) += count;
}
std::fs::remove_file(other.path).unwrap();
self.commit()
}
pub fn score(&self, region: &Region) -> f64 {
self.fast_count
.get(region.id() as usize)
.map(|count| match count {
Some(count) => *count as f64 / self.total_counts as f64,
None => 0.0,
})
.unwrap_or(0.0)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn simple() {
let temp_dir = file_store::gen_temp_dir().unwrap();
let mut a = RegionCount::open(temp_dir.as_ref().join("a").join("region_count.json"));
a.increment(&Region::Denmark);
a.increment(&Region::Denmark);
a.increment(&Region::US);
let mut b = RegionCount::open(temp_dir.as_ref().join("b").join("region_count.json"));
b.increment(&Region::US);
b.increment(&Region::Germany);
a.merge(b);
assert_eq!(a.map.get(&Region::Denmark), Some(&2));
assert_eq!(a.map.get(&Region::US), Some(&2));
assert_eq!(a.map.get(&Region::Germany), Some(&1));
assert_eq!(a.score(&Region::Denmark), 0.4);
assert_eq!(a.score(&Region::France), 0.0);
}
}