web search done right
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{
collections::HashMap,
fs,
ops::{Deref, DerefMut},
path::Path,
};
use itertools::intersperse;
use url::Url;
use utoipa::ToSchema;
use crate::query::parser::Term;
pub const BANG_PREFIXES: [char; 2] = ['!', '!'];
#[derive(
Clone,
Debug,
PartialEq,
Eq,
serde::Serialize,
serde::Deserialize,
bincode::Encode,
bincode::Decode,
ToSchema,
)]
#[serde(rename_all = "camelCase")]
pub struct Bang {
#[serde(rename = "c")]
pub(crate) category: Option<String>,
#[serde(rename = "sc")]
pub(crate) sub_category: Option<String>,
#[serde(rename = "d")]
pub(crate) domain: Option<String>,
#[serde(rename = "r")]
pub(crate) ranking: Option<usize>,
#[serde(rename = "s")]
pub(crate) site: Option<String>,
#[serde(rename = "t")]
pub(crate) tag: String,
#[serde(rename = "u")]
pub(crate) url: String,
}
/// Wrapper around `Url` that implements `ToSchema` for `Url`.
#[derive(
Debug,
serde::Serialize,
serde::Deserialize,
bincode::Encode,
bincode::Decode,
PartialEq,
Eq,
Clone,
ToSchema,
)]
#[schema(value_type = String, title = "Url")]
pub struct UrlWrapper(#[bincode(with_serde)] Url);
impl DerefMut for UrlWrapper {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl Deref for UrlWrapper {
type Target = Url;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl From<Url> for UrlWrapper {
fn from(url: Url) -> Self {
Self(url)
}
}
#[derive(
Debug,
serde::Serialize,
serde::Deserialize,
bincode::Encode,
bincode::Decode,
PartialEq,
Eq,
Clone,
ToSchema,
)]
#[serde(rename_all = "camelCase")]
pub struct BangHit {
pub bang: Bang,
pub redirect_to: UrlWrapper,
}
pub struct Bangs {
bangs: HashMap<String, Bang>,
}
impl Bangs {
pub fn empty() -> Self {
Self {
bangs: HashMap::new(),
}
}
pub fn from_path<P: AsRef<Path>>(path: P) -> Self {
let json = fs::read_to_string(path).unwrap();
Self::from_json(json.as_str())
}
pub fn from_json(json: &str) -> Self {
let all_bangs: Vec<Bang> = serde_json::from_str(json).unwrap();
Self {
bangs: all_bangs
.into_iter()
.map(|bang| (bang.tag.clone(), bang))
.collect(),
}
}
pub fn get(&self, terms: &[Term]) -> Option<BangHit> {
for possible_bang in terms.iter().filter_map(|term| {
if let Term::PossibleBang { prefix: _, bang } = term {
Some(bang)
} else {
None
}
}) {
if let Some(bang) = self.bangs.get(possible_bang) {
let query = intersperse(
terms
.iter()
.filter(|term| {
if let Term::PossibleBang { prefix: _, bang } = term {
bang != possible_bang
} else {
true
}
})
.map(|term| term.to_string()),
" ".to_string(),
)
.collect::<String>();
let query = urlencoding::encode(query.as_str()).into_owned();
let url = bang.url.replace("{{{s}}}", query.as_str());
return Url::parse(url.as_str())
.or_else(|_| Url::parse(&format!("https://{}", url)))
.ok()
.map(|url| BangHit {
bang: bang.clone(),
redirect_to: url.into(),
});
}
}
None
}
}
#[cfg(test)]
mod tests {
use crate::query::parser::parse;
use super::*;
#[test]
fn simple_bang() {
let bangs = Bangs::from_json(
r#"[{
"c": "Multimedia",
"d": "www.youtube.com",
"r": 1646,
"s": "Youtube",
"sc": "Video",
"t": "ty",
"u": "https://www.youtube.com/results?search_query={{{s}}}"
}]"#,
);
assert_eq!(bangs.get(&parse("no bangs").unwrap()), None);
assert_eq!(bangs.get(&parse("!no bangs").unwrap()), None);
assert_eq!(
bangs.get(&parse("!ty bangs").unwrap()),
Some(BangHit {
bang: Bang {
category: Some("Multimedia".to_string()),
sub_category: Some("Video".to_string()),
domain: Some("www.youtube.com".to_string()),
ranking: Some(1646),
site: Some("Youtube".to_string()),
tag: "ty".to_string(),
url: "https://www.youtube.com/results?search_query={{{s}}}".to_string()
},
redirect_to: Url::parse("https://www.youtube.com/results?search_query=bangs")
.unwrap()
.into()
})
);
}
}