Skip to content

web search done right

added bangs

Mikkel Denker04c33127a488
diff --git a/justfile b/justfile
index 926b5c17..83e5f048 100644
--- a/justfile
+++ b/justfile
@@ -17,7 +17,7 @@
cargo run --release -- webgraph local configs/webgraph/local.toml
@frontend:
- cargo watch -x 'run -- frontend data/index data/queries_us.csv data/entity'
+ cargo watch -x 'run -- frontend data/index data/queries_us.csv data/entity data/bangs.json'
@local:
wget -nc -P data/warc_files https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-05/segments/1642320299852.23/warc/CC-MAIN-20220116093137-20220116123137-00049.warc.gz
diff --git a/src/bangs.rs b/src/bangs.rs
new file mode 100644
index 00000000..0e7955d5
--- /dev/null
+++ b/src/bangs.rs
@@ -0,0 +1,174 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{collections::HashMap, fs, path::Path};
+
+use itertools::intersperse;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ query::{parser::Term, Query},
+ webpage::Url,
+};
+
+pub const BANG_PREFIX: char = '!';
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Bang {
+ #[serde(rename = "c")]
+ category: Option<String>,
+
+ #[serde(rename = "sc")]
+ sub_category: Option<String>,
+
+ #[serde(rename = "d")]
+ domain: Option<String>,
+
+ #[serde(rename = "r")]
+ ranking: Option<usize>,
+
+ #[serde(rename = "s")]
+ site: Option<String>,
+
+ #[serde(rename = "t")]
+ tag: String,
+
+ #[serde(rename = "u")]
+ url: String,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct BangHit {
+ bang: Bang,
+ pub redirect_to: Url,
+}
+
+pub struct Bangs {
+ bangs: HashMap<String, Bang>,
+}
+
+impl Bangs {
+ pub fn from_path<P: AsRef<Path>>(path: P) -> Self {
+ let json = fs::read_to_string(path).unwrap();
+
+ Self::from_json(json.as_str())
+ }
+
+ pub fn from_json(json: &str) -> Self {
+ let all_bangs: Vec<Bang> = serde_json::from_str(json).unwrap();
+
+ Self {
+ bangs: all_bangs
+ .into_iter()
+ .map(|bang| (bang.tag.clone(), bang))
+ .collect(),
+ }
+ }
+
+ pub fn get(&self, query: &Query) -> Option<BangHit> {
+ for possible_bang in query.terms().iter().filter_map(|term| {
+ if let Term::PossibleBang(possible_bang) = term.as_ref() {
+ Some(possible_bang)
+ } else {
+ None
+ }
+ }) {
+ if let Some(bang) = self.bangs.get(possible_bang) {
+ return Some(BangHit {
+ bang: bang.clone(),
+ redirect_to: bang
+ .url
+ .replace(
+ "{{{s}}}",
+ intersperse(
+ query
+ .terms()
+ .iter()
+ .filter(|term| {
+ if let Term::PossibleBang(bang) = term.as_ref() {
+ bang != possible_bang
+ } else {
+ true
+ }
+ })
+ .map(|term| term.as_ref().to_string()),
+ " ".to_string(),
+ )
+ .collect::<String>()
+ .as_str(),
+ )
+ .into(),
+ });
+ }
+ }
+
+ None
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+
+ use tantivy::tokenizer::TokenizerManager;
+
+ use crate::schema::create_schema;
+
+ use super::*;
+
+ #[test]
+ fn simple_bang() {
+ let schema = Arc::new(create_schema());
+ let tokenizer_manager = TokenizerManager::new();
+
+ let bangs = Bangs::from_json(
+ r#"[{
+ "c": "Multimedia",
+ "d": "www.youtube.com",
+ "r": 1646,
+ "s": "Youtube",
+ "sc": "Video",
+ "t": "ty",
+ "u": "https://www.youtube.com/results?search_query={{{s}}}"
+ }]"#,
+ );
+
+ let query = Query::parse("no bangs", schema.clone(), &tokenizer_manager).unwrap();
+ assert_eq!(bangs.get(&query), None);
+
+ let query = Query::parse("!no bangs", schema.clone(), &tokenizer_manager).unwrap();
+ assert_eq!(bangs.get(&query), None);
+
+ let query = Query::parse("!ty bangs", schema, &tokenizer_manager).unwrap();
+ assert_eq!(
+ bangs.get(&query),
+ Some(BangHit {
+ bang: Bang {
+ category: Some("Multimedia".to_string()),
+ sub_category: Some("Video".to_string()),
+ domain: Some("www.youtube.com".to_string()),
+ ranking: Some(1646),
+ site: Some("Youtube".to_string()),
+ tag: "ty".to_string(),
+ url: "https://www.youtube.com/results?search_query={{{s}}}".to_string()
+ },
+ redirect_to: "https://www.youtube.com/results?search_query=bangs"
+ .to_string()
+ .into()
+ })
+ );
+ }
+}
diff --git a/src/entrypoint/frontend.rs b/src/entrypoint/frontend.rs
index 27b88c2d..fbe0a63b 100644
--- a/src/entrypoint/frontend.rs
+++ b/src/entrypoint/frontend.rs
@@ -22,9 +22,10 @@ pub async fn run(
index_path: &str,
queries_csv_path: &str,
entity_index_path: Option<String>,
+ bangs_path: Option<String>,
host: &str,
) -> Result<()> {
- let app = router(index_path, queries_csv_path, entity_index_path)?;
+ let app = router(index_path, queries_csv_path, entity_index_path, bangs_path)?;
let addr = host.parse()?;
tracing::info!("listening on {}", addr);
axum::Server::bind(&addr)
diff --git a/src/frontend/mod.rs b/src/frontend/mod.rs
index 2c41b481..714e572a 100644
--- a/src/frontend/mod.rs
+++ b/src/frontend/mod.rs
@@ -17,7 +17,8 @@
use axum::{Extension, Router};
use crate::{
- autosuggest::Autosuggest, entity_index::EntityIndex, index::Index, searcher::Searcher,
+ autosuggest::Autosuggest, bangs::Bangs, entity_index::EntityIndex, index::Index,
+ searcher::Searcher,
};
use anyhow::Result;
use std::sync::Arc;
@@ -64,11 +65,13 @@ pub fn router(
index_path: &str,
queries_csv_path: &str,
entity_index_path: Option<String>,
+ bangs_path: Option<String>,
) -> Result<Router> {
let entity_index = entity_index_path.map(|path| EntityIndex::open(path).unwrap());
+ let bangs = bangs_path.map(Bangs::from_path);
let search_index = Index::open(index_path)?;
let autosuggest = Autosuggest::load_csv(queries_csv_path)?;
- let searcher = Searcher::new(search_index, entity_index);
+ let searcher = Searcher::new(search_index, entity_index, bangs);
let state = Arc::new(State {
searcher,
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index 7c0e257c..47477a31 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -21,7 +21,7 @@ use itertools::{intersperse, Itertools};
use crate::{
entity_index::{entity::Span, StoredEntity},
inverted_index::RetrievedWebpage,
- searcher::Searcher,
+ searcher::{SearchResult, Searcher},
webpage::{
region::{Region, ALL_REGIONS},
Url,
@@ -270,60 +270,64 @@ pub async fn route(
});
match state.searcher.search(query.as_str(), selected_region) {
- Ok(result) => {
- let search_result = result
- .webpages
- .documents
- .into_iter()
- .map(|mut webpage| {
- webpage.primary_image_uuid = webpage.primary_image_uuid.and_then(|uuid| {
- if state.searcher.primary_image(uuid.clone()).is_some() {
- Some(uuid)
- } else {
- None
- }
- });
- webpage
- })
- .map(DisplayedWebpage::from)
- .collect();
-
- let entity = result
- .entity
- .map(|entity| DisplayedEntity::from(entity, &state.searcher));
- let spell_correction = result.spell_corrected_query;
-
- let num_matches = thousand_sep_number(result.webpages.num_docs);
-
- let search_duration_sec = format!("{:.2}", result.search_duration_ms as f64 / 1000.0);
-
- let all_regions = ALL_REGIONS
- .into_iter()
- .map(|region| {
- if let Some(selected_region) = selected_region {
- if region == selected_region {
- RegionSelection::Selected(region)
+ Ok(result) => match result {
+ SearchResult::Websites(result) => {
+ let search_result = result
+ .webpages
+ .documents
+ .into_iter()
+ .map(|mut webpage| {
+ webpage.primary_image_uuid = webpage.primary_image_uuid.and_then(|uuid| {
+ if state.searcher.primary_image(uuid.clone()).is_some() {
+ Some(uuid)
+ } else {
+ None
+ }
+ });
+ webpage
+ })
+ .map(DisplayedWebpage::from)
+ .collect();
+
+ let entity = result
+ .entity
+ .map(|entity| DisplayedEntity::from(entity, &state.searcher));
+ let spell_correction = result.spell_corrected_query;
+
+ let num_matches = thousand_sep_number(result.webpages.num_docs);
+
+ let search_duration_sec =
+ format!("{:.2}", result.search_duration_ms as f64 / 1000.0);
+
+ let all_regions = ALL_REGIONS
+ .into_iter()
+ .map(|region| {
+ if let Some(selected_region) = selected_region {
+ if region == selected_region {
+ RegionSelection::Selected(region)
+ } else {
+ RegionSelection::Unselected(region)
+ }
} else {
RegionSelection::Unselected(region)
}
- } else {
- RegionSelection::Unselected(region)
- }
- })
- .collect();
-
- let template = SearchTemplate {
- search_result,
- query,
- entity,
- spell_correction,
- num_matches,
- search_duration_sec,
- all_regions,
- };
-
- HtmlTemplate(template).into_response()
- }
+ })
+ .collect();
+
+ let template = SearchTemplate {
+ search_result,
+ query,
+ entity,
+ spell_correction,
+ num_matches,
+ search_duration_sec,
+ all_regions,
+ };
+
+ HtmlTemplate(template).into_response()
+ }
+ SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),
+ },
Err(Error::EmptyQuery) => Redirect::to("/").into_response(),
Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
}
diff --git a/src/lib.rs b/src/lib.rs
index cf8d4406..2630bf53 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -33,6 +33,7 @@ mod inverted_index;
pub mod mapreduce;
mod autosuggest;
+mod bangs;
mod directory;
mod entity_index;
mod exponential_backoff;
diff --git a/src/main.rs b/src/main.rs
index c1962384..643c1009 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -48,6 +48,7 @@ enum Commands {
index_path: String,
queries_csv_path: String,
entity_index_path: Option<String>,
+ bangs_path: Option<String>,
#[clap(default_value = "0.0.0.0:3000")]
host: String,
},
@@ -136,6 +137,7 @@ fn main() -> Result<()> {
queries_csv_path,
host,
entity_index_path,
+ bangs_path,
} => tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
@@ -143,6 +145,7 @@ fn main() -> Result<()> {
&index_path,
&queries_csv_path,
entity_index_path,
+ bangs_path,
&host,
))?,
}
diff --git a/src/query/mod.rs b/src/query/mod.rs
index ade301a4..7e2eaf38 100644
--- a/src/query/mod.rs
+++ b/src/query/mod.rs
@@ -25,7 +25,7 @@ use tantivy::{
tokenizer::TokenizerManager,
};
-mod parser;
+pub mod parser;
use parser::Term;
const MAX_SIMILAR_TERMS: usize = 10;
@@ -89,6 +89,10 @@ impl Query {
.collect()
}
+ pub fn terms(&self) -> &[Box<Term>] {
+ &self.terms
+ }
+
pub fn is_empty(&self) -> bool {
self.terms.is_empty()
}
diff --git a/src/query/parser.rs b/src/query/parser.rs
index 30a71063..dbae4f62 100644
--- a/src/query/parser.rs
+++ b/src/query/parser.rs
@@ -20,7 +20,10 @@ use tantivy::{
tokenizer::{TextAnalyzer, TokenizerManager},
};
-use crate::schema::{Field, ALL_FIELDS};
+use crate::{
+ bangs::BANG_PREFIX,
+ schema::{Field, ALL_FIELDS},
+};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Term {
@@ -30,6 +33,52 @@ pub enum Term {
Title(String),
Body(String),
Url(String),
+ PossibleBang(String),
+}
+
+impl ToString for Term {
+ fn to_string(&self) -> String {
+ match self {
+ Term::Simple(term) => term.clone(),
+ Term::Not(term) => "-".to_string() + term.to_string().as_str(),
+ Term::Site(site) => "site:".to_string() + site.as_str(),
+ Term::Title(title) => "intitle:".to_string() + title.as_str(),
+ Term::Body(body) => "inbody:".to_string() + body.as_str(),
+ Term::Url(url) => "inurl:".to_string() + url.as_str(),
+ Term::PossibleBang(bang) => "!".to_string() + bang.as_str(),
+ }
+ }
+}
+
+fn simple_into_tantivy(
+ term: &str,
+ fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
+ tokenizer_manager: &TokenizerManager,
+) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
+ let (backlink_field, backlink_field_entry) = fields
+ .iter()
+ .find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::BacklinkText))
+ .unwrap();
+
+ vec![
+ (
+ Occur::Must,
+ Box::new(BooleanQuery::new(Term::into_tantivy_simple(
+ term,
+ fields,
+ tokenizer_manager,
+ ))),
+ ),
+ (
+ Occur::Should,
+ Box::new(Term::tantivy_term_query(
+ backlink_field,
+ backlink_field_entry,
+ tokenizer_manager,
+ term,
+ )),
+ ),
+ ]
}
impl Term {
@@ -39,34 +88,7 @@ impl Term {
tokenizer_manager: &TokenizerManager,
) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
match self {
- Term::Simple(term) => {
- let (backlink_field, backlink_field_entry) = fields
- .iter()
- .find(|(field, _)| {
- matches!(ALL_FIELDS[field.field_id() as usize], Field::BacklinkText)
- })
- .unwrap();
-
- vec![
- (
- Occur::Must,
- Box::new(BooleanQuery::new(Term::into_tantivy_simple(
- term,
- fields,
- tokenizer_manager,
- ))),
- ),
- (
- Occur::Should,
- Box::new(Term::tantivy_term_query(
- backlink_field,
- backlink_field_entry,
- tokenizer_manager,
- term,
- )),
- ),
- ]
- }
+ Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager),
Term::Not(subterm) => vec![(
Occur::MustNot,
Box::new(BooleanQuery::new(
@@ -115,6 +137,13 @@ impl Term {
Term::tantivy_term_query(field, entry, tokenizer_manager, url),
)]
}
+ Term::PossibleBang(text) => {
+ let mut term = String::new();
+ term.push(BANG_PREFIX);
+ term.push_str(text);
+
+ simple_into_tantivy(&term, fields, tokenizer_manager)
+ }
}
}
@@ -250,6 +279,8 @@ fn parse_term(term: &str) -> Box<Term> {
} else {
Box::new(Term::Simple(term.to_string()))
}
+ } else if let Some(bang) = term.strip_prefix(BANG_PREFIX) {
+ Box::new(Term::PossibleBang(bang.to_string()))
} else {
Box::new(Term::Simple(term.to_string()))
}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 68353c7d..7a28349a 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -107,7 +107,11 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
- let result = searcher.search("example", None).expect("Search failed");
+ let result = searcher
+ .search("example", None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
assert_eq!(result.webpages.documents.len(), 2);
assert_eq!(result.webpages.documents[0].url, "https://www.b.com");
assert_eq!(result.webpages.documents[1].url, "https://www.a.com");
@@ -178,7 +182,11 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
- let result = searcher.search("dr dk", None).expect("Search failed");
+ let result = searcher
+ .search("dr dk", None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
assert_eq!(result.webpages.documents.len(), 3);
assert_eq!(result.webpages.documents[0].url, "https://www.dr.dk");
@@ -233,7 +241,11 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
- let result = searcher.search("title", None).expect("Search failed");
+ let result = searcher
+ .search("title", None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
assert_eq!(result.webpages.documents[0].url, "https://www.new.com");
}
@@ -295,7 +307,11 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
- let result = searcher.search("test", None).expect("Search failed");
+ let result = searcher
+ .search("test", None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
assert_eq!(result.webpages.documents.len(), 2);
assert_eq!(result.webpages.documents[0].url, "https://www.first.com");
@@ -348,7 +364,11 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = Searcher::from(index);
- let result = searcher.search("test", None).expect("Search failed");
+ let result = searcher
+ .search("test", None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
assert_eq!(result.webpages.documents.len(), 2);
assert_eq!(result.webpages.documents[0].url, "https://www.first.com");
diff --git a/src/searcher.rs b/src/searcher.rs
index 79329109..05f9d34a 100644
--- a/src/searcher.rs
+++ b/src/searcher.rs
@@ -19,6 +19,7 @@ use std::time::Instant;
use uuid::Uuid;
+use crate::bangs::{BangHit, Bangs};
use crate::entity_index::{EntityIndex, StoredEntity};
use crate::image_store::Image;
use crate::index::Index;
@@ -30,29 +31,37 @@ use crate::webpage::Url;
use crate::{Error, Result};
#[derive(Debug)]
-pub struct SearchResult {
+pub struct WebsitesResult {
pub spell_corrected_query: Option<String>,
pub webpages: InvertedIndexSearchResult,
pub entity: Option<StoredEntity>,
pub search_duration_ms: u128,
}
+#[derive(Debug)]
+pub enum SearchResult {
+ Websites(WebsitesResult),
+ Bang(BangHit),
+}
+
pub struct Searcher {
index: Index,
entity_index: Option<EntityIndex>,
+ bangs: Option<Bangs>,
}
impl From<Index> for Searcher {
fn from(index: Index) -> Self {
- Self::new(index, None)
+ Self::new(index, None, None)
}
}
impl Searcher {
- pub fn new(index: Index, entity_index: Option<EntityIndex>) -> Self {
+ pub fn new(index: Index, entity_index: Option<EntityIndex>, bangs: Option<Bangs>) -> Self {
Searcher {
index,
entity_index,
+ bangs,
}
}
}
@@ -68,6 +77,12 @@ impl Searcher {
return Err(Error::EmptyQuery);
}
+ if let Some(bangs) = self.bangs.as_ref() {
+ if let Some(bang) = bangs.get(&query) {
+ return Ok(SearchResult::Bang(bang));
+ }
+ }
+
let mut ranker = Ranker::new(self.index.region_count.clone());
if let Some(region) = selected_region {
@@ -86,12 +101,12 @@ impl Searcher {
let search_duration_ms = start.elapsed().as_millis();
- Ok(SearchResult {
+ Ok(SearchResult::Websites(WebsitesResult {
webpages,
entity,
spell_corrected_query: correction,
search_duration_ms,
- })
+ }))
}
pub fn favicon(&self, site: &Url) -> Option<Image> {
@@ -117,3 +132,13 @@ impl Searcher {
.and_then(|index| index.get_attribute_occurrence(attribute))
}
}
+impl SearchResult {
+ #[cfg(test)]
+ pub fn into_websites(self) -> Option<WebsitesResult> {
+ if let SearchResult::Websites(res) = self {
+ Some(res)
+ } else {
+ None
+ }
+ }
+}
diff --git a/src/snippet.rs b/src/snippet.rs
index 78849a76..289f2688 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -354,7 +354,9 @@ Survey in 2016, 2017, and 2018."#;
let result = searcher
.search("rust language", None)
- .expect("Search failed");
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
assert_eq!(result.webpages.num_docs, 1);
assert_eq!(result.webpages.documents.len(), 1);
@@ -390,7 +392,11 @@ Survey in 2016, 2017, and 2018."#;
let searcher = Searcher::from(index);
- let result = searcher.search("describe", None).expect("Search failed");
+ let result = searcher
+ .search("describe", None)
+ .expect("Search failed")
+ .into_websites()
+ .unwrap();
assert_eq!(result.webpages.num_docs, 1);
assert_eq!(result.webpages.documents.len(), 1);