web search done right
added bangs
Mikkel Denker04c33127a488
diff --git a/justfile b/justfileindex 926b5c17..83e5f048 100644--- a/justfile+++ b/justfile@@ -17,7 +17,7 @@cargo run --release -- webgraph local configs/webgraph/local.toml@frontend:- cargo watch -x 'run -- frontend data/index data/queries_us.csv data/entity'+ cargo watch -x 'run -- frontend data/index data/queries_us.csv data/entity data/bangs.json'@local:wget -nc -P data/warc_files https://data.commoncrawl.org/crawl-data/CC-MAIN-2022-05/segments/1642320299852.23/warc/CC-MAIN-20220116093137-20220116123137-00049.warc.gzdiff --git a/src/bangs.rs b/src/bangs.rsnew file mode 100644index 00000000..0e7955d5--- /dev/null+++ b/src/bangs.rs@@ -0,0 +1,174 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{collections::HashMap, fs, path::Path};++use itertools::intersperse;+use serde::{Deserialize, Serialize};++use crate::{+ query::{parser::Term, Query},+ webpage::Url,+};++pub const BANG_PREFIX: char = '!';++#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct Bang {+ #[serde(rename = "c")]+ category: Option<String>,++ #[serde(rename = "sc")]+ sub_category: Option<String>,++ #[serde(rename = "d")]+ domain: Option<String>,++ #[serde(rename = "r")]+ ranking: Option<usize>,++ #[serde(rename = "s")]+ site: Option<String>,++ #[serde(rename = "t")]+ tag: String,++ #[serde(rename = "u")]+ url: String,+}++#[derive(Debug, PartialEq, Eq)]+pub struct BangHit {+ bang: Bang,+ pub redirect_to: Url,+}++pub struct Bangs {+ bangs: HashMap<String, Bang>,+}++impl Bangs {+ pub fn from_path<P: AsRef<Path>>(path: P) -> Self {+ let json = fs::read_to_string(path).unwrap();++ Self::from_json(json.as_str())+ }++ pub fn from_json(json: &str) -> Self {+ let all_bangs: Vec<Bang> = serde_json::from_str(json).unwrap();++ Self {+ bangs: all_bangs+ .into_iter()+ .map(|bang| (bang.tag.clone(), bang))+ .collect(),+ }+ }++ pub fn get(&self, query: &Query) -> Option<BangHit> {+ for possible_bang in query.terms().iter().filter_map(|term| {+ if let Term::PossibleBang(possible_bang) = term.as_ref() {+ Some(possible_bang)+ } else {+ None+ }+ }) {+ if let Some(bang) = self.bangs.get(possible_bang) {+ return Some(BangHit {+ bang: bang.clone(),+ redirect_to: bang+ .url+ .replace(+ "{{{s}}}",+ intersperse(+ query+ .terms()+ .iter()+ .filter(|term| {+ if let Term::PossibleBang(bang) = term.as_ref() {+ bang != possible_bang+ } else {+ true+ }+ })+ .map(|term| term.as_ref().to_string()),+ " ".to_string(),+ )+ .collect::<String>()+ .as_str(),+ )+ .into(),+ });+ }+ }++ None+ }+}++#[cfg(test)]+mod tests {+ use std::sync::Arc;++ use tantivy::tokenizer::TokenizerManager;++ use crate::schema::create_schema;++ use super::*;++ #[test]+ fn simple_bang() {+ let schema = Arc::new(create_schema());+ let tokenizer_manager = TokenizerManager::new();++ let bangs = Bangs::from_json(+ r#"[{+ "c": "Multimedia",+ "d": "www.youtube.com",+ "r": 1646,+ "s": "Youtube",+ "sc": "Video",+ "t": "ty",+ "u": "https://www.youtube.com/results?search_query={{{s}}}"+ }]"#,+ );++ let query = Query::parse("no bangs", schema.clone(), &tokenizer_manager).unwrap();+ assert_eq!(bangs.get(&query), None);++ let query = Query::parse("!no bangs", schema.clone(), &tokenizer_manager).unwrap();+ assert_eq!(bangs.get(&query), None);++ let query = Query::parse("!ty bangs", schema, &tokenizer_manager).unwrap();+ assert_eq!(+ bangs.get(&query),+ Some(BangHit {+ bang: Bang {+ category: Some("Multimedia".to_string()),+ sub_category: Some("Video".to_string()),+ domain: Some("www.youtube.com".to_string()),+ ranking: Some(1646),+ site: Some("Youtube".to_string()),+ tag: "ty".to_string(),+ url: "https://www.youtube.com/results?search_query={{{s}}}".to_string()+ },+ redirect_to: "https://www.youtube.com/results?search_query=bangs"+ .to_string()+ .into()+ })+ );+ }+}diff --git a/src/entrypoint/frontend.rs b/src/entrypoint/frontend.rsindex 27b88c2d..fbe0a63b 100644--- a/src/entrypoint/frontend.rs+++ b/src/entrypoint/frontend.rs@@ -22,9 +22,10 @@ pub async fn run(index_path: &str,queries_csv_path: &str,entity_index_path: Option<String>,+ bangs_path: Option<String>,host: &str,) -> Result<()> {- let app = router(index_path, queries_csv_path, entity_index_path)?;+ let app = router(index_path, queries_csv_path, entity_index_path, bangs_path)?;let addr = host.parse()?;tracing::info!("listening on {}", addr);axum::Server::bind(&addr)diff --git a/src/frontend/mod.rs b/src/frontend/mod.rsindex 2c41b481..714e572a 100644--- a/src/frontend/mod.rs+++ b/src/frontend/mod.rs@@ -17,7 +17,8 @@use axum::{Extension, Router};use crate::{- autosuggest::Autosuggest, entity_index::EntityIndex, index::Index, searcher::Searcher,+ autosuggest::Autosuggest, bangs::Bangs, entity_index::EntityIndex, index::Index,+ searcher::Searcher,};use anyhow::Result;use std::sync::Arc;@@ -64,11 +65,13 @@ pub fn router(index_path: &str,queries_csv_path: &str,entity_index_path: Option<String>,+ bangs_path: Option<String>,) -> Result<Router> {let entity_index = entity_index_path.map(|path| EntityIndex::open(path).unwrap());+ let bangs = bangs_path.map(Bangs::from_path);let search_index = Index::open(index_path)?;let autosuggest = Autosuggest::load_csv(queries_csv_path)?;- let searcher = Searcher::new(search_index, entity_index);+ let searcher = Searcher::new(search_index, entity_index, bangs);let state = Arc::new(State {searcher,diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex 7c0e257c..47477a31 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -21,7 +21,7 @@ use itertools::{intersperse, Itertools};use crate::{entity_index::{entity::Span, StoredEntity},inverted_index::RetrievedWebpage,- searcher::Searcher,+ searcher::{SearchResult, Searcher},webpage::{region::{Region, ALL_REGIONS},Url,@@ -270,60 +270,64 @@ pub async fn route(});match state.searcher.search(query.as_str(), selected_region) {- Ok(result) => {- let search_result = result- .webpages- .documents- .into_iter()- .map(|mut webpage| {- webpage.primary_image_uuid = webpage.primary_image_uuid.and_then(|uuid| {- if state.searcher.primary_image(uuid.clone()).is_some() {- Some(uuid)- } else {- None- }- });- webpage- })- .map(DisplayedWebpage::from)- .collect();-- let entity = result- .entity- .map(|entity| DisplayedEntity::from(entity, &state.searcher));- let spell_correction = result.spell_corrected_query;-- let num_matches = thousand_sep_number(result.webpages.num_docs);-- let search_duration_sec = format!("{:.2}", result.search_duration_ms as f64 / 1000.0);-- let all_regions = ALL_REGIONS- .into_iter()- .map(|region| {- if let Some(selected_region) = selected_region {- if region == selected_region {- RegionSelection::Selected(region)+ Ok(result) => match result {+ SearchResult::Websites(result) => {+ let search_result = result+ .webpages+ .documents+ .into_iter()+ .map(|mut webpage| {+ webpage.primary_image_uuid = webpage.primary_image_uuid.and_then(|uuid| {+ if state.searcher.primary_image(uuid.clone()).is_some() {+ Some(uuid)+ } else {+ None+ }+ });+ webpage+ })+ .map(DisplayedWebpage::from)+ .collect();++ let entity = result+ .entity+ .map(|entity| DisplayedEntity::from(entity, &state.searcher));+ let spell_correction = result.spell_corrected_query;++ let num_matches = thousand_sep_number(result.webpages.num_docs);++ let search_duration_sec =+ format!("{:.2}", result.search_duration_ms as f64 / 1000.0);++ let all_regions = ALL_REGIONS+ .into_iter()+ .map(|region| {+ if let Some(selected_region) = selected_region {+ if region == selected_region {+ RegionSelection::Selected(region)+ } else {+ RegionSelection::Unselected(region)+ }} else {RegionSelection::Unselected(region)}- } else {- RegionSelection::Unselected(region)- }- })- .collect();-- let template = SearchTemplate {- search_result,- query,- entity,- spell_correction,- num_matches,- search_duration_sec,- all_regions,- };-- HtmlTemplate(template).into_response()- }+ })+ .collect();++ let template = SearchTemplate {+ search_result,+ query,+ entity,+ spell_correction,+ num_matches,+ search_duration_sec,+ all_regions,+ };++ HtmlTemplate(template).into_response()+ }+ SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),+ },Err(Error::EmptyQuery) => Redirect::to("/").into_response(),Err(_) => panic!("Search failed"), // TODO: show 500 status to user here}diff --git a/src/lib.rs b/src/lib.rsindex cf8d4406..2630bf53 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -33,6 +33,7 @@ mod inverted_index;pub mod mapreduce;mod autosuggest;+mod bangs;mod directory;mod entity_index;mod exponential_backoff;diff --git a/src/main.rs b/src/main.rsindex c1962384..643c1009 100644--- a/src/main.rs+++ b/src/main.rs@@ -48,6 +48,7 @@ enum Commands {index_path: String,queries_csv_path: String,entity_index_path: Option<String>,+ bangs_path: Option<String>,#[clap(default_value = "0.0.0.0:3000")]host: String,},@@ -136,6 +137,7 @@ fn main() -> Result<()> {queries_csv_path,host,entity_index_path,+ bangs_path,} => tokio::runtime::Builder::new_multi_thread().enable_all().build()?@@ -143,6 +145,7 @@ fn main() -> Result<()> {&index_path,&queries_csv_path,entity_index_path,+ bangs_path,&host,))?,}diff --git a/src/query/mod.rs b/src/query/mod.rsindex ade301a4..7e2eaf38 100644--- a/src/query/mod.rs+++ b/src/query/mod.rs@@ -25,7 +25,7 @@ use tantivy::{tokenizer::TokenizerManager,};-mod parser;+pub mod parser;use parser::Term;const MAX_SIMILAR_TERMS: usize = 10;@@ -89,6 +89,10 @@ impl Query {.collect()}+ pub fn terms(&self) -> &[Box<Term>] {+ &self.terms+ }+pub fn is_empty(&self) -> bool {self.terms.is_empty()}diff --git a/src/query/parser.rs b/src/query/parser.rsindex 30a71063..dbae4f62 100644--- a/src/query/parser.rs+++ b/src/query/parser.rs@@ -20,7 +20,10 @@ use tantivy::{tokenizer::{TextAnalyzer, TokenizerManager},};-use crate::schema::{Field, ALL_FIELDS};+use crate::{+ bangs::BANG_PREFIX,+ schema::{Field, ALL_FIELDS},+};#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub enum Term {@@ -30,6 +33,52 @@ pub enum Term {Title(String),Body(String),Url(String),+ PossibleBang(String),+}++impl ToString for Term {+ fn to_string(&self) -> String {+ match self {+ Term::Simple(term) => term.clone(),+ Term::Not(term) => "-".to_string() + term.to_string().as_str(),+ Term::Site(site) => "site:".to_string() + site.as_str(),+ Term::Title(title) => "intitle:".to_string() + title.as_str(),+ Term::Body(body) => "inbody:".to_string() + body.as_str(),+ Term::Url(url) => "inurl:".to_string() + url.as_str(),+ Term::PossibleBang(bang) => "!".to_string() + bang.as_str(),+ }+ }+}++fn simple_into_tantivy(+ term: &str,+ fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],+ tokenizer_manager: &TokenizerManager,+) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {+ let (backlink_field, backlink_field_entry) = fields+ .iter()+ .find(|(field, _)| matches!(ALL_FIELDS[field.field_id() as usize], Field::BacklinkText))+ .unwrap();++ vec![+ (+ Occur::Must,+ Box::new(BooleanQuery::new(Term::into_tantivy_simple(+ term,+ fields,+ tokenizer_manager,+ ))),+ ),+ (+ Occur::Should,+ Box::new(Term::tantivy_term_query(+ backlink_field,+ backlink_field_entry,+ tokenizer_manager,+ term,+ )),+ ),+ ]}impl Term {@@ -39,34 +88,7 @@ impl Term {tokenizer_manager: &TokenizerManager,) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {match self {- Term::Simple(term) => {- let (backlink_field, backlink_field_entry) = fields- .iter()- .find(|(field, _)| {- matches!(ALL_FIELDS[field.field_id() as usize], Field::BacklinkText)- })- .unwrap();-- vec![- (- Occur::Must,- Box::new(BooleanQuery::new(Term::into_tantivy_simple(- term,- fields,- tokenizer_manager,- ))),- ),- (- Occur::Should,- Box::new(Term::tantivy_term_query(- backlink_field,- backlink_field_entry,- tokenizer_manager,- term,- )),- ),- ]- }+ Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager),Term::Not(subterm) => vec![(Occur::MustNot,Box::new(BooleanQuery::new(@@ -115,6 +137,13 @@ impl Term {Term::tantivy_term_query(field, entry, tokenizer_manager, url),)]}+ Term::PossibleBang(text) => {+ let mut term = String::new();+ term.push(BANG_PREFIX);+ term.push_str(text);++ simple_into_tantivy(&term, fields, tokenizer_manager)+ }}}@@ -250,6 +279,8 @@ fn parse_term(term: &str) -> Box<Term> {} else {Box::new(Term::Simple(term.to_string()))}+ } else if let Some(bang) = term.strip_prefix(BANG_PREFIX) {+ Box::new(Term::PossibleBang(bang.to_string()))} else {Box::new(Term::Simple(term.to_string()))}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 68353c7d..7a28349a 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -107,7 +107,11 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);- let result = searcher.search("example", None).expect("Search failed");+ let result = searcher+ .search("example", None)+ .expect("Search failed")+ .into_websites()+ .unwrap();assert_eq!(result.webpages.documents.len(), 2);assert_eq!(result.webpages.documents[0].url, "https://www.b.com");assert_eq!(result.webpages.documents[1].url, "https://www.a.com");@@ -178,7 +182,11 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);- let result = searcher.search("dr dk", None).expect("Search failed");+ let result = searcher+ .search("dr dk", None)+ .expect("Search failed")+ .into_websites()+ .unwrap();assert_eq!(result.webpages.documents.len(), 3);assert_eq!(result.webpages.documents[0].url, "https://www.dr.dk");@@ -233,7 +241,11 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);- let result = searcher.search("title", None).expect("Search failed");+ let result = searcher+ .search("title", None)+ .expect("Search failed")+ .into_websites()+ .unwrap();assert_eq!(result.webpages.documents[0].url, "https://www.new.com");}@@ -295,7 +307,11 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);- let result = searcher.search("test", None).expect("Search failed");+ let result = searcher+ .search("test", None)+ .expect("Search failed")+ .into_websites()+ .unwrap();assert_eq!(result.webpages.documents.len(), 2);assert_eq!(result.webpages.documents[0].url, "https://www.first.com");@@ -348,7 +364,11 @@ mod tests {index.commit().expect("failed to commit index");let searcher = Searcher::from(index);- let result = searcher.search("test", None).expect("Search failed");+ let result = searcher+ .search("test", None)+ .expect("Search failed")+ .into_websites()+ .unwrap();assert_eq!(result.webpages.documents.len(), 2);assert_eq!(result.webpages.documents[0].url, "https://www.first.com");diff --git a/src/searcher.rs b/src/searcher.rsindex 79329109..05f9d34a 100644--- a/src/searcher.rs+++ b/src/searcher.rs@@ -19,6 +19,7 @@ use std::time::Instant;use uuid::Uuid;+use crate::bangs::{BangHit, Bangs};use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;@@ -30,29 +31,37 @@ use crate::webpage::Url;use crate::{Error, Result};#[derive(Debug)]-pub struct SearchResult {+pub struct WebsitesResult {pub spell_corrected_query: Option<String>,pub webpages: InvertedIndexSearchResult,pub entity: Option<StoredEntity>,pub search_duration_ms: u128,}+#[derive(Debug)]+pub enum SearchResult {+ Websites(WebsitesResult),+ Bang(BangHit),+}+pub struct Searcher {index: Index,entity_index: Option<EntityIndex>,+ bangs: Option<Bangs>,}impl From<Index> for Searcher {fn from(index: Index) -> Self {- Self::new(index, None)+ Self::new(index, None, None)}}impl Searcher {- pub fn new(index: Index, entity_index: Option<EntityIndex>) -> Self {+ pub fn new(index: Index, entity_index: Option<EntityIndex>, bangs: Option<Bangs>) -> Self {Searcher {index,entity_index,+ bangs,}}}@@ -68,6 +77,12 @@ impl Searcher {return Err(Error::EmptyQuery);}+ if let Some(bangs) = self.bangs.as_ref() {+ if let Some(bang) = bangs.get(&query) {+ return Ok(SearchResult::Bang(bang));+ }+ }+let mut ranker = Ranker::new(self.index.region_count.clone());if let Some(region) = selected_region {@@ -86,12 +101,12 @@ impl Searcher {let search_duration_ms = start.elapsed().as_millis();- Ok(SearchResult {+ Ok(SearchResult::Websites(WebsitesResult {webpages,entity,spell_corrected_query: correction,search_duration_ms,- })+ }))}pub fn favicon(&self, site: &Url) -> Option<Image> {@@ -117,3 +132,13 @@ impl Searcher {.and_then(|index| index.get_attribute_occurrence(attribute))}}+impl SearchResult {+ #[cfg(test)]+ pub fn into_websites(self) -> Option<WebsitesResult> {+ if let SearchResult::Websites(res) = self {+ Some(res)+ } else {+ None+ }+ }+}diff --git a/src/snippet.rs b/src/snippet.rsindex 78849a76..289f2688 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -354,7 +354,9 @@ Survey in 2016, 2017, and 2018."#;let result = searcher.search("rust language", None)- .expect("Search failed");+ .expect("Search failed")+ .into_websites()+ .unwrap();assert_eq!(result.webpages.num_docs, 1);assert_eq!(result.webpages.documents.len(), 1);@@ -390,7 +392,11 @@ Survey in 2016, 2017, and 2018."#;let searcher = Searcher::from(index);- let result = searcher.search("describe", None).expect("Search failed");+ let result = searcher+ .search("describe", None)+ .expect("Search failed")+ .into_websites()+ .unwrap();assert_eq!(result.webpages.num_docs, 1);assert_eq!(result.webpages.documents.len(), 1);