web search done right
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
inverted_index::InvertedIndex,
query::parser::TermCompound,
ranking::SignalCoefficients,
schema::text_field,
search_ctx::Ctx,
searcher::SearchQuery,
webpage::{region::Region, safety_classifier},
Error, Result,
};
use optics::{HostRankings, Optic};
use tantivy::query::{BooleanQuery, Occur, QueryClone};
mod const_query;
pub mod intersection;
pub mod optic;
pub mod parser;
mod pattern_query;
mod plan;
pub mod union;
use self::{optic::AsMultipleTantivyQuery, parser::SimpleOrPhrase};
use parser::Term;
pub const MAX_TERMS_FOR_NGRAM_LOOKUPS: usize = 16;
#[derive(Debug)]
pub struct Query {
simple_terms_text: Vec<String>,
tantivy_query: Box<dyn tantivy::query::Query>,
host_rankings: HostRankings,
offset: usize,
region: Option<Region>,
optics: Vec<Optic>,
top_n: usize,
count_results_exact: bool,
signal_coefficients: SignalCoefficients,
lang: Option<whatlang::Lang>,
}
impl Clone for Query {
fn clone(&self) -> Self {
Self {
simple_terms_text: self.simple_terms_text.clone(),
tantivy_query: self.tantivy_query.box_clone(),
host_rankings: self.host_rankings.clone(),
offset: self.offset,
region: self.region,
optics: self.optics.clone(),
top_n: self.top_n,
count_results_exact: self.count_results_exact,
signal_coefficients: self.signal_coefficients.clone(),
lang: self.lang,
}
}
}
impl Query {
pub fn parse(ctx: &Ctx, query: &SearchQuery, index: &InvertedIndex) -> Result<Query> {
let lang = whatlang::detect_lang(&query.query);
let parsed_terms = parser::truncate(parser::parse(&query.query)?);
if parsed_terms.is_empty() {
tracing::error!("No terms found in query");
return Err(Error::EmptyQuery.into());
}
if parsed_terms
.iter()
.all(|t| matches!(t, Term::PossibleBang { .. }))
{
tracing::error!("No non-bang terms found in query");
return Err(Error::EmptyQuery.into());
}
let simple_terms_text: Vec<String> = parsed_terms
.iter()
.filter_map(|term| term.as_simple_text().map(|s| s.to_string()))
.flat_map(|term| {
// term might be a phrase, so we split it into words
term.split_ascii_whitespace()
.map(|s| s.to_string())
.collect::<Vec<_>>()
})
.collect();
let mut plan = plan::initial(parsed_terms).expect("terms are not empty and not all bangs");
let schema = index.schema();
if query.safe_search {
plan = plan.and(plan::Node::Not(Box::new(plan::Node::Term(
plan::Term::new(
parser::SimpleTerm::from(safety_classifier::Label::NSFW.to_string()).into(),
text_field::SafetyClassification.into(),
),
))));
}
let mut tantivy_query = plan
.into_query()
.as_tantivy(lang.as_ref(), &schema)
.expect("there should at least be one field in the index");
let mut optics = Vec::new();
if let Some(site_rankigns_optic) = query.host_rankings.clone().map(|sr| sr.into_optic()) {
optics.push(site_rankigns_optic);
}
if let Some(optic) = &query.optic {
optics.push(optic.clone());
}
for optic in &optics {
let mut subqueries = vec![(Occur::Must, tantivy_query.box_clone())];
subqueries.append(&mut optic.as_multiple_tantivy(&schema, &ctx.columnfield_reader));
tantivy_query = Box::new(BooleanQuery::new(subqueries));
}
Ok(Query {
host_rankings: optics.iter().fold(HostRankings::default(), |mut acc, el| {
acc.merge_into(el.host_rankings.clone());
acc
}),
simple_terms_text,
tantivy_query,
optics,
offset: query.num_results * query.page,
region: query.selected_region,
top_n: query.num_results,
count_results_exact: query.count_results_exact,
signal_coefficients: query.signal_coefficients(),
lang,
})
}
pub fn count_results_exact(&self) -> bool {
self.count_results_exact
}
pub fn simple_terms(&self) -> &[String] {
&self.simple_terms_text
}
pub fn optics(&self) -> &[Optic] {
&self.optics
}
pub fn num_results(&self) -> usize {
self.top_n
}
pub fn offset(&self) -> usize {
self.offset
}
pub fn region(&self) -> Option<&Region> {
self.region.as_ref()
}
pub fn host_rankings(&self) -> &HostRankings {
&self.host_rankings
}
pub fn signal_coefficients(&self) -> SignalCoefficients {
self.signal_coefficients.clone()
}
pub fn lang(&self) -> Option<whatlang::Lang> {
self.lang
}
}
impl tantivy::query::Query for Query {
fn weight(
&self,
enable_scoring: tantivy::query::EnableScoring,
) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
self.tantivy_query.weight(enable_scoring)
}
fn query_terms<'a>(&'a self, visitor: &mut dyn FnMut(&'a tantivy::Term, bool)) {
self.tantivy_query.query_terms(visitor)
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use crate::{index::Index, rand_words, searcher::LocalSearcher, webpage::Webpage};
use proptest::prelude::*;
use tokio::sync::RwLock;
use super::*;
fn empty_index() -> (InvertedIndex, file_store::temp::TempDir) {
InvertedIndex::temporary().unwrap()
}
#[test]
fn simple_parse() {
let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let query = Query::parse(
&ctx,
&SearchQuery {
query: "this is a simple query".to_string(),
..Default::default()
},
&index,
)
.expect("Failed to parse query");
assert_eq!(
query.simple_terms(),
vec![
"this".to_string(),
"is".to_string(),
"a".to_string(),
"simple".to_string(),
"query".to_string(),
]
);
}
#[test]
fn parse_trailing_leading_whitespace() {
let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let query = Query::parse(
&ctx,
&SearchQuery {
query: " this is a simple query ".to_string(),
..Default::default()
},
&index,
)
.expect("Failed to parse query");
assert_eq!(
query.simple_terms(),
vec![
"this".to_string(),
"is".to_string(),
"a".to_string(),
"simple".to_string(),
"query".to_string(),
]
);
}
#[test]
fn parse_weird_characters() {
let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let terms = Query::parse(
&ctx,
&SearchQuery {
query: "123".to_string(),
..Default::default()
},
&index,
)
.expect("Failed to parse query")
.simple_terms()
.to_vec();
assert_eq!(terms, vec!["123".to_string()]);
let terms = Query::parse(
&ctx,
&SearchQuery {
query: "123 33".to_string(),
..Default::default()
},
&index,
)
.expect("Failed to parse query")
.simple_terms()
.to_vec();
assert_eq!(terms, vec!["123".to_string(), "33".to_string()]);
let terms = Query::parse(
&ctx,
&SearchQuery {
query: "term! term# $".to_string(),
..Default::default()
},
&index,
)
.expect("Failed to parse query")
.simple_terms()
.to_vec();
assert_eq!(
terms,
vec!["term!".to_string(), "term#".to_string(), "$".to_string()]
);
}
#[test]
fn simple_terms_phrase() {
let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let terms = Query::parse(
&ctx,
&SearchQuery {
query: "\"test term\"".to_string(),
..Default::default()
},
&index,
)
.expect("Failed to parse query")
.simple_terms()
.to_vec();
assert_eq!(terms, vec!["test".to_string(), "term".to_string()]);
}
#[test]
fn not_query() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let query = SearchQuery {
query: "test -website".to_string(),
..Default::default()
};
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website
</body>
</html>
"#,
"https://www.first.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This test page does not contain the forbidden word
</body>
</html>
"#,
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.second.com/");
}
#[test]
fn site_query() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website
</body>
</html>
"#,
"https://www.first.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This test page does not contain the forbidden word
</body>
</html>
"#,
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This test page does not contain the forbidden word
</body>
</html>
"#,
"https://www.second.com/first",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "test site:first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "test site:www.first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "test -site:first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
assert!(result
.webpages
.iter()
.all(|w| w.url != "https://www.first.com/"));
}
#[test]
fn links_to_query() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website
<a href="https://www.second.com/example/abc">Second</a>
</body>
</html>
"#,
"https://www.first.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This test page does not contain the forbidden word
<a href="https://www.first.com">First</a>
</body>
</html>
"#,
"https://www.second.com/example/abc",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "test linksto:first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.second.com/example/abc");
let query = SearchQuery {
query: "test linkto:www.first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.second.com/example/abc");
let query = SearchQuery {
query: "test -linkto:first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "test linkto:second.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "test linkto:www.second.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "test linkto:second.com/example".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "test linksto:second.com/example/abc".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
#[test]
fn links_to_uppercase() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website
<a href="https://www.SeCoNd.CoM/eXaMpLe/AbC">Second</a>
</body>
</html>
"#,
"https://www.first.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This test page does not contain the forbidden word
<a href="https://www.first.com">First</a>
</body>
</html>
"#,
"https://www.second.com/example/AbC",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "test linkto:second.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
#[test]
fn title_query() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website
</body>
</html>
"#,
"https://www.first.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a test website
</body>
</html>
"#,
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "intitle:website".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
#[test]
fn url_query() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website
</body>
</html>
"#,
"https://www.first.com/forum",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a test website
</body>
</html>
"#,
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "test inurl:forum".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/forum");
}
#[test]
fn empty_query() {
let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let query = Query::parse(
&ctx,
&SearchQuery {
query: "".to_string(),
..Default::default()
},
&index,
);
assert!(query.is_err());
assert_eq!(
query.err().unwrap().to_string(),
anyhow::Error::from(Error::EmptyQuery).to_string()
);
}
#[test]
fn query_term_only_special_char() {
let (index, _dir) = empty_index();
let ctx = index.local_search_ctx();
let _query = Query::parse(
&ctx,
&SearchQuery {
query: "&".to_string(),
..Default::default()
},
&index,
)
.expect("Failed to parse query");
}
#[test]
fn site_query_split_domain() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.the-first.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This test page does not contain the forbidden word {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "test site:first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 0);
let query = SearchQuery {
query: "test site:the-first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.the-first.com/");
let query = SearchQuery {
query: "test site:www.the-first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.the-first.com/");
}
#[test]
fn phrase_query() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.first.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a bad test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "\"Test website\"".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "\"Test website\" site:www.second.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 0);
}
#[test]
fn match_compound_words() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.first.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Testwebsite</title>
</head>
<body>
This is a testwebsite {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "testwebsite".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
let query = SearchQuery {
query: "test website".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
}
#[test]
fn deduplicate_terms() {
let a = parser::parse("the the the the the").unwrap();
let a = plan::initial(a).unwrap();
let a = a.into_query();
let b = parser::parse("the the the the the the the the the the the the").unwrap();
let b = plan::initial(b).unwrap();
let b = b.into_query();
assert_eq!(a.len(), b.len());
}
#[test]
fn safe_search() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
let mut webpage = Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.sfw.com",
)
.unwrap();
webpage.safety_classification = Some(safety_classifier::Label::SFW);
webpage.html.set_clean_text("sfw".to_string());
index.insert(&webpage).expect("failed to insert webpage");
let mut webpage = Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.nsfw.com",
)
.unwrap();
webpage.safety_classification = Some(safety_classifier::Label::NSFW);
webpage.html.set_clean_text("nsfw".to_string());
index.insert(&webpage).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "test".to_string(),
safe_search: false,
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
let query = SearchQuery {
query: "test".to_string(),
safe_search: true,
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.sfw.com/");
}
#[test]
fn suffix_domain_prefix_path_site_operator() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.first.com/example",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.third.io",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "test site:.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
let query = SearchQuery {
query: "test site:.com/example".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
let query = SearchQuery {
query: "test site:first.com/example".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
let query = SearchQuery {
query: "test site:first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
let query = SearchQuery {
query: "test site:www.first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
}
#[test]
fn exact_url_operator() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.first.com/example",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.third.io",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "test exacturl:https://www.first.com/example".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
let query = SearchQuery {
query: "test exacturl:https://www.first.com".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 0);
}
#[test]
fn mix_phrase_term_query() {
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test website</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.first.com/example",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.second.com",
)
.unwrap(),
)
.expect("failed to insert webpage");
index
.insert(
&Webpage::test_parse(
&format!(
r#"
<html>
<head>
<title>Test test</title>
</head>
<body>
This is a test website {}
</body>
</html>
"#,
rand_words(1000)
),
"https://www.third.io",
)
.unwrap(),
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::builder(Arc::new(RwLock::new(index))).build();
let query = SearchQuery {
query: "\"test test\" website".to_string(),
..Default::default()
};
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
}
fn fixture(query: &str) -> Result<(), TestCaseError> {
if query.trim().is_empty() {
return Ok(());
}
let parsed_terms = parser::truncate(
parser::parse(query).map_err(|_| TestCaseError::fail("parse failed"))?,
);
let plan =
plan::initial(parsed_terms).ok_or(TestCaseError::fail("plan should not be empty"))?;
let _ = plan.into_query();
Ok(())
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(4096))]
#[test]
fn test_query_parse_non_panic(query in ".*") {
fixture(&query)?;
}
}
}