web search done right
handle more edge cases in optics correctly
Mikkel Denker363340820155
diff --git a/core/src/api/search.rs b/core/src/api/search.rsindex 9c78c87a..e9322171 100644--- a/core/src/api/search.rs+++ b/core/src/api/search.rs@@ -131,7 +131,7 @@ pub async fn route(.search(&SearchQuery {query: query.clone(),selected_region,- optic: optic,+ optic,page: skip_pages,site_rankings,num_results: NUM_RESULTS_PER_PAGE,diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex 8ded1f7a..80595c86 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -44,7 +44,7 @@ pub struct Query {site_rankings: SiteRankings,offset: usize,region: Option<Region>,- optic: Option<Optic>,+ optics: Vec<Optic>,top_n: usize,}@@ -89,23 +89,17 @@ impl Query {let mut tantivy_query = Box::new(BooleanQuery::new(queries));- let mut optic = query- .site_rankings- .clone()- .map(|site_rankings| site_rankings.into_optic());+ let mut optics = Vec::new();+ if let Some(site_rankigns_optic) = query.site_rankings.clone().map(|sr| sr.into_optic()) {+ optics.push(site_rankigns_optic);+ }if let Some(optic_program) = &query.optic {- match optic.as_mut() {- Some(inner) => {- optic = Some(inner.clone().try_merge(Optic::parse(optic_program)?)?);- }- None => {- optic = Some(Optic::parse(optic_program)?);- }- }+ let optic = Optic::parse(optic_program)?;+ optics.push(optic);}- if let Some(optic) = &optic {+ for optic in &optics {let mut subqueries = vec![(Occur::Must, tantivy_query.box_clone())];subqueries.append(&mut optic.as_multiple_tantivy(&schema, &ctx.fastfield_reader));tantivy_query = Box::new(BooleanQuery::new(subqueries));@@ -113,13 +107,16 @@ impl Query {Ok(Query {terms,- site_rankings: optic- .as_ref()- .map(|optic| optic.site_rankings.clone())- .unwrap_or_default(),+ site_rankings: optics.clone().into_iter().fold(+ SiteRankings::default(),+ |mut acc, el| {+ acc.merge_into(el.site_rankings);+ acc+ },+ ),simple_terms_text,tantivy_query,- optic,+ optics,offset: query.num_results * query.page,region: query.selected_region,top_n: query.num_results,@@ -134,8 +131,8 @@ impl Query {&self.terms}- pub fn optic(&self) -> Option<&Optic> {- self.optic.as_ref()+ pub fn optics(&self) -> &[Optic] {+ &self.optics}pub fn is_empty(&self) -> bool {@@ -159,7 +156,19 @@ impl Query {}pub fn signal_coefficients(&self) -> Option<SignalCoefficient> {- self.optic.as_ref().map(SignalCoefficient::from_optic)+ if self.optics.is_empty() {+ return None;+ }++ Some(+ self.optics+ .iter()+ .fold(SignalCoefficient::default(), |mut acc, optic| {+ let coeffs = SignalCoefficient::from_optic(optic);+ acc.merge_into(coeffs);+ acc+ }),+ )}}diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex cabdcecb..db365023 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -1269,4 +1269,336 @@ mod tests {.webpages;assert_eq!(res.len(), 1);}++ #[test]+ fn active_optic_with_blocked_sites() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>This is an example website</title>+ </head>+ <body>+ {CONTENT} {}+ This is an example+ </body>+ </html>+ "#,+ crate::rand_words(1000)+ ),+ "https://example.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ host_topic: None,+ dmoz_description: None,+ })+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ let searcher = LocalSearcher::from(index);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "DiscardNonMatching; Rule { Matches { Title(\"is\") }, Action(Boost(0)) }"+ .to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "DiscardNonMatching; Rule { Matches { Title(\"is\") }, Action(Boost(0)) }"+ .to_string(),+ ),+ site_rankings: Some(SiteRankings {+ liked: vec![],+ disliked: vec![],+ blocked: vec![String::from("example.com")],+ }),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 0);+ }++ #[test]+ fn empty_optic_noop() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>This is an example website</title>+ </head>+ <body>+ {CONTENT} {}+ This is an example+ </body>+ </html>+ "#,+ crate::rand_words(1000)+ ),+ "https://example.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ host_topic: None,+ dmoz_description: None,+ })+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ let searcher = LocalSearcher::from(index);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some("".to_string()),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some("Rule { Matches { Title(\"\") }, Action(Discard) }".to_string()),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);+ }++ #[test]+ fn wildcard_edge_cases() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>This is an example website</title>+ </head>+ <body>+ {CONTENT} {}+ This is an example+ </body>+ </html>+ "#,+ crate::rand_words(1000)+ ),+ "https://example.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ host_topic: None,+ dmoz_description: None,+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Another thing with no words in common</title>+ </head>+ <body>+ {CONTENT} {}+ This is an example+ </body>+ </html>+ "#,+ crate::rand_words(1000)+ ),+ "https://example.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ host_topic: None,+ dmoz_description: None,+ })+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ let searcher = LocalSearcher::from(index);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some("Rule { Matches { Title(\"*\") }, Action(Discard) }".to_string()),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 0);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some("Rule { Matches { Title(\"* is\") }, Action(Discard) }".to_string()),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Title(\"* This is\") }, Action(Discard) }".to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Title(\"example *\") }, Action(Discard) }".to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Title(\"example website *\") }, Action(Discard) }"+ .to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);+ }++ #[test]+ fn empty_double_anchor() {+ let mut index = Index::temporary().expect("Unable to open index");++ let mut page = Webpage {+ html: Html::parse(+ r#"+ <html>+ <head>+ <title>This is an example website</title>+ </head>+ <body>+ Test+ </body>+ </html>+ "#,+ "https://example.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ host_topic: None,+ dmoz_description: None,+ };++ page.html.set_clean_text("".to_string());++ index.insert(page).expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ let searcher = LocalSearcher::from(index);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "DiscardNonMatching; Rule { Matches { Content(\"||\") }, Action(Boost(0)) }"+ .to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "DiscardNonMatching; Rule { Matches { Content(\"|\") }, Action(Boost(0)) }"+ .to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);+ }}diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex bfead129..04d116f9 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -95,7 +95,11 @@ impl tantivy::query::Query for PatternQuery {) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {let bm25_weight = match scoring {tantivy::query::EnableScoring::Enabled(searcher) => {- Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)+ if self.raw_terms.is_empty() {+ None+ } else {+ Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)+ }}tantivy::query::EnableScoring::Disabled(_) => None,};@@ -150,6 +154,53 @@ impl PatternWeight {return Ok(None);}+ let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {+ Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),+ Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),+ Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),+ Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),+ Field::Text(TextField::Site) => Ok(FastField::NumSiteTokens),+ Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),+ Field::Text(TextField::FlattenedSchemaOrgJson) => {+ Ok(FastField::NumFlattenedSchemaTokens)+ }+ field => Err(TantivyError::InvalidArgument(format!(+ "{} is not supported in pattern query",+ field.name()+ ))),+ }?;++ // "*" matches everything+ if self.raw_terms.is_empty()+ && self+ .patterns+ .iter()+ .any(|p| matches!(p, PatternPart::Wildcard))+ {+ return Ok(Some(PatternScorer::Everything(AllScorer {+ doc: 0,+ max_doc: reader.max_doc(),+ })));+ }++ // "||" and "|" matches empty string++ if self.raw_terms.is_empty()+ && self+ .patterns+ .iter()+ .all(|p| matches!(p, PatternPart::Anchor))+ {+ return Ok(Some(PatternScorer::EmptyField(EmptyFieldScorer {+ num_tokens_fastfield,+ segment_reader: self.fastfield_reader.get_segment(&reader.segment_id()),+ all_scorer: AllScorer {+ doc: 0,+ max_doc: reader.max_doc(),+ },+ })));+ }+let similarity_weight = self.similarity_weight.as_ref()@@ -239,22 +290,6 @@ impl PatternWeight {}).collect();- let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {- Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),- Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),- Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),- Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),- Field::Text(TextField::Site) => Ok(FastField::NumSiteTokens),- Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),- Field::Text(TextField::FlattenedSchemaOrgJson) => {- Ok(FastField::NumFlattenedSchemaTokens)- }- field => Err(TantivyError::InvalidArgument(format!(- "{} is not supported in pattern query",- field.name()- ))),- }?;-Ok(Some(PatternScorer::Normal(NormalPatternScorer::new(similarity_weight,term_postings_list,@@ -314,6 +349,8 @@ impl tantivy::query::Weight for PatternWeight {enum PatternScorer {Normal(NormalPatternScorer),FastSiteDomain(FastSiteDomainPatternScorer),+ Everything(AllScorer),+ EmptyField(EmptyFieldScorer),}impl Scorer for PatternScorer {@@ -321,6 +358,8 @@ impl Scorer for PatternScorer {match self {PatternScorer::Normal(scorer) => scorer.score(),PatternScorer::FastSiteDomain(scorer) => scorer.score(),+ PatternScorer::Everything(scorer) => scorer.score(),+ PatternScorer::EmptyField(scorer) => scorer.score(),}}}@@ -330,6 +369,8 @@ impl DocSet for PatternScorer {match self {PatternScorer::Normal(scorer) => scorer.advance(),PatternScorer::FastSiteDomain(scorer) => scorer.advance(),+ PatternScorer::Everything(scorer) => scorer.advance(),+ PatternScorer::EmptyField(scorer) => scorer.advance(),}}@@ -337,6 +378,8 @@ impl DocSet for PatternScorer {match self {PatternScorer::Normal(scorer) => scorer.seek(target),PatternScorer::FastSiteDomain(scorer) => scorer.seek(target),+ PatternScorer::Everything(scorer) => scorer.seek(target),+ PatternScorer::EmptyField(scorer) => scorer.seek(target),}}@@ -344,6 +387,8 @@ impl DocSet for PatternScorer {match self {PatternScorer::Normal(scorer) => scorer.doc(),PatternScorer::FastSiteDomain(scorer) => scorer.doc(),+ PatternScorer::Everything(scorer) => scorer.doc(),+ PatternScorer::EmptyField(scorer) => scorer.doc(),}}@@ -351,6 +396,8 @@ impl DocSet for PatternScorer {match self {PatternScorer::Normal(scorer) => scorer.size_hint(),PatternScorer::FastSiteDomain(scorer) => scorer.size_hint(),+ PatternScorer::Everything(scorer) => scorer.size_hint(),+ PatternScorer::EmptyField(scorer) => scorer.size_hint(),}}}@@ -360,8 +407,102 @@ impl PatternScorer {match self {PatternScorer::Normal(scorer) => scorer.phrase_count(),PatternScorer::FastSiteDomain(scorer) => scorer.term_freq(),+ PatternScorer::Everything(_) => 0,+ PatternScorer::EmptyField(_) => 0,+ }+ }+}++struct AllScorer {+ doc: DocId,+ max_doc: DocId,+}++impl DocSet for AllScorer {+ fn advance(&mut self) -> DocId {+ if self.doc + 1 >= self.max_doc {+ self.doc = TERMINATED;+ return TERMINATED;+ }+ self.doc += 1;+ self.doc+ }++ fn seek(&mut self, target: DocId) -> DocId {+ if target >= self.max_doc {+ self.doc = TERMINATED;+ return TERMINATED;+ }+ self.doc = target;+ self.doc+ }++ fn doc(&self) -> DocId {+ self.doc+ }++ fn size_hint(&self) -> u32 {+ self.max_doc+ }+}++impl Scorer for AllScorer {+ fn score(&mut self) -> Score {+ 1.0+ }+}++struct EmptyFieldScorer {+ segment_reader: Arc<fastfield_reader::SegmentReader>,+ num_tokens_fastfield: FastField,+ all_scorer: AllScorer,+}++impl EmptyFieldScorer {+ fn num_tokes(&self, doc: DocId) -> u64 {+ let s: Option<u64> = self+ .segment_reader+ .get_field_reader(&self.num_tokens_fastfield)+ .get(&doc)+ .into();+ s.unwrap_or_default()+ }+}++impl DocSet for EmptyFieldScorer {+ fn advance(&mut self) -> DocId {+ let mut doc = self.all_scorer.advance();++ while doc != TERMINATED && self.num_tokes(doc) > 0 {+ doc = self.all_scorer.advance();+ }++ doc+ }++ fn doc(&self) -> DocId {+ self.all_scorer.doc()+ }++ fn seek(&mut self, target: DocId) -> DocId {+ self.all_scorer.seek(target);++ if self.doc() != TERMINATED && self.num_tokes(self.all_scorer.doc()) > 0 {+ self.advance()+ } else {+ self.doc()}}++ fn size_hint(&self) -> u32 {+ self.all_scorer.size_hint()+ }+}++impl Scorer for EmptyFieldScorer {+ fn score(&mut self) -> Score {+ 1.0+ }}struct FastSiteDomainPatternScorer {diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex b49df0e9..57c77c06 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -604,7 +604,7 @@ fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {}}-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Default)]pub struct SignalCoefficient {map: EnumMap<Signal, f64>,}@@ -633,6 +633,19 @@ impl SignalCoefficient {}}))}++ pub fn merge_into(&mut self, coeffs: SignalCoefficient) {+ for signal in ALL_SIGNALS {+ if let Some(coeff) = coeffs.get(&signal) {+ match self.map.get_mut(signal) {+ Some(existing_coeff) => *existing_coeff += coeff,+ None => {+ self.map.insert(signal, coeff);+ }+ }+ }+ }+ }}#[derive(Clone)]@@ -803,29 +816,27 @@ impl SignalAggregator {fastfield_reader: &fastfield_reader::FastFieldReader,) -> Vec<RuleBoost> {let mut optic_rule_boosts = Vec::new();+if let Some(query) = &self.query {- if let Some(optic) = query.optic() {- optic_rule_boosts = optic- .rules- .iter()- .filter(|rule| match rule.action {- optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,- optics::Action::Discard => false,- })- .filter_map(|rule| {- rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader)- })- .map(|(_, rule)| RuleBoost {- docset: rule- .query- .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))- .unwrap()- .scorer(segment_reader, 0.0)- .unwrap(),- boost: rule.boost,- })- .collect();- }+ optic_rule_boosts = query+ .optics()+ .iter()+ .flat_map(|o| o.rules.iter())+ .filter(|rule| match rule.action {+ optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,+ optics::Action::Discard => false,+ })+ .filter_map(|rule| rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader))+ .map(|(_, rule)| RuleBoost {+ docset: rule+ .query+ .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))+ .unwrap()+ .scorer(segment_reader, 0.0)+ .unwrap(),+ boost: rule.boost,+ })+ .collect();}optic_rule_boostsdiff --git a/optics/src/lib.rs b/optics/src/lib.rsindex a95322f3..671f3aab 100644--- a/optics/src/lib.rs+++ b/optics/src/lib.rs@@ -267,28 +267,6 @@ impl Optic {pub fn parse(optic: &str) -> Result<Self> {parse(optic)}-- pub fn try_merge(mut self, mut other: Self) -> Result<Self> {- self.rules.append(&mut other.rules);-- self.rankings.append(&mut other.rankings);-- self.discard_non_matching |= other.discard_non_matching;-- self.site_rankings- .liked- .append(&mut other.site_rankings.liked);-- self.site_rankings- .disliked- .append(&mut other.site_rankings.disliked);-- self.site_rankings- .blocked- .append(&mut other.site_rankings.blocked);-- Ok(self)- }}#[derive(Debug, PartialEq, Eq, Clone)]@@ -324,6 +302,12 @@ impl SiteRankings {..Default::default()}}++ pub fn merge_into(&mut self, site_rankings: SiteRankings) {+ self.liked.extend(site_rankings.liked);+ self.disliked.extend(site_rankings.disliked);+ self.blocked.extend(site_rankings.blocked);+ }}#[cfg(test)]