web search done right
fixed a bunch of problems with pattern_query implementation and wrote some tests to make sure it works correctly
Mikkel Denker5ab900eea599
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rsindex 6f40982d..36447e83 100644--- a/core/benches/search-preindexed-optic.rs+++ b/core/benches/search-preindexed-optic.rs@@ -17,7 +17,7 @@ macro_rules! bench {$searcher.search(&SearchQuery {query: $query.to_string(),- optic_program: Some($optic.to_string()),+ optic: Some($optic.to_string()),..Default::default()}).unwrap()diff --git a/core/src/api/search.rs b/core/src/api/search.rsindex 49b38343..9c78c87a 100644--- a/core/src/api/search.rs+++ b/core/src/api/search.rs@@ -131,7 +131,7 @@ pub async fn route(.search(&SearchQuery {query: query.clone(),selected_region,- optic_program: optic,+ optic: optic,page: skip_pages,site_rankings,num_results: NUM_RESULTS_PER_PAGE,diff --git a/core/src/query/intersection.rs b/core/src/query/intersection.rsindex 7a3a31e7..101e3941 100644--- a/core/src/query/intersection.rs+++ b/core/src/query/intersection.rs@@ -20,6 +20,7 @@ use tantivy::{DocId, DocSet, Score, TERMINATED};/// Creates a `DocSet` that iterate through the intersection of two or more `DocSet`s.pub struct Intersection<TDocSet: DocSet = Box<dyn Scorer>> {docsets: Vec<TDocSet>,+ old_idx: Vec<usize>,}fn go_to_first_doc<TDocSet: DocSet>(docsets: &mut [TDocSet]) -> DocId {@@ -39,9 +40,18 @@ fn go_to_first_doc<TDocSet: DocSet>(docsets: &mut [TDocSet]) -> DocId {impl<TDocSet: DocSet> Intersection<TDocSet> {pub(crate) fn new(mut docsets: Vec<TDocSet>) -> Intersection<TDocSet> {- docsets.sort_by_key(|docset| docset.size_hint());+ let mut docsets: Vec<(usize, TDocSet)> = docsets.drain(..).enumerate().collect();+ docsets.sort_by_key(|(_, docset)| docset.size_hint());+ let (tmp, mut docsets): (Vec<usize>, Vec<TDocSet>) = docsets.into_iter().unzip();++ let mut old_idx = vec![0; docsets.len()];++ for (new_idx, t) in tmp.into_iter().enumerate() {+ old_idx[t] = new_idx;+ }+go_to_first_doc(&mut docsets);- Intersection { docsets }+ Intersection { docsets, old_idx }}}@@ -106,6 +116,6 @@ whereimpl<TDocSet: DocSet> Intersection<TDocSet> {pub(crate) fn docset_mut_specialized(&mut self, ord: usize) -> &mut TDocSet {- &mut self.docsets[ord]+ &mut self.docsets[self.old_idx[ord]]}}diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex a7d741c3..8ded1f7a 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -94,7 +94,7 @@ impl Query {.clone().map(|site_rankings| site_rankings.into_optic());- if let Some(optic_program) = &query.optic_program {+ if let Some(optic_program) = &query.optic {match optic.as_mut() {Some(inner) => {optic = Some(inner.clone().try_merge(Optic::parse(optic_program)?)?);diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 98b8d916..cabdcecb 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -303,7 +303,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(r#"Rule {Matches {@@ -325,7 +325,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(r#"Rule {Matches {@@ -416,9 +416,7 @@ mod tests {let _ = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(- include_str!("../../../optics/testcases/quickstart.optic").to_string(),- ),+ optic: Some(include_str!("../../../optics/testcases/quickstart.optic").to_string()),..Default::default()}).unwrap()@@ -427,7 +425,7 @@ mod tests {let _ = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(include_str!("../../../optics/testcases/hacker_news.optic").to_string(),),..Default::default()@@ -438,7 +436,7 @@ mod tests {let _ = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(include_str!("../../../optics/testcases/copycats_removal.optic").to_string(),),..Default::default()@@ -549,7 +547,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(r#"DiscardNonMatching;Rule {@@ -698,7 +696,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(r#"Like(Site("www.a.com"));Like(Site("www.b.com"));@@ -816,7 +814,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(r#"DiscardNonMatching;Rule {@@ -838,7 +836,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(r#"DiscardNonMatching;Rule {@@ -860,7 +858,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(r#"DiscardNonMatching;Rule {@@ -882,7 +880,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "website".to_string(),- optic_program: Some(+ optic: Some(r#"DiscardNonMatching;Rule {@@ -944,7 +942,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "site:stackoverflow.com".to_string(),- optic_program: Some(+ optic: Some(r#"DiscardNonMatching;Rule {@@ -1049,7 +1047,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "example".to_string(),- optic_program: Some(+ optic: Some(r#"DiscardNonMatching;Rule {@@ -1113,11 +1111,162 @@ mod tests {let res = searcher.search(&SearchQuery {query: "avengers endgame".to_string(),- optic_program: Some(include_str!("../searcher/discussions.optic").to_string()),+ optic: Some(include_str!("../searcher/discussions.optic").to_string()),..Default::default()}).unwrap().webpages;assert!(res.is_empty());}++ #[test]+ fn special_pattern_syntax() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>This is an example website</title>+ </head>+ <body>+ {CONTENT} {}+ This is an example+ </body>+ </html>+ "#,+ crate::rand_words(1000)+ ),+ "https://example.com",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ host_topic: None,+ dmoz_description: None,+ })+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");++ let searcher = LocalSearcher::from(index);+ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);+ assert_eq!(res[0].url, "https://example.com");++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some("Rule { Matches { Title(\"is\") }, Action(Discard) }".to_string()),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 0);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some("Rule { Matches { Title(\"|is\") }, Action(Discard) }".to_string()),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some("Rule { Matches { Title(\"|This\") }, Action(Discard) }".to_string()),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 0);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Title(\"|This an\") }, Action(Discard) }".to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Title(\"|This * an\") }, Action(Discard) }".to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 0);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Site(\"example.com\") }, Action(Discard) }".to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 0);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Site(\"|example.com\") }, Action(Discard) }".to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 0);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Site(\"|example.com|\") }, Action(Discard) }".to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 0);++ let res = searcher+ .search(&SearchQuery {+ query: "example".to_string(),+ optic: Some(+ "Rule { Matches { Title(\"website.com|\") }, Action(Discard) }".to_string(),+ ),+ ..Default::default()+ })+ .unwrap()+ .webpages;+ assert_eq!(res.len(), 1);+ }}diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rsindex b3ce4a41..bfead129 100644--- a/core/src/query/pattern_query.rs+++ b/core/src/query/pattern_query.rs@@ -61,6 +61,7 @@ impl PatternQuery {let mut raw_terms = Vec::new();let tv_field = schema.get_field(Field::Text(field).name()).unwrap();+ let mut new_patterns = Vec::with_capacity(patterns.len());for pattern in &patterns {match pattern {@@ -68,17 +69,18 @@ impl PatternQuery {let mut stream = field.tokenizer().token_stream(text);while let Some(token) = stream.next() {+ new_patterns.push(PatternPart::Raw(token.text.clone()));let term = tantivy::Term::from_field_text(tv_field, &token.text);raw_terms.push(term);}}- PatternPart::Wildcard => {}- PatternPart::Anchor => {}+ PatternPart::Wildcard => new_patterns.push(PatternPart::Wildcard),+ PatternPart::Anchor => new_patterns.push(PatternPart::Anchor),}}Self {- patterns,+ patterns: new_patterns,field: tv_field,raw_terms,fastfield_reader,@@ -114,6 +116,7 @@ impl tantivy::query::Query for PatternQuery {}}+#[derive(Debug)]enum SmallPatternPart {Term,Wildcard,@@ -143,14 +146,78 @@ impl PatternWeight {reader: &SegmentReader,boost: Score,) -> tantivy::Result<Option<PatternScorer>> {+ if self.patterns.is_empty() {+ return Ok(None);+ }+let similarity_weight = self.similarity_weight.as_ref().map(|weight| weight.boost_by(boost));let fieldnorm_reader = self.fieldnorm_reader(reader)?;- let mut term_postings_list = Vec::new();+ // if pattern is of form Site("|site|") or Domain("|domain|")+ // we can use the field without tokenization to speed up the query significantly+ if self.patterns.len() >= 2+ && matches!(&self.patterns[0], PatternPart::Anchor)+ && matches!(&self.patterns[self.patterns.len() - 1], PatternPart::Anchor)+ && self.patterns[1..self.patterns.len() - 1]+ .iter()+ .all(|pattern| matches!(pattern, PatternPart::Raw(_)))+ && (matches!(+ ALL_FIELDS[self.field.field_id() as usize],+ Field::Text(TextField::Site)+ ) || matches!(+ ALL_FIELDS[self.field.field_id() as usize],+ Field::Text(TextField::Domain)+ ))+ {+ let term: String = self+ .patterns+ .iter()+ .filter_map(|p| match p {+ PatternPart::Raw(s) => Some(s.clone()),+ PatternPart::Wildcard => None,+ PatternPart::Anchor => None,+ })+ .collect();++ let field_no_tokenizer = match ALL_FIELDS[self.field.field_id() as usize] {+ Field::Text(TextField::Site) => Field::Text(TextField::SiteNoTokenizer),+ Field::Text(TextField::Domain) => Field::Text(TextField::DomainNoTokenizer),+ _ => unreachable!(),+ };++ let tv_field = reader+ .schema()+ .get_field(field_no_tokenizer.name())+ .unwrap();++ let term = tantivy::Term::from_field_text(tv_field, &term);++ let opt = match field_no_tokenizer {+ Field::Text(t) => t.index_option(),+ Field::Fast(_) => unreachable!(),+ };++ match reader.inverted_index(tv_field)?.read_postings(&term, opt)? {+ Some(posting) => {+ return Ok(Some(PatternScorer::FastSiteDomain(+ FastSiteDomainPatternScorer {+ similarity_weight,+ posting,+ fieldnorm_reader,+ },+ )));+ }+ None => {+ return Ok(None);+ }+ }+ }++ let mut term_postings_list = Vec::with_capacity(self.raw_terms.len());for term in &self.raw_terms {if let Some(postings) = reader.inverted_index(term.field())?@@ -188,7 +255,7 @@ impl PatternWeight {))),}?;- Ok(Some(PatternScorer::new(+ Ok(Some(PatternScorer::Normal(NormalPatternScorer::new(similarity_weight,term_postings_list,fieldnorm_reader,@@ -196,7 +263,7 @@ impl PatternWeight {reader.segment_id(),num_tokens_fastfield,self.fastfield_reader.clone(),- )))+ ))))}}@@ -232,19 +299,111 @@ impl tantivy::query::Weight for PatternWeight {}let fieldnorm_reader = self.fieldnorm_reader(reader)?;let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);- let phrase_count = scorer.phrase_count();+ let term_freq = scorer.term_freq();let mut explanation = Explanation::new("Pattern Scorer", scorer.score());explanation.add_detail(self.similarity_weight.as_ref().unwrap()- .explain(fieldnorm_id, phrase_count),+ .explain(fieldnorm_id, term_freq),);Ok(explanation)}}-struct PatternScorer {+enum PatternScorer {+ Normal(NormalPatternScorer),+ FastSiteDomain(FastSiteDomainPatternScorer),+}++impl Scorer for PatternScorer {+ fn score(&mut self) -> Score {+ match self {+ PatternScorer::Normal(scorer) => scorer.score(),+ PatternScorer::FastSiteDomain(scorer) => scorer.score(),+ }+ }+}++impl DocSet for PatternScorer {+ fn advance(&mut self) -> DocId {+ match self {+ PatternScorer::Normal(scorer) => scorer.advance(),+ PatternScorer::FastSiteDomain(scorer) => scorer.advance(),+ }+ }++ fn seek(&mut self, target: DocId) -> DocId {+ match self {+ PatternScorer::Normal(scorer) => scorer.seek(target),+ PatternScorer::FastSiteDomain(scorer) => scorer.seek(target),+ }+ }++ fn doc(&self) -> DocId {+ match self {+ PatternScorer::Normal(scorer) => scorer.doc(),+ PatternScorer::FastSiteDomain(scorer) => scorer.doc(),+ }+ }++ fn size_hint(&self) -> u32 {+ match self {+ PatternScorer::Normal(scorer) => scorer.size_hint(),+ PatternScorer::FastSiteDomain(scorer) => scorer.size_hint(),+ }+ }+}++impl PatternScorer {+ fn term_freq(&self) -> u32 {+ match self {+ PatternScorer::Normal(scorer) => scorer.phrase_count(),+ PatternScorer::FastSiteDomain(scorer) => scorer.term_freq(),+ }+ }+}++struct FastSiteDomainPatternScorer {+ similarity_weight: Option<Bm25Weight>,+ posting: SegmentPostings,+ fieldnorm_reader: FieldNormReader,+}+impl FastSiteDomainPatternScorer {+ fn term_freq(&self) -> u32 {+ self.posting.term_freq()+ }+}++impl Scorer for FastSiteDomainPatternScorer {+ fn score(&mut self) -> Score {+ self.similarity_weight+ .as_ref()+ .map(|w| {+ w.score(+ self.fieldnorm_reader.fieldnorm_id(self.doc()),+ self.posting.term_freq(),+ )+ })+ .unwrap_or_default()+ }+}++impl DocSet for FastSiteDomainPatternScorer {+ fn advance(&mut self) -> DocId {+ self.posting.advance()+ }++ fn doc(&self) -> DocId {+ self.posting.doc()+ }++ fn size_hint(&self) -> u32 {+ self.posting.size_hint()+ }+}++struct NormalPatternScorer {similarity_weight: Option<Bm25Weight>,fieldnorm_reader: FieldNormReader,intersection_docset: Intersection<SegmentPostings>,@@ -257,7 +416,7 @@ struct PatternScorer {segment_reader: Arc<fastfield_reader::SegmentReader>,}-impl PatternScorer {+impl NormalPatternScorer {fn new(similarity_weight: Option<Bm25Weight>,term_postings_list: Vec<SegmentPostings>,@@ -270,7 +429,7 @@ impl PatternScorer {let num_query_terms = term_postings_list.len();let segment_reader = fastfield_reader.get_segment(&segment);- Self {+ let mut s = Self {intersection_docset: Intersection::new(term_postings_list),num_query_terms,similarity_weight,@@ -281,7 +440,13 @@ impl PatternScorer {phrase_count: 0,num_tokens_field,segment_reader,+ };++ if !s.pattern_match() {+ s.advance();}++ s}fn phrase_count(&self) -> u32 {self.phrase_count@@ -303,7 +468,7 @@ impl PatternScorer {let mut intersection_len = self.left.len();let mut out = Vec::new();- let mut current_right_term = 1;+ let mut current_right_term = 0;let mut slop = 1;let num_tokens_doc: Option<u64> = self.segment_reader@@ -312,9 +477,14 @@ impl PatternScorer {.into();let num_tokens_doc = num_tokens_doc.unwrap();- for (i, pattern_part) in self.pattern.iter().enumerate().skip(1) {+ for (i, pattern_part) in self.pattern.iter().enumerate() {match pattern_part {SmallPatternPart::Term => {+ if current_right_term == 0 {+ current_right_term = 1;+ continue;+ }+{self.intersection_docset.docset_mut_specialized(current_right_term)@@ -365,7 +535,7 @@ impl PatternScorer {}}-impl Scorer for PatternScorer {+impl Scorer for NormalPatternScorer {fn score(&mut self) -> Score {self.similarity_weight.as_ref()@@ -378,7 +548,7 @@ impl Scorer for PatternScorer {}}-impl DocSet for PatternScorer {+impl DocSet for NormalPatternScorer {fn advance(&mut self) -> DocId {loop {let doc = self.intersection_docset.advance();@@ -388,15 +558,6 @@ impl DocSet for PatternScorer {}}- fn seek(&mut self, target: DocId) -> DocId {- debug_assert!(target >= self.doc());- let doc = self.intersection_docset.seek(target);- if doc == TERMINATED || self.pattern_match() {- return doc;- }- self.advance()- }-fn doc(&self) -> tantivy::DocId {self.intersection_docset.doc()}diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex fd0519a7..bace881a 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -771,7 +771,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "example".to_string(),- optic_program: Some(+ optic: Some(r#"Ranking(Signal("bm25_title"), 20000000);Ranking(Signal("host_centrality"), 0);@@ -788,7 +788,7 @@ mod tests {let res = searcher.search(&SearchQuery {query: "example".to_string(),- optic_program: Some(+ optic: Some(r#"Ranking(Signal("host_centrality"), 2000000)"#diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex bbdcd41c..bce162b0 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -173,7 +173,7 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {self.query = Some(query.clone());self.signal_coefficients = query- .optic_program+ .optic.as_ref().and_then(|p| optics::parse(p.as_str()).ok()).map(|optic| SignalCoefficient::from_optic(&optic));@@ -228,7 +228,7 @@ impl<T: AsRankingWebsite> Scorer<T> for Initial {fn set_query_info(&mut self, query: &SearchQuery) {self.signal_coefficients = query- .optic_program+ .optic.as_ref().and_then(|p| optics::parse(p.as_str()).ok()).map(|optic| SignalCoefficient::from_optic(&optic));diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 09bfe6cf..b49df0e9 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -663,7 +663,7 @@ struct SegmentReader {}pub struct SignalAggregator {- query: Option<Query>,+ query: Option<Arc<Query>>,query_signal_coefficients: Option<SignalCoefficient>,segment_reader: Option<SegmentReader>,personal_centrality: Option<Arc<online_harmonic::Scorer>>,@@ -731,7 +731,7 @@ impl SignalAggregator {selected_region: None,current_timestamp: None,linear_regression: None,- query,+ query: query.map(Arc::new),}}@@ -808,6 +808,10 @@ impl SignalAggregator {optic_rule_boosts = optic.rules.iter()+ .filter(|rule| match rule.action {+ optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,+ optics::Action::Discard => false,+ }).filter_map(|rule| {rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader)})diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 56cbd580..21cf1703 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -373,7 +373,7 @@ impl DistributedSearcher {let query = SearchQuery {query: query.query.clone(),num_results: 1,- optic_program: Some(include_str!("stackoverflow.optic").to_string()),+ optic: Some(include_str!("stackoverflow.optic").to_string()),..Default::default()};@@ -464,7 +464,7 @@ impl DistributedSearcher {&self,query: &SearchQuery,) -> Result<Option<Vec<DisplayedWebpage>>> {- if query.optic_program.is_some() || query.page > 0 {+ if query.optic.is_some() || query.page > 0 {return Ok(None);}@@ -473,7 +473,7 @@ impl DistributedSearcher {let mut query = SearchQuery {query: query.query.clone(),num_results: NUM_RESULTS,- optic_program: Some(include_str!("discussions.optic").to_string()),+ optic: Some(include_str!("discussions.optic").to_string()),site_rankings: query.site_rankings.clone(),return_ranking_signals: query.return_ranking_signals,..Default::default()diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 67ff6ce2..1438533d 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -60,7 +60,7 @@ pub struct SearchQuery {pub page: usize,pub num_results: usize,pub selected_region: Option<Region>,- pub optic_program: Option<String>,+ pub optic: Option<String>,pub site_rankings: Option<SiteRankings>,pub return_ranking_signals: bool,}@@ -84,7 +84,7 @@ impl Default for SearchQuery {page: Default::default(),num_results: NUM_RESULTS_PER_PAGE,selected_region: Default::default(),- optic_program: Default::default(),+ optic: Default::default(),site_rankings: Default::default(),return_ranking_signals: false,}diff --git a/justfile b/justfileindex 3a137168..a89006af 100644--- a/justfile+++ b/justfile@@ -18,7 +18,7 @@cd frontend; npm run dev@configure *ARGS:- pip3 install -r scripts/requirements.txt+ pip install -r scripts/requirements.txt./scripts/export_crossencoder./scripts/export_qa_model./scripts/export_abstractive_summary_modeldiff --git a/ltr/generate_data.py b/ltr/generate_data.pyindex a0cc6d46..b43fba57 100644--- a/ltr/generate_data.py+++ b/ltr/generate_data.py@@ -41,7 +41,7 @@ def search(q: str, top_n, optic=None):url = f"http://{STRACT_ENDPOINT}/beta/api/search"r = requests.post(- url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic_program": optic})+ url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic": optic})search_result = r.json()['webpages']res = []diff --git a/optics/src/lib.rs b/optics/src/lib.rsindex 5b6b8ff8..a95322f3 100644--- a/optics/src/lib.rs+++ b/optics/src/lib.rs@@ -20,7 +20,6 @@ mod lexer;use std::convert::TryFrom;use ast::RankingCoeff;-use logos::Logos;use serde::{Deserialize, Serialize};use thiserror::Error;@@ -154,21 +153,19 @@ impl TryFrom<RawMatchPart> for Matching {let mut pattern = Vec::new();if matches!(&loc, MatchLocation::Schema) {- for tok in PatternToken::lexer(&s) {+ for tok in PatternToken::lex(&s) {match tok {PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),PatternToken::Wildcard => return Err(Error::Pattern),PatternToken::Anchor => return Err(Error::Pattern),- PatternToken::Error => unreachable!("no patterns should fail lexing"),}}} else {- for tok in PatternToken::lexer(&s) {+ for tok in PatternToken::lex(&s) {match tok {PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),PatternToken::Wildcard => pattern.push(PatternPart::Wildcard),PatternToken::Anchor => pattern.push(PatternPart::Anchor),- PatternToken::Error => unreachable!("no patterns should fail lexing"),}}}@@ -180,19 +177,57 @@ impl TryFrom<RawMatchPart> for Matching {}}-#[derive(Logos, Debug)]+#[derive(Debug, PartialEq, Eq)]enum PatternToken {- #[regex(".*", |lex| lex.slice().to_string())]Raw(String),- #[token("*")]Wildcard,- #[token("|")]Anchor,+}+impl PatternToken {+ fn lex(s: &str) -> Vec<Self> {+ let mut tokens = Vec::new();++ let mut raw: Option<String> = None;++ for c in s.chars() {+ match c {+ '*' => {+ if let Some(cur_raw) = raw {+ let cur_raw = cur_raw.trim().to_string();+ tokens.push(Self::Raw(cur_raw));+ raw = None;+ }++ tokens.push(Self::Wildcard);+ }+ '|' => {+ if let Some(cur_raw) = raw {+ let cur_raw = cur_raw.trim().to_string();+ tokens.push(Self::Raw(cur_raw));+ raw = None;+ }++ tokens.push(Self::Anchor);+ }+ _ => {+ if raw.is_none() {+ raw = Some(String::new());+ }++ raw.as_mut().unwrap().push(c);+ }+ }+ }- #[error]- Error,+ if let Some(raw) = raw {+ let raw = raw.trim().to_string();+ tokens.push(Self::Raw(raw));+ }++ tokens+ }}#[derive(Debug, PartialEq, Eq, Clone)]@@ -290,3 +325,31 @@ impl SiteRankings {}}}++#[cfg(test)]+mod tests {+ use super::*;+ #[test]+ fn pattern_part() {+ let lex = PatternToken::lex("|test|");++ assert_eq!(+ lex,+ vec![+ PatternToken::Anchor,+ PatternToken::Raw("test".to_string()),+ PatternToken::Anchor+ ]+ );+ let lex = PatternToken::lex("test * string");++ assert_eq!(+ lex,+ vec![+ PatternToken::Raw("test".to_string()),+ PatternToken::Wildcard,+ PatternToken::Raw("string".to_string()),+ ]+ );+ }+}diff --git a/optics/testcases b/optics/testcasesindex d80320e3..227ddf88 160000--- a/optics/testcases+++ b/optics/testcases@@ -1 +1 @@-Subproject commit d80320e33b78ddb233cfec56af498f0703d34b12+Subproject commit 227ddf88dab040834f14de1d9efbeede8852e55bdiff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_modelindex 3f9efd5b..19840963 100755--- a/scripts/export_abstractive_summary_model+++ b/scripts/export_abstractive_summary_model@@ -1,4 +1,4 @@-#!/usr/bin/env python3+#!/usr/bin/env pythonfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizerfrom optimum.onnxruntime import ORTQuantizer, ORTModelForSeq2SeqLMfrom optimum.onnxruntime.configuration import AutoQuantizationConfigdiff --git a/scripts/export_crossencoder b/scripts/export_crossencoderindex 50475818..8e08053d 100755--- a/scripts/export_crossencoder+++ b/scripts/export_crossencoder@@ -1,4 +1,4 @@-#!/usr/bin/env python3+#!/usr/bin/env pythonimport osfrom optimum.onnxruntime import ORTModelForSequenceClassification, ORTQuantizerfrom optimum.onnxruntime.configuration import AutoQuantizationConfigdiff --git a/scripts/export_qa_model b/scripts/export_qa_modelindex 42be940d..39f057dc 100755--- a/scripts/export_qa_model+++ b/scripts/export_qa_model@@ -1,4 +1,4 @@-#!/usr/bin/env python3+#!/usr/bin/env pythonimport osfrom optimum.onnxruntime import ORTModelForQuestionAnswering, ORTQuantizerfrom optimum.onnxruntime.configuration import AutoQuantizationConfig