Skip to content

web search done right

fixed a bunch of problems with pattern_query implementation and wrote some tests to make sure it works correctly

Mikkel Denker5ab900eea599
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rs
index 6f40982d..36447e83 100644
--- a/core/benches/search-preindexed-optic.rs
+++ b/core/benches/search-preindexed-optic.rs
@@ -17,7 +17,7 @@ macro_rules! bench {
$searcher
.search(&SearchQuery {
query: $query.to_string(),
- optic_program: Some($optic.to_string()),
+ optic: Some($optic.to_string()),
..Default::default()
})
.unwrap()
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index 49b38343..9c78c87a 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -131,7 +131,7 @@ pub async fn route(
.search(&SearchQuery {
query: query.clone(),
selected_region,
- optic_program: optic,
+ optic: optic,
page: skip_pages,
site_rankings,
num_results: NUM_RESULTS_PER_PAGE,
diff --git a/core/src/query/intersection.rs b/core/src/query/intersection.rs
index 7a3a31e7..101e3941 100644
--- a/core/src/query/intersection.rs
+++ b/core/src/query/intersection.rs
@@ -20,6 +20,7 @@ use tantivy::{DocId, DocSet, Score, TERMINATED};
/// Creates a `DocSet` that iterate through the intersection of two or more `DocSet`s.
pub struct Intersection<TDocSet: DocSet = Box<dyn Scorer>> {
docsets: Vec<TDocSet>,
+ old_idx: Vec<usize>,
}
fn go_to_first_doc<TDocSet: DocSet>(docsets: &mut [TDocSet]) -> DocId {
@@ -39,9 +40,18 @@ fn go_to_first_doc<TDocSet: DocSet>(docsets: &mut [TDocSet]) -> DocId {
impl<TDocSet: DocSet> Intersection<TDocSet> {
pub(crate) fn new(mut docsets: Vec<TDocSet>) -> Intersection<TDocSet> {
- docsets.sort_by_key(|docset| docset.size_hint());
+ let mut docsets: Vec<(usize, TDocSet)> = docsets.drain(..).enumerate().collect();
+ docsets.sort_by_key(|(_, docset)| docset.size_hint());
+ let (tmp, mut docsets): (Vec<usize>, Vec<TDocSet>) = docsets.into_iter().unzip();
+
+ let mut old_idx = vec![0; docsets.len()];
+
+ for (new_idx, t) in tmp.into_iter().enumerate() {
+ old_idx[t] = new_idx;
+ }
+
go_to_first_doc(&mut docsets);
- Intersection { docsets }
+ Intersection { docsets, old_idx }
}
}
@@ -106,6 +116,6 @@ where
impl<TDocSet: DocSet> Intersection<TDocSet> {
pub(crate) fn docset_mut_specialized(&mut self, ord: usize) -> &mut TDocSet {
- &mut self.docsets[ord]
+ &mut self.docsets[self.old_idx[ord]]
}
}
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index a7d741c3..8ded1f7a 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -94,7 +94,7 @@ impl Query {
.clone()
.map(|site_rankings| site_rankings.into_optic());
- if let Some(optic_program) = &query.optic_program {
+ if let Some(optic_program) = &query.optic {
match optic.as_mut() {
Some(inner) => {
optic = Some(inner.clone().try_merge(Optic::parse(optic_program)?)?);
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 98b8d916..cabdcecb 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -303,7 +303,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
Rule {
Matches {
@@ -325,7 +325,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
Rule {
Matches {
@@ -416,9 +416,7 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
- include_str!("../../../optics/testcases/quickstart.optic").to_string(),
- ),
+ optic: Some(include_str!("../../../optics/testcases/quickstart.optic").to_string()),
..Default::default()
})
.unwrap()
@@ -427,7 +425,7 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
include_str!("../../../optics/testcases/hacker_news.optic").to_string(),
),
..Default::default()
@@ -438,7 +436,7 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
include_str!("../../../optics/testcases/copycats_removal.optic").to_string(),
),
..Default::default()
@@ -549,7 +547,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
DiscardNonMatching;
Rule {
@@ -698,7 +696,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
Like(Site("www.a.com"));
Like(Site("www.b.com"));
@@ -816,7 +814,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
DiscardNonMatching;
Rule {
@@ -838,7 +836,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
DiscardNonMatching;
Rule {
@@ -860,7 +858,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
DiscardNonMatching;
Rule {
@@ -882,7 +880,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "website".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
DiscardNonMatching;
Rule {
@@ -944,7 +942,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "site:stackoverflow.com".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
DiscardNonMatching;
Rule {
@@ -1049,7 +1047,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "example".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
DiscardNonMatching;
Rule {
@@ -1113,11 +1111,162 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "avengers endgame".to_string(),
- optic_program: Some(include_str!("../searcher/discussions.optic").to_string()),
+ optic: Some(include_str!("../searcher/discussions.optic").to_string()),
..Default::default()
})
.unwrap()
.webpages;
assert!(res.is_empty());
}
+
+ #[test]
+ fn special_pattern_syntax() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>This is an example website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ This is an example
+ </body>
+ </html>
+ "#,
+ crate::rand_words(1000)
+ ),
+ "https://example.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ host_topic: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let searcher = LocalSearcher::from(index);
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].url, "https://example.com");
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some("Rule { Matches { Title(\"is\") }, Action(Discard) }".to_string()),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 0);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some("Rule { Matches { Title(\"|is\") }, Action(Discard) }".to_string()),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some("Rule { Matches { Title(\"|This\") }, Action(Discard) }".to_string()),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 0);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Title(\"|This an\") }, Action(Discard) }".to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Title(\"|This * an\") }, Action(Discard) }".to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 0);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Site(\"example.com\") }, Action(Discard) }".to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 0);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Site(\"|example.com\") }, Action(Discard) }".to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 0);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Site(\"|example.com|\") }, Action(Discard) }".to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 0);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Title(\"website.com|\") }, Action(Discard) }".to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+ }
}
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index b3ce4a41..bfead129 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -61,6 +61,7 @@ impl PatternQuery {
let mut raw_terms = Vec::new();
let tv_field = schema.get_field(Field::Text(field).name()).unwrap();
+ let mut new_patterns = Vec::with_capacity(patterns.len());
for pattern in &patterns {
match pattern {
@@ -68,17 +69,18 @@ impl PatternQuery {
let mut stream = field.tokenizer().token_stream(text);
while let Some(token) = stream.next() {
+ new_patterns.push(PatternPart::Raw(token.text.clone()));
let term = tantivy::Term::from_field_text(tv_field, &token.text);
raw_terms.push(term);
}
}
- PatternPart::Wildcard => {}
- PatternPart::Anchor => {}
+ PatternPart::Wildcard => new_patterns.push(PatternPart::Wildcard),
+ PatternPart::Anchor => new_patterns.push(PatternPart::Anchor),
}
}
Self {
- patterns,
+ patterns: new_patterns,
field: tv_field,
raw_terms,
fastfield_reader,
@@ -114,6 +116,7 @@ impl tantivy::query::Query for PatternQuery {
}
}
+#[derive(Debug)]
enum SmallPatternPart {
Term,
Wildcard,
@@ -143,14 +146,78 @@ impl PatternWeight {
reader: &SegmentReader,
boost: Score,
) -> tantivy::Result<Option<PatternScorer>> {
+ if self.patterns.is_empty() {
+ return Ok(None);
+ }
+
let similarity_weight = self
.similarity_weight
.as_ref()
.map(|weight| weight.boost_by(boost));
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
- let mut term_postings_list = Vec::new();
+ // if pattern is of form Site("|site|") or Domain("|domain|")
+ // we can use the field without tokenization to speed up the query significantly
+ if self.patterns.len() >= 2
+ && matches!(&self.patterns[0], PatternPart::Anchor)
+ && matches!(&self.patterns[self.patterns.len() - 1], PatternPart::Anchor)
+ && self.patterns[1..self.patterns.len() - 1]
+ .iter()
+ .all(|pattern| matches!(pattern, PatternPart::Raw(_)))
+ && (matches!(
+ ALL_FIELDS[self.field.field_id() as usize],
+ Field::Text(TextField::Site)
+ ) || matches!(
+ ALL_FIELDS[self.field.field_id() as usize],
+ Field::Text(TextField::Domain)
+ ))
+ {
+ let term: String = self
+ .patterns
+ .iter()
+ .filter_map(|p| match p {
+ PatternPart::Raw(s) => Some(s.clone()),
+ PatternPart::Wildcard => None,
+ PatternPart::Anchor => None,
+ })
+ .collect();
+
+ let field_no_tokenizer = match ALL_FIELDS[self.field.field_id() as usize] {
+ Field::Text(TextField::Site) => Field::Text(TextField::SiteNoTokenizer),
+ Field::Text(TextField::Domain) => Field::Text(TextField::DomainNoTokenizer),
+ _ => unreachable!(),
+ };
+
+ let tv_field = reader
+ .schema()
+ .get_field(field_no_tokenizer.name())
+ .unwrap();
+
+ let term = tantivy::Term::from_field_text(tv_field, &term);
+
+ let opt = match field_no_tokenizer {
+ Field::Text(t) => t.index_option(),
+ Field::Fast(_) => unreachable!(),
+ };
+
+ match reader.inverted_index(tv_field)?.read_postings(&term, opt)? {
+ Some(posting) => {
+ return Ok(Some(PatternScorer::FastSiteDomain(
+ FastSiteDomainPatternScorer {
+ similarity_weight,
+ posting,
+ fieldnorm_reader,
+ },
+ )));
+ }
+ None => {
+ return Ok(None);
+ }
+ }
+ }
+
+ let mut term_postings_list = Vec::with_capacity(self.raw_terms.len());
for term in &self.raw_terms {
if let Some(postings) = reader
.inverted_index(term.field())?
@@ -188,7 +255,7 @@ impl PatternWeight {
))),
}?;
- Ok(Some(PatternScorer::new(
+ Ok(Some(PatternScorer::Normal(NormalPatternScorer::new(
similarity_weight,
term_postings_list,
fieldnorm_reader,
@@ -196,7 +263,7 @@ impl PatternWeight {
reader.segment_id(),
num_tokens_fastfield,
self.fastfield_reader.clone(),
- )))
+ ))))
}
}
@@ -232,19 +299,111 @@ impl tantivy::query::Weight for PatternWeight {
}
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
- let phrase_count = scorer.phrase_count();
+ let term_freq = scorer.term_freq();
let mut explanation = Explanation::new("Pattern Scorer", scorer.score());
explanation.add_detail(
self.similarity_weight
.as_ref()
.unwrap()
- .explain(fieldnorm_id, phrase_count),
+ .explain(fieldnorm_id, term_freq),
);
Ok(explanation)
}
}
-struct PatternScorer {
+enum PatternScorer {
+ Normal(NormalPatternScorer),
+ FastSiteDomain(FastSiteDomainPatternScorer),
+}
+
+impl Scorer for PatternScorer {
+ fn score(&mut self) -> Score {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.score(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.score(),
+ }
+ }
+}
+
+impl DocSet for PatternScorer {
+ fn advance(&mut self) -> DocId {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.advance(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.advance(),
+ }
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.seek(target),
+ PatternScorer::FastSiteDomain(scorer) => scorer.seek(target),
+ }
+ }
+
+ fn doc(&self) -> DocId {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.doc(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.doc(),
+ }
+ }
+
+ fn size_hint(&self) -> u32 {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.size_hint(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.size_hint(),
+ }
+ }
+}
+
+impl PatternScorer {
+ fn term_freq(&self) -> u32 {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.phrase_count(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.term_freq(),
+ }
+ }
+}
+
+struct FastSiteDomainPatternScorer {
+ similarity_weight: Option<Bm25Weight>,
+ posting: SegmentPostings,
+ fieldnorm_reader: FieldNormReader,
+}
+impl FastSiteDomainPatternScorer {
+ fn term_freq(&self) -> u32 {
+ self.posting.term_freq()
+ }
+}
+
+impl Scorer for FastSiteDomainPatternScorer {
+ fn score(&mut self) -> Score {
+ self.similarity_weight
+ .as_ref()
+ .map(|w| {
+ w.score(
+ self.fieldnorm_reader.fieldnorm_id(self.doc()),
+ self.posting.term_freq(),
+ )
+ })
+ .unwrap_or_default()
+ }
+}
+
+impl DocSet for FastSiteDomainPatternScorer {
+ fn advance(&mut self) -> DocId {
+ self.posting.advance()
+ }
+
+ fn doc(&self) -> DocId {
+ self.posting.doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.posting.size_hint()
+ }
+}
+
+struct NormalPatternScorer {
similarity_weight: Option<Bm25Weight>,
fieldnorm_reader: FieldNormReader,
intersection_docset: Intersection<SegmentPostings>,
@@ -257,7 +416,7 @@ struct PatternScorer {
segment_reader: Arc<fastfield_reader::SegmentReader>,
}
-impl PatternScorer {
+impl NormalPatternScorer {
fn new(
similarity_weight: Option<Bm25Weight>,
term_postings_list: Vec<SegmentPostings>,
@@ -270,7 +429,7 @@ impl PatternScorer {
let num_query_terms = term_postings_list.len();
let segment_reader = fastfield_reader.get_segment(&segment);
- Self {
+ let mut s = Self {
intersection_docset: Intersection::new(term_postings_list),
num_query_terms,
similarity_weight,
@@ -281,7 +440,13 @@ impl PatternScorer {
phrase_count: 0,
num_tokens_field,
segment_reader,
+ };
+
+ if !s.pattern_match() {
+ s.advance();
}
+
+ s
}
fn phrase_count(&self) -> u32 {
self.phrase_count
@@ -303,7 +468,7 @@ impl PatternScorer {
let mut intersection_len = self.left.len();
let mut out = Vec::new();
- let mut current_right_term = 1;
+ let mut current_right_term = 0;
let mut slop = 1;
let num_tokens_doc: Option<u64> = self
.segment_reader
@@ -312,9 +477,14 @@ impl PatternScorer {
.into();
let num_tokens_doc = num_tokens_doc.unwrap();
- for (i, pattern_part) in self.pattern.iter().enumerate().skip(1) {
+ for (i, pattern_part) in self.pattern.iter().enumerate() {
match pattern_part {
SmallPatternPart::Term => {
+ if current_right_term == 0 {
+ current_right_term = 1;
+ continue;
+ }
+
{
self.intersection_docset
.docset_mut_specialized(current_right_term)
@@ -365,7 +535,7 @@ impl PatternScorer {
}
}
-impl Scorer for PatternScorer {
+impl Scorer for NormalPatternScorer {
fn score(&mut self) -> Score {
self.similarity_weight
.as_ref()
@@ -378,7 +548,7 @@ impl Scorer for PatternScorer {
}
}
-impl DocSet for PatternScorer {
+impl DocSet for NormalPatternScorer {
fn advance(&mut self) -> DocId {
loop {
let doc = self.intersection_docset.advance();
@@ -388,15 +558,6 @@ impl DocSet for PatternScorer {
}
}
- fn seek(&mut self, target: DocId) -> DocId {
- debug_assert!(target >= self.doc());
- let doc = self.intersection_docset.seek(target);
- if doc == TERMINATED || self.pattern_match() {
- return doc;
- }
- self.advance()
- }
-
fn doc(&self) -> tantivy::DocId {
self.intersection_docset.doc()
}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index fd0519a7..bace881a 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -771,7 +771,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "example".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
Ranking(Signal("bm25_title"), 20000000);
Ranking(Signal("host_centrality"), 0);
@@ -788,7 +788,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
query: "example".to_string(),
- optic_program: Some(
+ optic: Some(
r#"
Ranking(Signal("host_centrality"), 2000000)
"#
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index bbdcd41c..bce162b0 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -173,7 +173,7 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
self.query = Some(query.clone());
self.signal_coefficients = query
- .optic_program
+ .optic
.as_ref()
.and_then(|p| optics::parse(p.as_str()).ok())
.map(|optic| SignalCoefficient::from_optic(&optic));
@@ -228,7 +228,7 @@ impl<T: AsRankingWebsite> Scorer<T> for Initial {
fn set_query_info(&mut self, query: &SearchQuery) {
self.signal_coefficients = query
- .optic_program
+ .optic
.as_ref()
.and_then(|p| optics::parse(p.as_str()).ok())
.map(|optic| SignalCoefficient::from_optic(&optic));
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 09bfe6cf..b49df0e9 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -663,7 +663,7 @@ struct SegmentReader {
}
pub struct SignalAggregator {
- query: Option<Query>,
+ query: Option<Arc<Query>>,
query_signal_coefficients: Option<SignalCoefficient>,
segment_reader: Option<SegmentReader>,
personal_centrality: Option<Arc<online_harmonic::Scorer>>,
@@ -731,7 +731,7 @@ impl SignalAggregator {
selected_region: None,
current_timestamp: None,
linear_regression: None,
- query,
+ query: query.map(Arc::new),
}
}
@@ -808,6 +808,10 @@ impl SignalAggregator {
optic_rule_boosts = optic
.rules
.iter()
+ .filter(|rule| match rule.action {
+ optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,
+ optics::Action::Discard => false,
+ })
.filter_map(|rule| {
rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader)
})
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 56cbd580..21cf1703 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -373,7 +373,7 @@ impl DistributedSearcher {
let query = SearchQuery {
query: query.query.clone(),
num_results: 1,
- optic_program: Some(include_str!("stackoverflow.optic").to_string()),
+ optic: Some(include_str!("stackoverflow.optic").to_string()),
..Default::default()
};
@@ -464,7 +464,7 @@ impl DistributedSearcher {
&self,
query: &SearchQuery,
) -> Result<Option<Vec<DisplayedWebpage>>> {
- if query.optic_program.is_some() || query.page > 0 {
+ if query.optic.is_some() || query.page > 0 {
return Ok(None);
}
@@ -473,7 +473,7 @@ impl DistributedSearcher {
let mut query = SearchQuery {
query: query.query.clone(),
num_results: NUM_RESULTS,
- optic_program: Some(include_str!("discussions.optic").to_string()),
+ optic: Some(include_str!("discussions.optic").to_string()),
site_rankings: query.site_rankings.clone(),
return_ranking_signals: query.return_ranking_signals,
..Default::default()
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 67ff6ce2..1438533d 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -60,7 +60,7 @@ pub struct SearchQuery {
pub page: usize,
pub num_results: usize,
pub selected_region: Option<Region>,
- pub optic_program: Option<String>,
+ pub optic: Option<String>,
pub site_rankings: Option<SiteRankings>,
pub return_ranking_signals: bool,
}
@@ -84,7 +84,7 @@ impl Default for SearchQuery {
page: Default::default(),
num_results: NUM_RESULTS_PER_PAGE,
selected_region: Default::default(),
- optic_program: Default::default(),
+ optic: Default::default(),
site_rankings: Default::default(),
return_ranking_signals: false,
}
diff --git a/justfile b/justfile
index 3a137168..a89006af 100644
--- a/justfile
+++ b/justfile
@@ -18,7 +18,7 @@
cd frontend; npm run dev
@configure *ARGS:
- pip3 install -r scripts/requirements.txt
+ pip install -r scripts/requirements.txt
./scripts/export_crossencoder
./scripts/export_qa_model
./scripts/export_abstractive_summary_model
diff --git a/ltr/generate_data.py b/ltr/generate_data.py
index a0cc6d46..b43fba57 100644
--- a/ltr/generate_data.py
+++ b/ltr/generate_data.py
@@ -41,7 +41,7 @@ def search(q: str, top_n, optic=None):
url = f"http://{STRACT_ENDPOINT}/beta/api/search"
r = requests.post(
- url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic_program": optic})
+ url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic": optic})
search_result = r.json()['webpages']
res = []
diff --git a/optics/src/lib.rs b/optics/src/lib.rs
index 5b6b8ff8..a95322f3 100644
--- a/optics/src/lib.rs
+++ b/optics/src/lib.rs
@@ -20,7 +20,6 @@ mod lexer;
use std::convert::TryFrom;
use ast::RankingCoeff;
-use logos::Logos;
use serde::{Deserialize, Serialize};
use thiserror::Error;
@@ -154,21 +153,19 @@ impl TryFrom<RawMatchPart> for Matching {
let mut pattern = Vec::new();
if matches!(&loc, MatchLocation::Schema) {
- for tok in PatternToken::lexer(&s) {
+ for tok in PatternToken::lex(&s) {
match tok {
PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),
PatternToken::Wildcard => return Err(Error::Pattern),
PatternToken::Anchor => return Err(Error::Pattern),
- PatternToken::Error => unreachable!("no patterns should fail lexing"),
}
}
} else {
- for tok in PatternToken::lexer(&s) {
+ for tok in PatternToken::lex(&s) {
match tok {
PatternToken::Raw(s) => pattern.push(PatternPart::Raw(s)),
PatternToken::Wildcard => pattern.push(PatternPart::Wildcard),
PatternToken::Anchor => pattern.push(PatternPart::Anchor),
- PatternToken::Error => unreachable!("no patterns should fail lexing"),
}
}
}
@@ -180,19 +177,57 @@ impl TryFrom<RawMatchPart> for Matching {
}
}
-#[derive(Logos, Debug)]
+#[derive(Debug, PartialEq, Eq)]
enum PatternToken {
- #[regex(".*", |lex| lex.slice().to_string())]
Raw(String),
- #[token("*")]
Wildcard,
- #[token("|")]
Anchor,
+}
+impl PatternToken {
+ fn lex(s: &str) -> Vec<Self> {
+ let mut tokens = Vec::new();
+
+ let mut raw: Option<String> = None;
+
+ for c in s.chars() {
+ match c {
+ '*' => {
+ if let Some(cur_raw) = raw {
+ let cur_raw = cur_raw.trim().to_string();
+ tokens.push(Self::Raw(cur_raw));
+ raw = None;
+ }
+
+ tokens.push(Self::Wildcard);
+ }
+ '|' => {
+ if let Some(cur_raw) = raw {
+ let cur_raw = cur_raw.trim().to_string();
+ tokens.push(Self::Raw(cur_raw));
+ raw = None;
+ }
+
+ tokens.push(Self::Anchor);
+ }
+ _ => {
+ if raw.is_none() {
+ raw = Some(String::new());
+ }
+
+ raw.as_mut().unwrap().push(c);
+ }
+ }
+ }
- #[error]
- Error,
+ if let Some(raw) = raw {
+ let raw = raw.trim().to_string();
+ tokens.push(Self::Raw(raw));
+ }
+
+ tokens
+ }
}
#[derive(Debug, PartialEq, Eq, Clone)]
@@ -290,3 +325,31 @@ impl SiteRankings {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn pattern_part() {
+ let lex = PatternToken::lex("|test|");
+
+ assert_eq!(
+ lex,
+ vec![
+ PatternToken::Anchor,
+ PatternToken::Raw("test".to_string()),
+ PatternToken::Anchor
+ ]
+ );
+ let lex = PatternToken::lex("test * string");
+
+ assert_eq!(
+ lex,
+ vec![
+ PatternToken::Raw("test".to_string()),
+ PatternToken::Wildcard,
+ PatternToken::Raw("string".to_string()),
+ ]
+ );
+ }
+}
diff --git a/optics/testcases b/optics/testcases
index d80320e3..227ddf88 160000
--- a/optics/testcases
+++ b/optics/testcases
@@ -1 +1 @@
-Subproject commit d80320e33b78ddb233cfec56af498f0703d34b12
+Subproject commit 227ddf88dab040834f14de1d9efbeede8852e55b
diff --git a/scripts/export_abstractive_summary_model b/scripts/export_abstractive_summary_model
index 3f9efd5b..19840963 100755
--- a/scripts/export_abstractive_summary_model
+++ b/scripts/export_abstractive_summary_model
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
from optimum.onnxruntime import ORTQuantizer, ORTModelForSeq2SeqLM
from optimum.onnxruntime.configuration import AutoQuantizationConfig
diff --git a/scripts/export_crossencoder b/scripts/export_crossencoder
index 50475818..8e08053d 100755
--- a/scripts/export_crossencoder
+++ b/scripts/export_crossencoder
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
import os
from optimum.onnxruntime import ORTModelForSequenceClassification, ORTQuantizer
from optimum.onnxruntime.configuration import AutoQuantizationConfig
diff --git a/scripts/export_qa_model b/scripts/export_qa_model
index 42be940d..39f057dc 100755
--- a/scripts/export_qa_model
+++ b/scripts/export_qa_model
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
import os
from optimum.onnxruntime import ORTModelForQuestionAnswering, ORTQuantizer
from optimum.onnxruntime.configuration import AutoQuantizationConfig