Skip to content

web search done right

10k short optic and optimizations to make large optics faster

Mikkel Denker1a8f1ec095b2
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rs
index 2659d427..4e3aa3e8 100644
--- a/core/benches/search-preindexed-optic.rs
+++ b/core/benches/search-preindexed-optic.rs
@@ -30,7 +30,7 @@ macro_rules! bench {
pub fn criterion_benchmark(c: &mut Criterion) {
let index = Index::open(INDEX_PATH).unwrap();
let searcher = LocalSearcher::new(index);
- let optic = include_str!("../../optics/testcases/hacker_news.optic");
+ let optic = include_str!("../../optics/testcases/10k_short.optic");
for _ in 0..10 {
bench!("the", searcher, optic, c);
diff --git a/core/src/api/optics.rs b/core/src/api/optics.rs
index 0c3919a1..ccbb97f4 100644
--- a/core/src/api/optics.rs
+++ b/core/src/api/optics.rs
@@ -18,7 +18,7 @@ use super::HtmlTemplate;
use askama::Template;
use axum::response::IntoResponse;
-pub const DEFAULT_OPTICS: [OpticLink; 3] = [
+pub const DEFAULT_OPTICS: [OpticLink; 4] = [
OpticLink {
name: "Copycats removal",
url:
@@ -35,6 +35,11 @@ pub const DEFAULT_OPTICS: [OpticLink; 3] = [
url: "https://raw.githubusercontent.com/StractOrg/sample-optics/main/discussions.optic",
description: "Only return results from forums or similar types of QA pages.",
},
+ OpticLink {
+ name: "10K Short",
+ url: "https://raw.githubusercontent.com/StractOrg/sample-optics/main/10k_short.optic",
+ description: "Remove the top 10,000 most popular websites from search results.",
+ },
];
#[derive(Debug, Clone)]
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index b4da0eaa..d9c955f6 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -640,7 +640,7 @@ mod tests {
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
let result =
@@ -669,7 +669,7 @@ mod tests {
let ctx = index.local_search_ctx();
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -715,7 +715,7 @@ mod tests {
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -759,7 +759,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -803,7 +803,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -880,7 +880,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -934,7 +934,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -977,7 +977,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -1045,7 +1045,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -1072,7 +1072,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -1102,7 +1102,7 @@ mod tests {
let ctx = index.local_search_ctx();
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
let result =
@@ -1151,7 +1151,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -1223,7 +1223,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -1287,7 +1287,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
@@ -1337,7 +1337,7 @@ mod tests {
)
.expect("Failed to parse query");
let ranker = Ranker::new(
- SignalAggregator::new(Some(query.clone())),
+ SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
);
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index bff62667..80f26ef7 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -106,13 +106,10 @@ impl Query {
Ok(Query {
terms,
- site_rankings: optics.clone().into_iter().fold(
- SiteRankings::default(),
- |mut acc, el| {
- acc.merge_into(el.site_rankings);
- acc
- },
- ),
+ site_rankings: optics.iter().fold(SiteRankings::default(), |mut acc, el| {
+ acc.merge_into(el.site_rankings.clone());
+ acc
+ }),
simple_terms_text,
tantivy_query,
optics,
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index cb6d25c4..264bb8dc 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -141,65 +141,83 @@ impl AsTantivyQuery for Matching {
fastfield_reader: &FastFieldReader,
) -> Box<dyn tantivy::query::Query> {
match &self.location {
- MatchLocation::Site => PatternQuery::new(
- self.pattern.clone(),
- TextField::Site,
- schema,
- fastfield_reader.clone(),
- )
- .box_clone(),
- MatchLocation::Url => PatternQuery::new(
- self.pattern.clone(),
- TextField::Url,
- schema,
- fastfield_reader.clone(),
- )
- .box_clone(),
- MatchLocation::Domain => PatternQuery::new(
- self.pattern.clone(),
- TextField::Domain,
- schema,
- fastfield_reader.clone(),
- )
- .box_clone(),
- MatchLocation::Title => PatternQuery::new(
- self.pattern.clone(),
- TextField::Title,
- schema,
- fastfield_reader.clone(),
- )
- .box_clone(),
- MatchLocation::Description => UnionQuery::from(vec![
+ MatchLocation::Site => ConstQuery::new(
PatternQuery::new(
self.pattern.clone(),
- TextField::Description,
+ TextField::Site,
schema,
fastfield_reader.clone(),
)
.box_clone(),
- PatternQuery::new(
+ 1.0,
+ )
+ .box_clone(),
+ MatchLocation::Url => Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::DmozDescription,
+ TextField::Url,
schema,
fastfield_reader.clone(),
- )
- .box_clone(),
+ )),
+ 1.0,
+ )),
+ MatchLocation::Domain => Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
+ self.pattern.clone(),
+ TextField::Domain,
+ schema,
+ fastfield_reader.clone(),
+ )),
+ 1.0,
+ )),
+ MatchLocation::Title => Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
+ self.pattern.clone(),
+ TextField::Title,
+ schema,
+ fastfield_reader.clone(),
+ )),
+ 1.0,
+ )),
+ MatchLocation::Description => UnionQuery::from(vec![
+ Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
+ self.pattern.clone(),
+ TextField::Description,
+ schema,
+ fastfield_reader.clone(),
+ )),
+ 1.0,
+ )) as Box<dyn tantivy::query::Query>,
+ Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
+ self.pattern.clone(),
+ TextField::DmozDescription,
+ schema,
+ fastfield_reader.clone(),
+ )),
+ 1.0,
+ )) as Box<dyn tantivy::query::Query>,
])
.box_clone(),
- MatchLocation::Content => PatternQuery::new(
- self.pattern.clone(),
- TextField::CleanBody,
- schema,
- fastfield_reader.clone(),
- )
- .box_clone(),
- MatchLocation::Schema => PatternQuery::new(
- self.pattern.clone(),
- TextField::FlattenedSchemaOrgJson,
- schema,
- fastfield_reader.clone(),
- )
- .box_clone(),
+ MatchLocation::Content => Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
+ self.pattern.clone(),
+ TextField::CleanBody,
+ schema,
+ fastfield_reader.clone(),
+ )),
+ 1.0,
+ )),
+ MatchLocation::Schema => Box::new(ConstQuery::new(
+ Box::new(PatternQuery::new(
+ self.pattern.clone(),
+ TextField::FlattenedSchemaOrgJson,
+ schema,
+ fastfield_reader.clone(),
+ )),
+ 1.0,
+ )),
}
}
}
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 04d116f9..bca06c57 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -36,6 +36,7 @@ use crate::{
#[derive(Clone)]
pub struct PatternQuery {
patterns: Vec<PatternPart>,
+ can_optimize_site_domain: bool,
field: tantivy::schema::Field,
raw_terms: Vec<tantivy::Term>,
fastfield_reader: FastFieldReader,
@@ -58,15 +59,49 @@ impl PatternQuery {
schema: &tantivy::schema::Schema,
fastfield_reader: FastFieldReader,
) -> Self {
- let mut raw_terms = Vec::new();
+ let field = Field::Text(field);
+ let tv_field = schema.get_field(field.name()).unwrap();
+
+ if can_optimize_site_domain(&patterns, field) {
+ if patterns.len() == 3 {
+ if let PatternPart::Raw(term) = &patterns[1] {
+ return Self {
+ patterns: Vec::new(),
+ field: tv_field,
+ can_optimize_site_domain: true,
+ raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],
+ fastfield_reader,
+ };
+ } else {
+ unreachable!()
+ }
+ } else {
+ let term: String = patterns
+ .iter()
+ .filter_map(|p| match p {
+ PatternPart::Raw(s) => Some(s.clone()),
+ PatternPart::Wildcard => None,
+ PatternPart::Anchor => None,
+ })
+ .collect();
+
+ return Self {
+ patterns,
+ field: tv_field,
+ can_optimize_site_domain: true,
+ raw_terms: vec![tantivy::Term::from_field_text(tv_field, &term)],
+ fastfield_reader,
+ };
+ }
+ }
- let tv_field = schema.get_field(Field::Text(field).name()).unwrap();
+ let mut raw_terms = Vec::with_capacity(patterns.len());
let mut new_patterns = Vec::with_capacity(patterns.len());
for pattern in &patterns {
match pattern {
PatternPart::Raw(text) => {
- let mut stream = field.tokenizer().token_stream(text);
+ let mut stream = field.as_text().unwrap().tokenizer().token_stream(text);
while let Some(token) = stream.next() {
new_patterns.push(PatternPart::Raw(token.text.clone()));
@@ -79,10 +114,13 @@ impl PatternQuery {
}
}
+ raw_terms.shrink_to_fit();
+
Self {
patterns: new_patterns,
field: tv_field,
raw_terms,
+ can_optimize_site_domain: false,
fastfield_reader,
}
}
@@ -104,6 +142,14 @@ impl tantivy::query::Query for PatternQuery {
tantivy::query::EnableScoring::Disabled(_) => None,
};
+ if self.can_optimize_site_domain {
+ return Ok(Box::new(FastSiteDomainPatternWeight {
+ term: self.raw_terms[0].clone(),
+ field: self.field,
+ similarity_weight: bm25_weight,
+ }));
+ }
+
Ok(Box::new(PatternWeight {
similarity_weight: bm25_weight,
raw_terms: self.raw_terms.clone(),
@@ -127,6 +173,121 @@ enum SmallPatternPart {
Anchor,
}
+/// if pattern is of form Site("|site|") or Domain("|domain|")
+/// we can use the field without tokenization to speed up the query significantly
+fn can_optimize_site_domain(patterns: &[PatternPart], field: Field) -> bool {
+ patterns.len() >= 2
+ && matches!(&patterns[0], PatternPart::Anchor)
+ && matches!(&patterns[patterns.len() - 1], PatternPart::Anchor)
+ && patterns[1..patterns.len() - 1]
+ .iter()
+ .all(|pattern| matches!(pattern, PatternPart::Raw(_)))
+ && (matches!(field, Field::Text(TextField::Site))
+ || matches!(field, Field::Text(TextField::Domain)))
+}
+
+struct FastSiteDomainPatternWeight {
+ term: tantivy::Term,
+ field: tantivy::schema::Field,
+ similarity_weight: Option<Bm25Weight>,
+}
+
+impl FastSiteDomainPatternWeight {
+ fn fieldnorm_reader(&self, reader: &SegmentReader) -> tantivy::Result<FieldNormReader> {
+ if self.similarity_weight.is_some() {
+ if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {
+ return Ok(fieldnorm_reader);
+ }
+ }
+ Ok(FieldNormReader::constant(reader.max_doc(), 1))
+ }
+
+ fn pattern_scorer(
+ &self,
+ reader: &SegmentReader,
+ boost: Score,
+ ) -> tantivy::Result<Option<FastSiteDomainPatternScorer>> {
+ let similarity_weight = self
+ .similarity_weight
+ .as_ref()
+ .map(|weight| weight.boost_by(boost));
+
+ let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+
+ let field_no_tokenizer = match ALL_FIELDS[self.field.field_id() as usize] {
+ Field::Text(TextField::Site) => Field::Text(TextField::SiteNoTokenizer),
+ Field::Text(TextField::Domain) => Field::Text(TextField::DomainNoTokenizer),
+ _ => unreachable!(),
+ };
+
+ let tv_field = reader
+ .schema()
+ .get_field(field_no_tokenizer.name())
+ .unwrap();
+
+ let opt = match field_no_tokenizer {
+ Field::Text(t) => t.index_option(),
+ Field::Fast(_) => unreachable!(),
+ };
+
+ match reader
+ .inverted_index(tv_field)?
+ .read_postings(&self.term, opt)?
+ {
+ Some(posting) => Ok(Some(FastSiteDomainPatternScorer {
+ similarity_weight,
+ posting,
+ fieldnorm_reader,
+ })),
+ None => Ok(None),
+ }
+ }
+}
+
+impl tantivy::query::Weight for FastSiteDomainPatternWeight {
+ fn scorer(
+ &self,
+ reader: &tantivy::SegmentReader,
+ boost: tantivy::Score,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
+ if let Some(scorer) = self.pattern_scorer(reader, boost)? {
+ Ok(Box::new(PatternScorer::FastSiteDomain(scorer)))
+ } else {
+ Ok(Box::new(EmptyScorer))
+ }
+ }
+
+ fn explain(
+ &self,
+ reader: &tantivy::SegmentReader,
+ doc: tantivy::DocId,
+ ) -> tantivy::Result<tantivy::query::Explanation> {
+ let scorer_opt = self.pattern_scorer(reader, 1.0)?;
+ if scorer_opt.is_none() {
+ return Err(TantivyError::InvalidArgument(format!(
+ "Document #({doc}) does not match (empty scorer)"
+ )));
+ }
+ let mut scorer = scorer_opt.unwrap();
+ if scorer.seek(doc) != doc {
+ return Err(TantivyError::InvalidArgument(format!(
+ "Document #({doc}) does not match"
+ )));
+ }
+ let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+ let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
+ let term_freq = scorer.term_freq();
+ let mut explanation = Explanation::new("Pattern Scorer", scorer.score());
+ explanation.add_detail(
+ self.similarity_weight
+ .as_ref()
+ .unwrap()
+ .explain(fieldnorm_id, term_freq),
+ );
+ Ok(explanation)
+ }
+}
+
struct PatternWeight {
similarity_weight: Option<Bm25Weight>,
patterns: Vec<PatternPart>,
@@ -208,65 +369,7 @@ impl PatternWeight {
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
- // if pattern is of form Site("|site|") or Domain("|domain|")
- // we can use the field without tokenization to speed up the query significantly
- if self.patterns.len() >= 2
- && matches!(&self.patterns[0], PatternPart::Anchor)
- && matches!(&self.patterns[self.patterns.len() - 1], PatternPart::Anchor)
- && self.patterns[1..self.patterns.len() - 1]
- .iter()
- .all(|pattern| matches!(pattern, PatternPart::Raw(_)))
- && (matches!(
- ALL_FIELDS[self.field.field_id() as usize],
- Field::Text(TextField::Site)
- ) || matches!(
- ALL_FIELDS[self.field.field_id() as usize],
- Field::Text(TextField::Domain)
- ))
- {
- let term: String = self
- .patterns
- .iter()
- .filter_map(|p| match p {
- PatternPart::Raw(s) => Some(s.clone()),
- PatternPart::Wildcard => None,
- PatternPart::Anchor => None,
- })
- .collect();
-
- let field_no_tokenizer = match ALL_FIELDS[self.field.field_id() as usize] {
- Field::Text(TextField::Site) => Field::Text(TextField::SiteNoTokenizer),
- Field::Text(TextField::Domain) => Field::Text(TextField::DomainNoTokenizer),
- _ => unreachable!(),
- };
-
- let tv_field = reader
- .schema()
- .get_field(field_no_tokenizer.name())
- .unwrap();
-
- let term = tantivy::Term::from_field_text(tv_field, &term);
-
- let opt = match field_no_tokenizer {
- Field::Text(t) => t.index_option(),
- Field::Fast(_) => unreachable!(),
- };
-
- match reader.inverted_index(tv_field)?.read_postings(&term, opt)? {
- Some(posting) => {
- return Ok(Some(PatternScorer::FastSiteDomain(
- FastSiteDomainPatternScorer {
- similarity_weight,
- posting,
- fieldnorm_reader,
- },
- )));
- }
- None => {
- return Ok(None);
- }
- }
- }
+ if can_optimize_site_domain(&self.patterns, ALL_FIELDS[self.field.field_id() as usize]) {}
let mut term_postings_list = Vec::with_capacity(self.raw_terms.len());
for term in &self.raw_terms {
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 57c77c06..2eb4773c 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -675,8 +675,14 @@ struct SegmentReader {
fastfield_reader: Arc<fastfield_reader::SegmentReader>,
}
+#[derive(Clone)]
+struct QueryData {
+ simple_terms: Vec<String>,
+ optic_rules: Vec<optics::Rule>,
+}
+
pub struct SignalAggregator {
- query: Option<Arc<Query>>,
+ query: Option<QueryData>,
query_signal_coefficients: Option<SignalCoefficient>,
segment_reader: Option<SegmentReader>,
personal_centrality: Option<Arc<online_harmonic::Scorer>>,
@@ -714,13 +720,20 @@ impl Clone for SignalAggregator {
impl std::fmt::Debug for SignalAggregator {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SignalAggregator")
- .field("query", &self.query)
+ .field(
+ "query",
+ &self
+ .query
+ .as_ref()
+ .map(|q| q.simple_terms.clone())
+ .unwrap_or_default(),
+ )
.finish()
}
}
impl SignalAggregator {
- pub fn new(query: Option<Query>) -> Self {
+ pub fn new(query: Option<&Query>) -> Self {
let query_signal_coefficients = query.as_ref().and_then(|q| q.signal_coefficients());
let fetch_time_ms_cache: Vec<_> = (0..1000)
@@ -731,6 +744,20 @@ impl SignalAggregator {
.map(|hours_since_update| 1.0 / ((hours_since_update as f64 + 1.0).log2()))
.collect();
+ let query = query.as_ref().map(|q| QueryData {
+ simple_terms: q.simple_terms().to_vec(),
+ optic_rules: q
+ .optics()
+ .iter()
+ .flat_map(|o| o.rules.iter())
+ .filter(|rule| match rule.action {
+ optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,
+ optics::Action::Discard => false,
+ })
+ .cloned()
+ .collect(),
+ });
+
Self {
segment_reader: None,
personal_centrality: None,
@@ -744,7 +771,7 @@ impl SignalAggregator {
selected_region: None,
current_timestamp: None,
linear_regression: None,
- query: query.map(Arc::new),
+ query,
}
}
@@ -757,12 +784,12 @@ impl SignalAggregator {
let schema = tv_searcher.schema();
if let Some(query) = &self.query {
- if !query.simple_terms().is_empty() {
+ if !query.simple_terms.is_empty() {
for signal in ALL_SIGNALS {
if let Some(text_field) = signal.as_textfield() {
let tv_field = schema.get_field(text_field.name()).unwrap();
let simple_query = itertools::intersperse(
- query.simple_terms().iter().map(|s| s.as_str()),
+ query.simple_terms.iter().map(|s| s.as_str()),
" ",
)
.collect::<String>();
@@ -819,13 +846,8 @@ impl SignalAggregator {
if let Some(query) = &self.query {
optic_rule_boosts = query
- .optics()
+ .optic_rules
.iter()
- .flat_map(|o| o.rules.iter())
- .filter(|rule| match rule.action {
- optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,
- optics::Action::Discard => false,
- })
.filter_map(|rule| rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader))
.map(|(_, rule)| RuleBoost {
docset: rule
@@ -852,9 +874,7 @@ impl SignalAggregator {
let schema = tv_searcher.schema();
if let Some(query) = &self.query {
- let simple_terms = query.simple_terms().to_vec();
-
- if simple_terms.len() < 2 {
+ if query.simple_terms.len() < 2 {
return proximity_scorers;
}
@@ -864,9 +884,9 @@ impl SignalAggregator {
for field in proximity_fields {
let tv_field = schema.get_field(field.name()).unwrap();
let tokenizer = field.tokenizer();
- let mut terms = Vec::with_capacity(simple_terms.len());
+ let mut terms = Vec::with_capacity(query.simple_terms.len());
- for term in &simple_terms {
+ for term in &query.simple_terms {
let mut stream = tokenizer.token_stream(term);
while let Some(term) = stream.next() {
let term = tantivy::Term::from_field_text(tv_field, &term.text);
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 63c2d3d1..167762fe 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -167,7 +167,7 @@ impl LocalSearcher {
RankingPipeline::ltr_for_query(&mut query, self.lambda_model.clone());
let parsed_query = self.parse_query(ctx, &query)?;
- let mut aggregator = SignalAggregator::new(Some(parsed_query.clone()));
+ let mut aggregator = SignalAggregator::new(Some(&parsed_query));
if let Some(store) = &self.centrality_store {
aggregator.set_personal_harmonic(online_centrality_scorer(
diff --git a/optics/testcases b/optics/testcases
index 227ddf88..0c60ed5f 160000
--- a/optics/testcases
+++ b/optics/testcases
@@ -1 +1 @@
-Subproject commit 227ddf88dab040834f14de1d9efbeede8852e55b
+Subproject commit 0c60ed5f0a6d144d22a30dc8cc12485327a67fe2
diff --git a/scripts/x_short_optic.py b/scripts/x_short_optic.py
new file mode 100644
index 00000000..6518dac9
--- /dev/null
+++ b/scripts/x_short_optic.py
@@ -0,0 +1,22 @@
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('million_short_file', type=str)
+parser.add_argument('-n', type=int, default=10_000)
+
+args = parser.parse_args()
+
+domains = []
+with open(args.million_short_file, 'r') as f:
+ for n, line in enumerate(f):
+ if n >= args.n:
+ break
+
+ domain = line.strip().split(',')[1]
+ domains.append(domain)
+
+optic = "// Generated from the following list: https://tranco-list.eu/ \n"
+for domain in domains:
+ optic += f"""Rule {{ Matches {{ Domain("|{domain}|") }}, Action(Discard) }};"""
+
+print(optic)