Skip to content

web search done right

handle more edge cases in optics correctly

Mikkel Denker363340820155
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index 9c78c87a..e9322171 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -131,7 +131,7 @@ pub async fn route(
.search(&SearchQuery {
query: query.clone(),
selected_region,
- optic: optic,
+ optic,
page: skip_pages,
site_rankings,
num_results: NUM_RESULTS_PER_PAGE,
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 8ded1f7a..80595c86 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -44,7 +44,7 @@ pub struct Query {
site_rankings: SiteRankings,
offset: usize,
region: Option<Region>,
- optic: Option<Optic>,
+ optics: Vec<Optic>,
top_n: usize,
}
@@ -89,23 +89,17 @@ impl Query {
let mut tantivy_query = Box::new(BooleanQuery::new(queries));
- let mut optic = query
- .site_rankings
- .clone()
- .map(|site_rankings| site_rankings.into_optic());
+ let mut optics = Vec::new();
+ if let Some(site_rankigns_optic) = query.site_rankings.clone().map(|sr| sr.into_optic()) {
+ optics.push(site_rankigns_optic);
+ }
if let Some(optic_program) = &query.optic {
- match optic.as_mut() {
- Some(inner) => {
- optic = Some(inner.clone().try_merge(Optic::parse(optic_program)?)?);
- }
- None => {
- optic = Some(Optic::parse(optic_program)?);
- }
- }
+ let optic = Optic::parse(optic_program)?;
+ optics.push(optic);
}
- if let Some(optic) = &optic {
+ for optic in &optics {
let mut subqueries = vec![(Occur::Must, tantivy_query.box_clone())];
subqueries.append(&mut optic.as_multiple_tantivy(&schema, &ctx.fastfield_reader));
tantivy_query = Box::new(BooleanQuery::new(subqueries));
@@ -113,13 +107,16 @@ impl Query {
Ok(Query {
terms,
- site_rankings: optic
- .as_ref()
- .map(|optic| optic.site_rankings.clone())
- .unwrap_or_default(),
+ site_rankings: optics.clone().into_iter().fold(
+ SiteRankings::default(),
+ |mut acc, el| {
+ acc.merge_into(el.site_rankings);
+ acc
+ },
+ ),
simple_terms_text,
tantivy_query,
- optic,
+ optics,
offset: query.num_results * query.page,
region: query.selected_region,
top_n: query.num_results,
@@ -134,8 +131,8 @@ impl Query {
&self.terms
}
- pub fn optic(&self) -> Option<&Optic> {
- self.optic.as_ref()
+ pub fn optics(&self) -> &[Optic] {
+ &self.optics
}
pub fn is_empty(&self) -> bool {
@@ -159,7 +156,19 @@ impl Query {
}
pub fn signal_coefficients(&self) -> Option<SignalCoefficient> {
- self.optic.as_ref().map(SignalCoefficient::from_optic)
+ if self.optics.is_empty() {
+ return None;
+ }
+
+ Some(
+ self.optics
+ .iter()
+ .fold(SignalCoefficient::default(), |mut acc, optic| {
+ let coeffs = SignalCoefficient::from_optic(optic);
+ acc.merge_into(coeffs);
+ acc
+ }),
+ )
}
}
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index cabdcecb..db365023 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -1269,4 +1269,336 @@ mod tests {
.webpages;
assert_eq!(res.len(), 1);
}
+
+ #[test]
+ fn active_optic_with_blocked_sites() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>This is an example website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ This is an example
+ </body>
+ </html>
+ "#,
+ crate::rand_words(1000)
+ ),
+ "https://example.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ host_topic: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let searcher = LocalSearcher::from(index);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "DiscardNonMatching; Rule { Matches { Title(\"is\") }, Action(Boost(0)) }"
+ .to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "DiscardNonMatching; Rule { Matches { Title(\"is\") }, Action(Boost(0)) }"
+ .to_string(),
+ ),
+ site_rankings: Some(SiteRankings {
+ liked: vec![],
+ disliked: vec![],
+ blocked: vec![String::from("example.com")],
+ }),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 0);
+ }
+
+ #[test]
+ fn empty_optic_noop() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>This is an example website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ This is an example
+ </body>
+ </html>
+ "#,
+ crate::rand_words(1000)
+ ),
+ "https://example.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ host_topic: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let searcher = LocalSearcher::from(index);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some("".to_string()),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some("Rule { Matches { Title(\"\") }, Action(Discard) }".to_string()),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+ }
+
+ #[test]
+ fn wildcard_edge_cases() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>This is an example website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ This is an example
+ </body>
+ </html>
+ "#,
+ crate::rand_words(1000)
+ ),
+ "https://example.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ host_topic: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Another thing with no words in common</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ This is an example
+ </body>
+ </html>
+ "#,
+ crate::rand_words(1000)
+ ),
+ "https://example.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ host_topic: None,
+ dmoz_description: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let searcher = LocalSearcher::from(index);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some("Rule { Matches { Title(\"*\") }, Action(Discard) }".to_string()),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 0);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some("Rule { Matches { Title(\"* is\") }, Action(Discard) }".to_string()),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Title(\"* This is\") }, Action(Discard) }".to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Title(\"example *\") }, Action(Discard) }".to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "Rule { Matches { Title(\"example website *\") }, Action(Discard) }"
+ .to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+ }
+
+ #[test]
+ fn empty_double_anchor() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ let mut page = Webpage {
+ html: Html::parse(
+ r#"
+ <html>
+ <head>
+ <title>This is an example website</title>
+ </head>
+ <body>
+ Test
+ </body>
+ </html>
+ "#,
+ "https://example.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ host_topic: None,
+ dmoz_description: None,
+ };
+
+ page.html.set_clean_text("".to_string());
+
+ index.insert(page).expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let searcher = LocalSearcher::from(index);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "DiscardNonMatching; Rule { Matches { Content(\"||\") }, Action(Boost(0)) }"
+ .to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+
+ let res = searcher
+ .search(&SearchQuery {
+ query: "example".to_string(),
+ optic: Some(
+ "DiscardNonMatching; Rule { Matches { Content(\"|\") }, Action(Boost(0)) }"
+ .to_string(),
+ ),
+ ..Default::default()
+ })
+ .unwrap()
+ .webpages;
+ assert_eq!(res.len(), 1);
+ }
}
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index bfead129..04d116f9 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -95,7 +95,11 @@ impl tantivy::query::Query for PatternQuery {
) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
let bm25_weight = match scoring {
tantivy::query::EnableScoring::Enabled(searcher) => {
- Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)
+ if self.raw_terms.is_empty() {
+ None
+ } else {
+ Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)
+ }
}
tantivy::query::EnableScoring::Disabled(_) => None,
};
@@ -150,6 +154,53 @@ impl PatternWeight {
return Ok(None);
}
+ let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {
+ Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),
+ Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),
+ Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),
+ Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),
+ Field::Text(TextField::Site) => Ok(FastField::NumSiteTokens),
+ Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),
+ Field::Text(TextField::FlattenedSchemaOrgJson) => {
+ Ok(FastField::NumFlattenedSchemaTokens)
+ }
+ field => Err(TantivyError::InvalidArgument(format!(
+ "{} is not supported in pattern query",
+ field.name()
+ ))),
+ }?;
+
+ // "*" matches everything
+ if self.raw_terms.is_empty()
+ && self
+ .patterns
+ .iter()
+ .any(|p| matches!(p, PatternPart::Wildcard))
+ {
+ return Ok(Some(PatternScorer::Everything(AllScorer {
+ doc: 0,
+ max_doc: reader.max_doc(),
+ })));
+ }
+
+ // "||" and "|" matches empty string
+
+ if self.raw_terms.is_empty()
+ && self
+ .patterns
+ .iter()
+ .all(|p| matches!(p, PatternPart::Anchor))
+ {
+ return Ok(Some(PatternScorer::EmptyField(EmptyFieldScorer {
+ num_tokens_fastfield,
+ segment_reader: self.fastfield_reader.get_segment(&reader.segment_id()),
+ all_scorer: AllScorer {
+ doc: 0,
+ max_doc: reader.max_doc(),
+ },
+ })));
+ }
+
let similarity_weight = self
.similarity_weight
.as_ref()
@@ -239,22 +290,6 @@ impl PatternWeight {
})
.collect();
- let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {
- Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),
- Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),
- Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),
- Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),
- Field::Text(TextField::Site) => Ok(FastField::NumSiteTokens),
- Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),
- Field::Text(TextField::FlattenedSchemaOrgJson) => {
- Ok(FastField::NumFlattenedSchemaTokens)
- }
- field => Err(TantivyError::InvalidArgument(format!(
- "{} is not supported in pattern query",
- field.name()
- ))),
- }?;
-
Ok(Some(PatternScorer::Normal(NormalPatternScorer::new(
similarity_weight,
term_postings_list,
@@ -314,6 +349,8 @@ impl tantivy::query::Weight for PatternWeight {
enum PatternScorer {
Normal(NormalPatternScorer),
FastSiteDomain(FastSiteDomainPatternScorer),
+ Everything(AllScorer),
+ EmptyField(EmptyFieldScorer),
}
impl Scorer for PatternScorer {
@@ -321,6 +358,8 @@ impl Scorer for PatternScorer {
match self {
PatternScorer::Normal(scorer) => scorer.score(),
PatternScorer::FastSiteDomain(scorer) => scorer.score(),
+ PatternScorer::Everything(scorer) => scorer.score(),
+ PatternScorer::EmptyField(scorer) => scorer.score(),
}
}
}
@@ -330,6 +369,8 @@ impl DocSet for PatternScorer {
match self {
PatternScorer::Normal(scorer) => scorer.advance(),
PatternScorer::FastSiteDomain(scorer) => scorer.advance(),
+ PatternScorer::Everything(scorer) => scorer.advance(),
+ PatternScorer::EmptyField(scorer) => scorer.advance(),
}
}
@@ -337,6 +378,8 @@ impl DocSet for PatternScorer {
match self {
PatternScorer::Normal(scorer) => scorer.seek(target),
PatternScorer::FastSiteDomain(scorer) => scorer.seek(target),
+ PatternScorer::Everything(scorer) => scorer.seek(target),
+ PatternScorer::EmptyField(scorer) => scorer.seek(target),
}
}
@@ -344,6 +387,8 @@ impl DocSet for PatternScorer {
match self {
PatternScorer::Normal(scorer) => scorer.doc(),
PatternScorer::FastSiteDomain(scorer) => scorer.doc(),
+ PatternScorer::Everything(scorer) => scorer.doc(),
+ PatternScorer::EmptyField(scorer) => scorer.doc(),
}
}
@@ -351,6 +396,8 @@ impl DocSet for PatternScorer {
match self {
PatternScorer::Normal(scorer) => scorer.size_hint(),
PatternScorer::FastSiteDomain(scorer) => scorer.size_hint(),
+ PatternScorer::Everything(scorer) => scorer.size_hint(),
+ PatternScorer::EmptyField(scorer) => scorer.size_hint(),
}
}
}
@@ -360,8 +407,102 @@ impl PatternScorer {
match self {
PatternScorer::Normal(scorer) => scorer.phrase_count(),
PatternScorer::FastSiteDomain(scorer) => scorer.term_freq(),
+ PatternScorer::Everything(_) => 0,
+ PatternScorer::EmptyField(_) => 0,
+ }
+ }
+}
+
+struct AllScorer {
+ doc: DocId,
+ max_doc: DocId,
+}
+
+impl DocSet for AllScorer {
+ fn advance(&mut self) -> DocId {
+ if self.doc + 1 >= self.max_doc {
+ self.doc = TERMINATED;
+ return TERMINATED;
+ }
+ self.doc += 1;
+ self.doc
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ if target >= self.max_doc {
+ self.doc = TERMINATED;
+ return TERMINATED;
+ }
+ self.doc = target;
+ self.doc
+ }
+
+ fn doc(&self) -> DocId {
+ self.doc
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.max_doc
+ }
+}
+
+impl Scorer for AllScorer {
+ fn score(&mut self) -> Score {
+ 1.0
+ }
+}
+
+struct EmptyFieldScorer {
+ segment_reader: Arc<fastfield_reader::SegmentReader>,
+ num_tokens_fastfield: FastField,
+ all_scorer: AllScorer,
+}
+
+impl EmptyFieldScorer {
+ fn num_tokes(&self, doc: DocId) -> u64 {
+ let s: Option<u64> = self
+ .segment_reader
+ .get_field_reader(&self.num_tokens_fastfield)
+ .get(&doc)
+ .into();
+ s.unwrap_or_default()
+ }
+}
+
+impl DocSet for EmptyFieldScorer {
+ fn advance(&mut self) -> DocId {
+ let mut doc = self.all_scorer.advance();
+
+ while doc != TERMINATED && self.num_tokes(doc) > 0 {
+ doc = self.all_scorer.advance();
+ }
+
+ doc
+ }
+
+ fn doc(&self) -> DocId {
+ self.all_scorer.doc()
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ self.all_scorer.seek(target);
+
+ if self.doc() != TERMINATED && self.num_tokes(self.all_scorer.doc()) > 0 {
+ self.advance()
+ } else {
+ self.doc()
}
}
+
+ fn size_hint(&self) -> u32 {
+ self.all_scorer.size_hint()
+ }
+}
+
+impl Scorer for EmptyFieldScorer {
+ fn score(&mut self) -> Score {
+ 1.0
+ }
}
struct FastSiteDomainPatternScorer {
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index b49df0e9..57c77c06 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -604,7 +604,7 @@ fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {
}
}
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
pub struct SignalCoefficient {
map: EnumMap<Signal, f64>,
}
@@ -633,6 +633,19 @@ impl SignalCoefficient {
}
}))
}
+
+ pub fn merge_into(&mut self, coeffs: SignalCoefficient) {
+ for signal in ALL_SIGNALS {
+ if let Some(coeff) = coeffs.get(&signal) {
+ match self.map.get_mut(signal) {
+ Some(existing_coeff) => *existing_coeff += coeff,
+ None => {
+ self.map.insert(signal, coeff);
+ }
+ }
+ }
+ }
+ }
}
#[derive(Clone)]
@@ -803,29 +816,27 @@ impl SignalAggregator {
fastfield_reader: &fastfield_reader::FastFieldReader,
) -> Vec<RuleBoost> {
let mut optic_rule_boosts = Vec::new();
+
if let Some(query) = &self.query {
- if let Some(optic) = query.optic() {
- optic_rule_boosts = optic
- .rules
- .iter()
- .filter(|rule| match rule.action {
- optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,
- optics::Action::Discard => false,
- })
- .filter_map(|rule| {
- rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader)
- })
- .map(|(_, rule)| RuleBoost {
- docset: rule
- .query
- .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))
- .unwrap()
- .scorer(segment_reader, 0.0)
- .unwrap(),
- boost: rule.boost,
- })
- .collect();
- }
+ optic_rule_boosts = query
+ .optics()
+ .iter()
+ .flat_map(|o| o.rules.iter())
+ .filter(|rule| match rule.action {
+ optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,
+ optics::Action::Discard => false,
+ })
+ .filter_map(|rule| rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader))
+ .map(|(_, rule)| RuleBoost {
+ docset: rule
+ .query
+ .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))
+ .unwrap()
+ .scorer(segment_reader, 0.0)
+ .unwrap(),
+ boost: rule.boost,
+ })
+ .collect();
}
optic_rule_boosts
diff --git a/optics/src/lib.rs b/optics/src/lib.rs
index a95322f3..671f3aab 100644
--- a/optics/src/lib.rs
+++ b/optics/src/lib.rs
@@ -267,28 +267,6 @@ impl Optic {
pub fn parse(optic: &str) -> Result<Self> {
parse(optic)
}
-
- pub fn try_merge(mut self, mut other: Self) -> Result<Self> {
- self.rules.append(&mut other.rules);
-
- self.rankings.append(&mut other.rankings);
-
- self.discard_non_matching |= other.discard_non_matching;
-
- self.site_rankings
- .liked
- .append(&mut other.site_rankings.liked);
-
- self.site_rankings
- .disliked
- .append(&mut other.site_rankings.disliked);
-
- self.site_rankings
- .blocked
- .append(&mut other.site_rankings.blocked);
-
- Ok(self)
- }
}
#[derive(Debug, PartialEq, Eq, Clone)]
@@ -324,6 +302,12 @@ impl SiteRankings {
..Default::default()
}
}
+
+ pub fn merge_into(&mut self, site_rankings: SiteRankings) {
+ self.liked.extend(site_rankings.liked);
+ self.disliked.extend(site_rankings.disliked);
+ self.blocked.extend(site_rankings.blocked);
+ }
}
#[cfg(test)]