Skip to content

web search done right

update optics syntax to allow for multi-stage ranking

Mikkel Denkere0162eebccc0
diff --git a/Cargo.lock b/Cargo.lock
index e1eba30f..d1cdee35 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2046,6 +2046,7 @@ dependencies = [
name = "optics"
version = "0.1.0"
dependencies = [
+ "itertools",
"lalrpop",
"lalrpop-util",
"logos",
diff --git a/core/src/lib.rs b/core/src/lib.rs
index a7e93c1f..51c7d4a0 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -215,7 +215,7 @@ pub enum Error {
#[error("Spell dictionary error")]
Spell(#[from] crate::spell::dictionary::DictionaryError),
- #[error("Could not parse optic")]
+ #[error("Optic")]
Optics(#[from] optics::Error),
#[error("Query cannot be completely empty")]
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index eeea88e6..2de09e87 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -23,6 +23,7 @@ pub mod signal;
use std::sync::Arc;
+use ::optics::SiteRankings;
use initial::InitialScoreTweaker;
use tantivy::collector::Collector;
@@ -31,8 +32,17 @@ use crate::{
fastfield_cache::FastFieldCache,
inverted_index,
searcher::NUM_RESULTS_PER_PAGE,
- webgraph::centrality::{approximate_harmonic, topic},
- webpage::region::{Region, RegionCount},
+ webgraph::{
+ centrality::{
+ approximate_harmonic::{self, ApproximatedHarmonicCentrality, Scorer},
+ topic,
+ },
+ Node,
+ },
+ webpage::{
+ region::{Region, RegionCount},
+ Url,
+ },
};
pub use self::signal::*;
@@ -136,6 +146,24 @@ impl Ranker {
}
}
+pub fn online_centrality_scorer(
+ site_rankings: &SiteRankings,
+ approx_harmonic: &ApproximatedHarmonicCentrality,
+) -> Scorer {
+ let mut liked_nodes = Vec::new();
+ let mut disliked_nodes = Vec::new();
+
+ for site in &site_rankings.liked {
+ liked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());
+ }
+
+ for site in &site_rankings.disliked {
+ disliked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());
+ }
+
+ approx_harmonic.scorer(&liked_nodes, &disliked_nodes)
+}
+
#[cfg(test)]
mod tests {
use std::collections::HashMap;
@@ -767,8 +795,12 @@ mod tests {
original: "example".to_string(),
optic_program: Some(
r#"
- Ranking{Field("title"), 20000000};
- Ranking{Signal("host_centrality"), 0};
+ RankingPipeline {
+ Stage {
+ Ranking{Field("title"), 20000000},
+ Ranking{Signal("host_centrality"), 0}
+ }
+ }
"#
.to_string(),
),
@@ -786,7 +818,11 @@ mod tests {
original: "example".to_string(),
optic_program: Some(
r#"
- Ranking{Signal("host_centrality"), 2000000};
+ RankingPipeline {
+ Stage {
+ Ranking{Signal("host_centrality"), 2000000}
+ }
+ }
"#
.to_string(),
),
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 163b8c0e..f677abec 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -14,64 +14,39 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use optics::{Optic, SiteRankings};
+use optics::ast::RankingStage;
-use crate::{
- schema::Field,
- webgraph::{
- centrality::approximate_harmonic::{ApproximatedHarmonicCentrality, Scorer},
- Node,
- },
- webpage::Url,
-};
+use crate::schema::Field;
use super::{Signal, SignalAggregator};
pub const SCALE: f32 = 10.0;
pub trait CreateAggregator {
- fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator;
+ fn aggregator(&self) -> SignalAggregator;
}
-fn centrality_scorer(
- site_rankings: &SiteRankings,
- approx_harmonic: &ApproximatedHarmonicCentrality,
-) -> Scorer {
- let mut liked_nodes = Vec::new();
- let mut disliked_nodes = Vec::new();
-
- for site in &site_rankings.liked {
- liked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());
- }
-
- for site in &site_rankings.disliked {
- disliked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());
- }
-
- approx_harmonic.scorer(&liked_nodes, &disliked_nodes)
-}
-
-impl CreateAggregator for Optic {
- fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator {
- let mut aggregator = SignalAggregator::new(
+impl CreateAggregator for RankingStage {
+ fn aggregator(&self) -> SignalAggregator {
+ let aggregator = SignalAggregator::new(
self.coefficients
- .clone()
- .into_iter()
- .filter_map(|(name, coeff)| {
- Signal::from_string(name).map(|signal| (signal, coeff))
+ .iter()
+ .filter_map(|coeff| match &coeff.target {
+ optics::ast::RankingTarget::Signal(s) => {
+ Signal::from_string(s.clone()).map(|s| (s, coeff.score))
+ }
+ optics::ast::RankingTarget::Field(_) => None,
+ }),
+ self.coefficients
+ .iter()
+ .filter_map(|coeff| match &coeff.target {
+ optics::ast::RankingTarget::Signal(_) => None,
+ optics::ast::RankingTarget::Field(f) => Field::from_name(f.clone())
+ .and_then(|f| f.as_text())
+ .map(|f| (f, coeff.score)),
}),
- self.boosts.clone().into_iter().filter_map(|(name, boost)| {
- match Field::from_name(name) {
- Some(field) => field.as_text().map(|text_field| (text_field, boost)),
- _ => None,
- }
- }),
);
- if let Some(approx) = approx {
- aggregator.add_personal_harmonic(centrality_scorer(&self.site_rankings, approx));
- }
-
aggregator
}
}
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index d732f1db..d001b5dc 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -107,7 +107,19 @@ trait Scorer<T: AsRankingWebsite>: Send + Sync {
fn score(&self, websites: &mut [T]);
}
-struct ReRanker {}
+struct ReRanker {
+ crossencoder: f64,
+ prev_score: f64,
+}
+
+impl Default for ReRanker {
+ fn default() -> Self {
+ Self {
+ crossencoder: 1.0,
+ prev_score: 1.0,
+ }
+ }
+}
impl<T: AsRankingWebsite> Scorer<T> for ReRanker {
fn score(&self, websites: &mut [T]) {
@@ -116,13 +128,26 @@ impl<T: AsRankingWebsite> Scorer<T> for ReRanker {
}
}
-struct PrioritizeText {}
+struct PrioritizeText {
+ bm25: f64,
+ prev_score: f64,
+}
+
+impl Default for PrioritizeText {
+ fn default() -> Self {
+ Self {
+ bm25: 1.0,
+ prev_score: 1.0,
+ }
+ }
+}
impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {
fn score(&self, websites: &mut [T]) {
for website in websites {
let bm25 = website.as_ranking().pointer.score.bm25 as f64;
- website.as_mut_ranking().score += bm25;
+ let prev_score = website.as_ranking().score;
+ website.as_mut_ranking().score = self.bm25 * bm25 + self.prev_score * prev_score;
}
}
}
@@ -208,9 +233,9 @@ pub struct RankingPipeline<T: AsRankingWebsite> {
impl<T: AsRankingWebsite> RankingPipeline<T> {
fn create() -> Self {
let last_stage = RankingStage {
- scorer: Box::new(ReRanker {}),
+ scorer: Box::<ReRanker>::default(),
prev: Prev::Node(Box::new(RankingStage {
- scorer: Box::new(PrioritizeText {}),
+ scorer: Box::<PrioritizeText>::default(),
prev: Prev::Initial,
memory: None,
top_n: 100,
@@ -227,6 +252,8 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
}
pub fn for_query(query: &mut SearchQuery) -> Self {
+ dbg!(&query.optic_program);
+
let mut pipeline = Self::create();
pipeline.offset = query.offset;
@@ -406,4 +433,70 @@ mod tests {
prev = res;
}
}
+
+ #[test]
+ fn multistage_coefficients() {
+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {
+ optic_program: Some(
+ r#"
+ RankingPipeline {
+ Stage {},
+ Stage {
+ Ranking {
+ Signal("bm25", 3),
+ Signal("prev_score", 2),
+ },
+ },
+ Stage {
+ Ranking {
+ Signal("crossencoder", 4),
+ Signal("prev_score", 3),
+ },
+ },
+ }
+ "#
+ .to_string(),
+ ),
+ ..Default::default()
+ });
+
+ let w = RankingWebsite {
+ pointer: WebsitePointer {
+ score: Score {
+ bm25: 1.0,
+ total: 1.0,
+ },
+ hashes: Hashes {
+ site: Prehashed(0),
+ title: Prehashed(0),
+ url: Prehashed(0),
+ simhash: 0,
+ },
+ address: DocAddress {
+ segment: 0,
+ doc_id: 0,
+ },
+ },
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ topic_centrality: 0.0,
+ personal_centrality: 0.0,
+ query_centrality: 0.0,
+ title: String::new(),
+ clean_body: String::new(),
+ score: 1.0,
+ };
+
+ let mut test = [w.clone()];
+
+ pipeline.last_stage.scorer.score(&mut test);
+ assert_eq!(test[0].score, 5.0);
+
+ let mut test = [w];
+
+ if let Prev::Node(prev) = pipeline.last_stage.prev {
+ prev.scorer.score(&mut test);
+ assert_eq!(test[0].score, 3.0);
+ }
+ }
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 82d55af1..94a88888 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -29,7 +29,7 @@ use crate::query::Query;
use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::optics::CreateAggregator;
use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
-use crate::ranking::{Ranker, SignalAggregator};
+use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
use crate::spell::Correction;
use crate::webgraph::centrality::topic::TopicCentrality;
use crate::webpage::region::Region;
@@ -84,17 +84,27 @@ impl LocalSearcher {
}
fn parse_query(&self, query: &SearchQuery, optic: Option<&Optic>) -> Result<Query> {
- let query_aggregator = optic
- .as_ref()
- .map(|optic| {
- optic.aggregator(
- self.centrality_store
- .as_ref()
- .map(|centrality_store| &centrality_store.approx_harmonic),
- )
+ let mut query_aggregator = optic
+ .and_then(|optic| {
+ optic
+ .pipeline
+ .as_ref()
+ .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))
})
.unwrap_or_default();
+ if let (Some(optic), Some(approx_harmonic)) = (
+ optic,
+ self.centrality_store
+ .as_ref()
+ .map(|store| &store.approx_harmonic),
+ ) {
+ query_aggregator.add_personal_harmonic(online_centrality_scorer(
+ &optic.site_rankings,
+ approx_harmonic,
+ ));
+ }
+
let mut parsed_query = Query::parse(
&query.original,
self.index.schema(),
@@ -133,19 +143,33 @@ impl LocalSearcher {
optics.push(site_rankings.clone().into_optic())
}
- let optic = optics
- .into_iter()
- .fold(Optic::default(), |acc, elem| acc.merge(elem));
+ let mut optic = Optic::default();
+
+ for o in optics {
+ optic = optic.try_merge(o)?;
+ }
parsed_query.set_optic(&optic, &self.index);
+ let mut aggregator = optic
+ .pipeline
+ .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))
+ .unwrap_or_default();
+
+ if let Some(approx_harmonic) = self
+ .centrality_store
+ .as_ref()
+ .map(|store| &store.approx_harmonic)
+ {
+ aggregator.add_personal_harmonic(online_centrality_scorer(
+ &optic.site_rankings,
+ approx_harmonic,
+ ));
+ }
+
let mut ranker = Ranker::new(
self.index.region_count.clone(),
- optic.aggregator(
- self.centrality_store
- .as_ref()
- .map(|centrality_store| &centrality_store.approx_harmonic),
- ),
+ aggregator,
self.index.inverted_index.fastfield_cache(),
);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 391a6ee1..a127eedb 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -266,7 +266,6 @@ impl<S: Store> Webgraph<S> {
distances
}
- #[allow(unused)]
pub fn distances(&self, source: Node) -> HashMap<Node, usize> {
self.full
.as_ref()
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index e5e700be..d428cbab 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -1841,4 +1841,49 @@ mod tests {
assert_eq!(webpage.dmoz_description(), None)
}
+
+ #[test]
+ fn links() {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <title>Best website</title>
+ <meta name="meta1" content="value">
+ <link href="link.com" />
+ <script src="test.com"></script>
+ </head>
+ <body>
+ <a href="example.com">Link to example</a>
+ <p>{CONTENT}</p>
+ </body>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever");
+
+ assert_eq!(webpage.title(), Some("Best website".to_string()));
+
+ assert_eq!(
+ webpage.links(),
+ vec![
+ Link {
+ source: "https://www.example.com/whatever".to_string().into(),
+ destination: "example.com".to_string().into(),
+ text: "Link to example".to_string()
+ },
+ Link {
+ source: "https://www.example.com/whatever".to_string().into(),
+ destination: "test.com".to_string().into(),
+ text: String::new()
+ },
+ Link {
+ source: "https://www.example.com/whatever".to_string().into(),
+ destination: "link.com".to_string().into(),
+ text: String::new()
+ },
+ ]
+ );
+ }
}
diff --git a/optics-lsp/Cargo.lock b/optics-lsp/Cargo.lock
index f7a6605f..ba91e960 100644
--- a/optics-lsp/Cargo.lock
+++ b/optics-lsp/Cargo.lock
@@ -340,6 +340,7 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
name = "optics"
version = "0.1.0"
dependencies = [
+ "itertools",
"lalrpop",
"lalrpop-util",
"logos",
diff --git a/optics-lsp/optic.tmLanguage.json b/optics-lsp/optic.tmLanguage.json
index c8446442..257f04df 100644
--- a/optics-lsp/optic.tmLanguage.json
+++ b/optics-lsp/optic.tmLanguage.json
@@ -26,11 +26,11 @@
]
},
"keywords": {
- "match": "\\b(Matches|Signal|Field|Site|Url|Domain|Title|Description|Content|Schema|Action|Boost|Downrank|Discard)\\b",
+ "match": "\\b(Matches|Signal|Field|Site|Url|Domain|Title|Description|Content|Schema|Action|Boost|Downrank|Discard|Ranking|Stage)\\b",
"name": "entity.name.function"
},
"control": {
- "match": "\\b(DiscardNonMatching|Rule|Ranking|Like|Dislike)\\b",
+ "match": "\\b(DiscardNonMatching|Rule|Like|Dislike|RankingPipeline)\\b",
"name": "keyword.control"
},
"punctuation": {
diff --git a/optics-lsp/package.json b/optics-lsp/package.json
index b973b619..3dcd049d 100644
--- a/optics-lsp/package.json
+++ b/optics-lsp/package.json
@@ -1,6 +1,6 @@
{
"name": "Optics",
- "version": "0.0.4",
+ "version": "0.0.6",
"engines": {
"vscode": "^1.73.0"
},
diff --git a/optics-lsp/src/lib.rs b/optics-lsp/src/lib.rs
index 026b0bd3..bd81a0a6 100644
--- a/optics-lsp/src/lib.rs
+++ b/optics-lsp/src/lib.rs
@@ -154,6 +154,13 @@ impl OpticsBackend {
It consists of a `Matches` block and an optional `Action`. Any search result that matches the `Matches` block \
will have the `Action` applied to it. The action can either `Boost`, `Downrank` or `Discard` a result. An empty `Action` is \
equivalent to a `Boost` of 1.".to_string()),
+
+ optics::Token::RankingPipeline => Some("The final ranking consists of multiple stages in a pipeline. Each stage receives the
+ best scoring webpages from the stage before it, and uses more accurate yet computationally expensive algorithms to rank
+ the pages for the next stage.".to_string()),
+
+ optics::Token::Stage => Some("The final ranking consists of multiple stages in a pipeline. Each stage has different fields and
+ signals available to it.".to_string()),
optics::Token::Matches => Some("`Matches` dictates the set of criteria a search result should match in order to have the action applied to it. \
A search result must match all the parts of the `Matches` block in order to match the specific rule.".to_string()),
@@ -384,6 +391,7 @@ fn err_to_diagnostic(err: optics::Error, source: &str) -> Diagnostic {
..Default::default()
}
},
+ optics::Error::RankingStagesMismatch => unreachable!("this error cannot occur at compile time"),
}
}
diff --git a/optics/Cargo.toml b/optics/Cargo.toml
index be9b76ca..8c5269ac 100644
--- a/optics/Cargo.toml
+++ b/optics/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
thiserror = { workspace = true }
lalrpop-util = { workspace = true }
once_cell = { workspace = true }
+itertools = { workspace = true }
logos = { workspace = true }
serde = { workspace = true }
diff --git a/optics/src/ast.rs b/optics/src/ast.rs
index a875cc68..756bc9e1 100644
--- a/optics/src/ast.rs
+++ b/optics/src/ast.rs
@@ -16,6 +16,7 @@
use super::Error;
use super::Result as ModResult;
+use itertools::Itertools;
use lalrpop_util::lalrpop_mod;
use super::lexer;
@@ -25,13 +26,13 @@ lalrpop_mod!(pub parser, "/parser.rs");
pub static PARSER: once_cell::sync::Lazy<parser::BlocksParser> =
once_cell::sync::Lazy::new(parser::BlocksParser::new);
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
pub enum RankingTarget {
Signal(String),
Field(String),
}
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
pub struct RankingCoeff {
pub target: RankingTarget,
pub score: f64,
@@ -40,11 +41,38 @@ pub struct RankingCoeff {
#[derive(Debug, PartialEq)]
pub struct RawOptic {
pub rules: Vec<RawRule>,
- pub rankings: Vec<RankingCoeff>,
+ pub rankings: Vec<RankingPipeline>,
pub site_preferences: Vec<RawSitePreference>,
pub discard_non_matching: bool,
}
+#[derive(Debug, PartialEq, Clone)]
+pub struct RankingPipeline {
+ pub stages: Vec<RankingStage>,
+}
+impl RankingPipeline {
+ pub(crate) fn try_merge(&mut self, other_pipeline: RankingPipeline) -> ModResult<()> {
+ if self.stages.len() != other_pipeline.stages.len() {
+ return Err(Error::RankingStagesMismatch);
+ }
+
+ for (t, mut o) in self
+ .stages
+ .iter_mut()
+ .zip_eq(other_pipeline.stages.into_iter())
+ {
+ t.coefficients.append(&mut o.coefficients)
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct RankingStage {
+ pub coefficients: Vec<RankingCoeff>,
+}
+
impl From<Vec<RawOpticBlock>> for RawOptic {
fn from(blocks: Vec<RawOpticBlock>) -> Self {
let mut rules = Vec::new();
@@ -74,7 +102,7 @@ impl From<Vec<RawOpticBlock>> for RawOptic {
pub enum RawOpticBlock {
Rule(RawRule),
SitePreference(RawSitePreference),
- Ranking(RankingCoeff),
+ Ranking(RankingPipeline),
DiscardNonMatching,
}
@@ -148,12 +176,17 @@ mod tests {
let optic = parse(
r#"
// this is a normal comment
- Ranking{Signal("host_centrality"), 3};
/*
this is a block comment
*/
- Ranking{Signal("bm25"), 100};
- Ranking{Field("url"), 2};
+ RankingPipeline {
+ Stage {
+ Ranking{Signal("host_centrality"), 3},
+ Ranking{Signal("bm25"), 100},
+ Ranking{Field("url"), 2},
+ }
+ };
+
Rule {
Matches {
Url("/this/is/a/*/pattern")
@@ -187,20 +220,24 @@ mod tests {
action: None,
},
],
- rankings: vec![
- RankingCoeff {
- target: RankingTarget::Signal("host_centrality".to_string()),
- score: 3.0,
- },
- RankingCoeff {
- target: RankingTarget::Signal("bm25".to_string()),
- score: 100.0,
- },
- RankingCoeff {
- target: RankingTarget::Field("url".to_string()),
- score: 2.0,
- },
- ],
+ rankings: vec![RankingPipeline {
+ stages: vec![RankingStage {
+ coefficients: vec![
+ RankingCoeff {
+ target: RankingTarget::Signal("host_centrality".to_string()),
+ score: 3.0,
+ },
+ RankingCoeff {
+ target: RankingTarget::Signal("bm25".to_string()),
+ score: 100.0,
+ },
+ RankingCoeff {
+ target: RankingTarget::Field("url".to_string()),
+ score: 2.0,
+ },
+ ]
+ }]
+ }],
site_preferences: vec![],
discard_non_matching: false,
}
diff --git a/optics/src/lexer.rs b/optics/src/lexer.rs
index ff95aaa5..eb76060b 100644
--- a/optics/src/lexer.rs
+++ b/optics/src/lexer.rs
@@ -29,7 +29,9 @@ pub enum Token<'a> {
DiscardNonMatching,
Rule,
+ RankingPipeline,
Ranking,
+ Stage,
Signal,
Field,
Matches,
@@ -62,7 +64,9 @@ impl<'a> ToString for Token<'a> {
Token::CloseParenthesis => ")".to_string(),
Token::DiscardNonMatching => "DiscardNonMatching".to_string(),
Token::Rule => "Rule".to_string(),
+ Token::RankingPipeline => "RankingPipeline".to_string(),
Token::Ranking => "Ranking".to_string(),
+ Token::Stage => "Stage".to_string(),
Token::Signal => "Signal".to_string(),
Token::Field => "Field".to_string(),
Token::Matches => "Matches".to_string(),
@@ -115,8 +119,12 @@ enum Outer<'a> {
DiscardNonMatching,
#[token("Rule")]
Rule,
+ #[token("RankingPipeline")]
+ RankingPipeline,
#[token("Ranking")]
Ranking,
+ #[token("Stage")]
+ Stage,
#[token("Signal")]
Signal,
#[token("Field")]
@@ -295,6 +303,8 @@ impl<'source> Iterator for LexerBridge<'source> {
Outer::CloseParenthesis => Some(Ok((s.start, Token::CloseParenthesis, s.end))),
Outer::Rule => Some(Ok((s.start, Token::Rule, s.end))),
Outer::Ranking => Some(Ok((s.start, Token::Ranking, s.end))),
+ Outer::Stage => Some(Ok((s.start, Token::Stage, s.end))),
+ Outer::RankingPipeline => Some(Ok((s.start, Token::RankingPipeline, s.end))),
Outer::Signal => Some(Ok((s.start, Token::Signal, s.end))),
Outer::Field => Some(Ok((s.start, Token::Field, s.end))),
Outer::Matches => Some(Ok((s.start, Token::Matches, s.end))),
diff --git a/optics/src/lib.rs b/optics/src/lib.rs
index da5b212a..05594128 100644
--- a/optics/src/lib.rs
+++ b/optics/src/lib.rs
@@ -17,17 +17,18 @@
pub mod ast;
mod lexer;
-use std::{collections::HashMap, convert::TryFrom};
+use std::convert::TryFrom;
+use ast::RankingPipeline;
use logos::Logos;
use serde::{Deserialize, Serialize};
use thiserror::Error;
-use self::ast::{RankingTarget, RawAction, RawMatchPart, RawOptic, RawRule};
+use self::ast::{RawAction, RawMatchPart, RawOptic, RawRule};
pub use lexer::lex;
pub use lexer::Token;
-type Result<T> = std::result::Result<T, Error>;
+pub(crate) type Result<T> = std::result::Result<T, Error>;
#[derive(Error, Debug, Clone)]
pub enum Error {
@@ -57,6 +58,9 @@ pub enum Error {
#[error("Unknown parse error")]
Unknown(usize, usize),
+ #[error("Ranking stages mismatch")]
+ RankingStagesMismatch,
+
#[error("Unsupported pattern")]
Pattern,
}
@@ -77,20 +81,6 @@ impl TryFrom<RawOptic> for Optic {
rules.push(Rule::try_from(rule)?);
}
- let mut boosts = HashMap::new();
- let mut coefficients = HashMap::new();
-
- for ranking in raw.rankings {
- match ranking.target {
- RankingTarget::Signal(name) => {
- coefficients.insert(name, ranking.score);
- }
- RankingTarget::Field(name) => {
- boosts.insert(name, ranking.score);
- }
- }
- }
-
let mut liked_sites = Vec::new();
let mut disliked_sites = Vec::new();
@@ -103,8 +93,7 @@ impl TryFrom<RawOptic> for Optic {
Ok(Self {
rules,
- coefficients,
- boosts,
+ pipeline: raw.rankings.into_iter().next(),
discard_non_matching: raw.discard_non_matching,
site_rankings: SiteRankings {
liked: liked_sites,
@@ -233,18 +222,25 @@ pub enum Action {
#[derive(Debug, Default, Clone)]
pub struct Optic {
- pub coefficients: HashMap<String, f64>,
- pub boosts: HashMap<String, f64>,
+ pub pipeline: Option<RankingPipeline>,
pub site_rankings: SiteRankings,
pub rules: Vec<Rule>,
pub discard_non_matching: bool,
}
impl Optic {
- pub fn merge(mut self, mut other: Self) -> Self {
+ pub fn try_merge(mut self, mut other: Self) -> Result<Self> {
self.rules.append(&mut other.rules);
- self.coefficients.extend(other.coefficients.into_iter());
- self.boosts.extend(other.boosts.into_iter());
+
+ self.pipeline = match (self.pipeline, other.pipeline) {
+ (None, None) => None,
+ (None, Some(pipeline)) => Some(pipeline),
+ (Some(pipeline), None) => Some(pipeline),
+ (Some(mut self_pipeline), Some(other_pipeline)) => {
+ self_pipeline.try_merge(other_pipeline)?;
+ Some(self_pipeline)
+ }
+ };
self.discard_non_matching |= other.discard_non_matching;
@@ -260,7 +256,7 @@ impl Optic {
.blocked
.append(&mut other.site_rankings.blocked);
- self
+ Ok(self)
}
}
diff --git a/optics/src/parser.lalrpop b/optics/src/parser.lalrpop
index f03f7bc2..74a992f2 100644
--- a/optics/src/parser.lalrpop
+++ b/optics/src/parser.lalrpop
@@ -18,7 +18,7 @@ pub Blocks: Vec<RawOpticBlock> = <Sep<";", Block>>;
Block: RawOpticBlock = {
<Rule> => RawOpticBlock::Rule(<>),
- <Ranking> => RawOpticBlock::Ranking(<>),
+ <RankingPipeline> => RawOpticBlock::Ranking(<>),
<SitePreference> => RawOpticBlock::SitePreference(<>),
"DiscardNonMatching" => RawOpticBlock::DiscardNonMatching,
}
@@ -68,6 +68,18 @@ InnerAction: RawAction= {
"Discard" => RawAction::Discard,
}
+RankingPipeline: RankingPipeline = {
+ "RankingPipeline" "{" <Sep<",", RankingStage>> "}" => RankingPipeline {
+ stages: <>
+ }
+}
+
+RankingStage: RankingStage = {
+ "Stage" "{" <Sep<",", Ranking>> "}" => RankingStage {
+ coefficients: <>
+ }
+}
+
Ranking: RankingCoeff = {
"Ranking" "{" <target:RankingTarget> "," <l:@L> <score:Number> <r:@R> "}" =>? {
@@ -107,7 +119,9 @@ extern {
"DiscardNonMatching" => Token::DiscardNonMatching,
"Rule" => Token::Rule,
+ "RankingPipeline" => Token::RankingPipeline,
"Ranking" => Token::Ranking,
+ "Stage" => Token::Stage,
"Signal" => Token::Signal,
"Field" => Token::Field,
"Matches" => Token::Matches,