web search done right
update optics syntax to allow for multi-stage ranking
Mikkel Denkere0162eebccc0
diff --git a/Cargo.lock b/Cargo.lockindex e1eba30f..d1cdee35 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2046,6 +2046,7 @@ dependencies = [name = "optics"version = "0.1.0"dependencies = [+ "itertools","lalrpop","lalrpop-util","logos",diff --git a/core/src/lib.rs b/core/src/lib.rsindex a7e93c1f..51c7d4a0 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -215,7 +215,7 @@ pub enum Error {#[error("Spell dictionary error")]Spell(#[from] crate::spell::dictionary::DictionaryError),- #[error("Could not parse optic")]+ #[error("Optic")]Optics(#[from] optics::Error),#[error("Query cannot be completely empty")]diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex eeea88e6..2de09e87 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -23,6 +23,7 @@ pub mod signal;use std::sync::Arc;+use ::optics::SiteRankings;use initial::InitialScoreTweaker;use tantivy::collector::Collector;@@ -31,8 +32,17 @@ use crate::{fastfield_cache::FastFieldCache,inverted_index,searcher::NUM_RESULTS_PER_PAGE,- webgraph::centrality::{approximate_harmonic, topic},- webpage::region::{Region, RegionCount},+ webgraph::{+ centrality::{+ approximate_harmonic::{self, ApproximatedHarmonicCentrality, Scorer},+ topic,+ },+ Node,+ },+ webpage::{+ region::{Region, RegionCount},+ Url,+ },};pub use self::signal::*;@@ -136,6 +146,24 @@ impl Ranker {}}+pub fn online_centrality_scorer(+ site_rankings: &SiteRankings,+ approx_harmonic: &ApproximatedHarmonicCentrality,+) -> Scorer {+ let mut liked_nodes = Vec::new();+ let mut disliked_nodes = Vec::new();++ for site in &site_rankings.liked {+ liked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());+ }++ for site in &site_rankings.disliked {+ disliked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());+ }++ approx_harmonic.scorer(&liked_nodes, &disliked_nodes)+}+#[cfg(test)]mod tests {use std::collections::HashMap;@@ -767,8 +795,12 @@ mod tests {original: "example".to_string(),optic_program: Some(r#"- Ranking{Field("title"), 20000000};- Ranking{Signal("host_centrality"), 0};+ RankingPipeline {+ Stage {+ Ranking{Field("title"), 20000000},+ Ranking{Signal("host_centrality"), 0}+ }+ }"#.to_string(),),@@ -786,7 +818,11 @@ mod tests {original: "example".to_string(),optic_program: Some(r#"- Ranking{Signal("host_centrality"), 2000000};+ RankingPipeline {+ Stage {+ Ranking{Signal("host_centrality"), 2000000}+ }+ }"#.to_string(),),diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 163b8c0e..f677abec 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -14,64 +14,39 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use optics::{Optic, SiteRankings};+use optics::ast::RankingStage;-use crate::{- schema::Field,- webgraph::{- centrality::approximate_harmonic::{ApproximatedHarmonicCentrality, Scorer},- Node,- },- webpage::Url,-};+use crate::schema::Field;use super::{Signal, SignalAggregator};pub const SCALE: f32 = 10.0;pub trait CreateAggregator {- fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator;+ fn aggregator(&self) -> SignalAggregator;}-fn centrality_scorer(- site_rankings: &SiteRankings,- approx_harmonic: &ApproximatedHarmonicCentrality,-) -> Scorer {- let mut liked_nodes = Vec::new();- let mut disliked_nodes = Vec::new();-- for site in &site_rankings.liked {- liked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());- }-- for site in &site_rankings.disliked {- disliked_nodes.push(Node::from_url(&Url::from(site.clone())).into_host());- }-- approx_harmonic.scorer(&liked_nodes, &disliked_nodes)-}--impl CreateAggregator for Optic {- fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator {- let mut aggregator = SignalAggregator::new(+impl CreateAggregator for RankingStage {+ fn aggregator(&self) -> SignalAggregator {+ let aggregator = SignalAggregator::new(self.coefficients- .clone()- .into_iter()- .filter_map(|(name, coeff)| {- Signal::from_string(name).map(|signal| (signal, coeff))+ .iter()+ .filter_map(|coeff| match &coeff.target {+ optics::ast::RankingTarget::Signal(s) => {+ Signal::from_string(s.clone()).map(|s| (s, coeff.score))+ }+ optics::ast::RankingTarget::Field(_) => None,+ }),+ self.coefficients+ .iter()+ .filter_map(|coeff| match &coeff.target {+ optics::ast::RankingTarget::Signal(_) => None,+ optics::ast::RankingTarget::Field(f) => Field::from_name(f.clone())+ .and_then(|f| f.as_text())+ .map(|f| (f, coeff.score)),}),- self.boosts.clone().into_iter().filter_map(|(name, boost)| {- match Field::from_name(name) {- Some(field) => field.as_text().map(|text_field| (text_field, boost)),- _ => None,- }- }),);- if let Some(approx) = approx {- aggregator.add_personal_harmonic(centrality_scorer(&self.site_rankings, approx));- }-aggregator}}diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex d732f1db..d001b5dc 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -107,7 +107,19 @@ trait Scorer<T: AsRankingWebsite>: Send + Sync {fn score(&self, websites: &mut [T]);}-struct ReRanker {}+struct ReRanker {+ crossencoder: f64,+ prev_score: f64,+}++impl Default for ReRanker {+ fn default() -> Self {+ Self {+ crossencoder: 1.0,+ prev_score: 1.0,+ }+ }+}impl<T: AsRankingWebsite> Scorer<T> for ReRanker {fn score(&self, websites: &mut [T]) {@@ -116,13 +128,26 @@ impl<T: AsRankingWebsite> Scorer<T> for ReRanker {}}-struct PrioritizeText {}+struct PrioritizeText {+ bm25: f64,+ prev_score: f64,+}++impl Default for PrioritizeText {+ fn default() -> Self {+ Self {+ bm25: 1.0,+ prev_score: 1.0,+ }+ }+}impl<T: AsRankingWebsite> Scorer<T> for PrioritizeText {fn score(&self, websites: &mut [T]) {for website in websites {let bm25 = website.as_ranking().pointer.score.bm25 as f64;- website.as_mut_ranking().score += bm25;+ let prev_score = website.as_ranking().score;+ website.as_mut_ranking().score = self.bm25 * bm25 + self.prev_score * prev_score;}}}@@ -208,9 +233,9 @@ pub struct RankingPipeline<T: AsRankingWebsite> {impl<T: AsRankingWebsite> RankingPipeline<T> {fn create() -> Self {let last_stage = RankingStage {- scorer: Box::new(ReRanker {}),+ scorer: Box::<ReRanker>::default(),prev: Prev::Node(Box::new(RankingStage {- scorer: Box::new(PrioritizeText {}),+ scorer: Box::<PrioritizeText>::default(),prev: Prev::Initial,memory: None,top_n: 100,@@ -227,6 +252,8 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}pub fn for_query(query: &mut SearchQuery) -> Self {+ dbg!(&query.optic_program);+let mut pipeline = Self::create();pipeline.offset = query.offset;@@ -406,4 +433,70 @@ mod tests {prev = res;}}++ #[test]+ fn multistage_coefficients() {+ let pipeline = RankingPipeline::for_query(&mut SearchQuery {+ optic_program: Some(+ r#"+ RankingPipeline {+ Stage {},+ Stage {+ Ranking {+ Signal("bm25", 3),+ Signal("prev_score", 2),+ },+ },+ Stage {+ Ranking {+ Signal("crossencoder", 4),+ Signal("prev_score", 3),+ },+ },+ }+ "#+ .to_string(),+ ),+ ..Default::default()+ });++ let w = RankingWebsite {+ pointer: WebsitePointer {+ score: Score {+ bm25: 1.0,+ total: 1.0,+ },+ hashes: Hashes {+ site: Prehashed(0),+ title: Prehashed(0),+ url: Prehashed(0),+ simhash: 0,+ },+ address: DocAddress {+ segment: 0,+ doc_id: 0,+ },+ },+ host_centrality: 0.0,+ page_centrality: 0.0,+ topic_centrality: 0.0,+ personal_centrality: 0.0,+ query_centrality: 0.0,+ title: String::new(),+ clean_body: String::new(),+ score: 1.0,+ };++ let mut test = [w.clone()];++ pipeline.last_stage.scorer.score(&mut test);+ assert_eq!(test[0].score, 5.0);++ let mut test = [w];++ if let Prev::Node(prev) = pipeline.last_stage.prev {+ prev.scorer.score(&mut test);+ assert_eq!(test[0].score, 3.0);+ }+ }}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 82d55af1..94a88888 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -29,7 +29,7 @@ use crate::query::Query;use crate::ranking::centrality_store::CentralityStore;use crate::ranking::optics::CreateAggregator;use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};-use crate::ranking::{Ranker, SignalAggregator};+use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};use crate::spell::Correction;use crate::webgraph::centrality::topic::TopicCentrality;use crate::webpage::region::Region;@@ -84,17 +84,27 @@ impl LocalSearcher {}fn parse_query(&self, query: &SearchQuery, optic: Option<&Optic>) -> Result<Query> {- let query_aggregator = optic- .as_ref()- .map(|optic| {- optic.aggregator(- self.centrality_store- .as_ref()- .map(|centrality_store| ¢rality_store.approx_harmonic),- )+ let mut query_aggregator = optic+ .and_then(|optic| {+ optic+ .pipeline+ .as_ref()+ .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))}).unwrap_or_default();+ if let (Some(optic), Some(approx_harmonic)) = (+ optic,+ self.centrality_store+ .as_ref()+ .map(|store| &store.approx_harmonic),+ ) {+ query_aggregator.add_personal_harmonic(online_centrality_scorer(+ &optic.site_rankings,+ approx_harmonic,+ ));+ }+let mut parsed_query = Query::parse(&query.original,self.index.schema(),@@ -133,19 +143,33 @@ impl LocalSearcher {optics.push(site_rankings.clone().into_optic())}- let optic = optics- .into_iter()- .fold(Optic::default(), |acc, elem| acc.merge(elem));+ let mut optic = Optic::default();++ for o in optics {+ optic = optic.try_merge(o)?;+ }parsed_query.set_optic(&optic, &self.index);+ let mut aggregator = optic+ .pipeline+ .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))+ .unwrap_or_default();++ if let Some(approx_harmonic) = self+ .centrality_store+ .as_ref()+ .map(|store| &store.approx_harmonic)+ {+ aggregator.add_personal_harmonic(online_centrality_scorer(+ &optic.site_rankings,+ approx_harmonic,+ ));+ }+let mut ranker = Ranker::new(self.index.region_count.clone(),- optic.aggregator(- self.centrality_store- .as_ref()- .map(|centrality_store| ¢rality_store.approx_harmonic),- ),+ aggregator,self.index.inverted_index.fastfield_cache(),);diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 391a6ee1..a127eedb 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -266,7 +266,6 @@ impl<S: Store> Webgraph<S> {distances}- #[allow(unused)]pub fn distances(&self, source: Node) -> HashMap<Node, usize> {self.full.as_ref()diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex e5e700be..d428cbab 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -1841,4 +1841,49 @@ mod tests {assert_eq!(webpage.dmoz_description(), None)}++ #[test]+ fn links() {+ let raw = format!(+ r#"+ <html>+ <head>+ <title>Best website</title>+ <meta name="meta1" content="value">+ <link href="link.com" />+ <script src="test.com"></script>+ </head>+ <body>+ <a href="example.com">Link to example</a>+ <p>{CONTENT}</p>+ </body>+ </html>+ "#+ );++ let webpage = Html::parse(&raw, "https://www.example.com/whatever");++ assert_eq!(webpage.title(), Some("Best website".to_string()));++ assert_eq!(+ webpage.links(),+ vec![+ Link {+ source: "https://www.example.com/whatever".to_string().into(),+ destination: "example.com".to_string().into(),+ text: "Link to example".to_string()+ },+ Link {+ source: "https://www.example.com/whatever".to_string().into(),+ destination: "test.com".to_string().into(),+ text: String::new()+ },+ Link {+ source: "https://www.example.com/whatever".to_string().into(),+ destination: "link.com".to_string().into(),+ text: String::new()+ },+ ]+ );+ }}diff --git a/optics-lsp/Cargo.lock b/optics-lsp/Cargo.lockindex f7a6605f..ba91e960 100644--- a/optics-lsp/Cargo.lock+++ b/optics-lsp/Cargo.lock@@ -340,6 +340,7 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"name = "optics"version = "0.1.0"dependencies = [+ "itertools","lalrpop","lalrpop-util","logos",diff --git a/optics-lsp/optic.tmLanguage.json b/optics-lsp/optic.tmLanguage.jsonindex c8446442..257f04df 100644--- a/optics-lsp/optic.tmLanguage.json+++ b/optics-lsp/optic.tmLanguage.json@@ -26,11 +26,11 @@]},"keywords": {- "match": "\\b(Matches|Signal|Field|Site|Url|Domain|Title|Description|Content|Schema|Action|Boost|Downrank|Discard)\\b",+ "match": "\\b(Matches|Signal|Field|Site|Url|Domain|Title|Description|Content|Schema|Action|Boost|Downrank|Discard|Ranking|Stage)\\b","name": "entity.name.function"},"control": {- "match": "\\b(DiscardNonMatching|Rule|Ranking|Like|Dislike)\\b",+ "match": "\\b(DiscardNonMatching|Rule|Like|Dislike|RankingPipeline)\\b","name": "keyword.control"},"punctuation": {diff --git a/optics-lsp/package.json b/optics-lsp/package.jsonindex b973b619..3dcd049d 100644--- a/optics-lsp/package.json+++ b/optics-lsp/package.json@@ -1,6 +1,6 @@{"name": "Optics",- "version": "0.0.4",+ "version": "0.0.6","engines": {"vscode": "^1.73.0"},diff --git a/optics-lsp/src/lib.rs b/optics-lsp/src/lib.rsindex 026b0bd3..bd81a0a6 100644--- a/optics-lsp/src/lib.rs+++ b/optics-lsp/src/lib.rs@@ -154,6 +154,13 @@ impl OpticsBackend {It consists of a `Matches` block and an optional `Action`. Any search result that matches the `Matches` block \will have the `Action` applied to it. The action can either `Boost`, `Downrank` or `Discard` a result. An empty `Action` is \equivalent to a `Boost` of 1.".to_string()),++ optics::Token::RankingPipeline => Some("The final ranking consists of multiple stages in a pipeline. Each stage receives the+ best scoring webpages from the stage before it, and uses more accurate yet computationally expensive algorithms to rank+ the pages for the next stage.".to_string()),++ optics::Token::Stage => Some("The final ranking consists of multiple stages in a pipeline. Each stage has different fields and+ signals available to it.".to_string()),optics::Token::Matches => Some("`Matches` dictates the set of criteria a search result should match in order to have the action applied to it. \A search result must match all the parts of the `Matches` block in order to match the specific rule.".to_string()),@@ -384,6 +391,7 @@ fn err_to_diagnostic(err: optics::Error, source: &str) -> Diagnostic {..Default::default()}},+ optics::Error::RankingStagesMismatch => unreachable!("this error cannot occur at compile time"),}}diff --git a/optics/Cargo.toml b/optics/Cargo.tomlindex be9b76ca..8c5269ac 100644--- a/optics/Cargo.toml+++ b/optics/Cargo.toml@@ -7,6 +7,7 @@ edition = "2021"thiserror = { workspace = true }lalrpop-util = { workspace = true }once_cell = { workspace = true }+itertools = { workspace = true }logos = { workspace = true }serde = { workspace = true }diff --git a/optics/src/ast.rs b/optics/src/ast.rsindex a875cc68..756bc9e1 100644--- a/optics/src/ast.rs+++ b/optics/src/ast.rs@@ -16,6 +16,7 @@use super::Error;use super::Result as ModResult;+use itertools::Itertools;use lalrpop_util::lalrpop_mod;use super::lexer;@@ -25,13 +26,13 @@ lalrpop_mod!(pub parser, "/parser.rs");pub static PARSER: once_cell::sync::Lazy<parser::BlocksParser> =once_cell::sync::Lazy::new(parser::BlocksParser::new);-#[derive(Debug, PartialEq)]+#[derive(Debug, PartialEq, Clone)]pub enum RankingTarget {Signal(String),Field(String),}-#[derive(Debug, PartialEq)]+#[derive(Debug, PartialEq, Clone)]pub struct RankingCoeff {pub target: RankingTarget,pub score: f64,@@ -40,11 +41,38 @@ pub struct RankingCoeff {#[derive(Debug, PartialEq)]pub struct RawOptic {pub rules: Vec<RawRule>,- pub rankings: Vec<RankingCoeff>,+ pub rankings: Vec<RankingPipeline>,pub site_preferences: Vec<RawSitePreference>,pub discard_non_matching: bool,}+#[derive(Debug, PartialEq, Clone)]+pub struct RankingPipeline {+ pub stages: Vec<RankingStage>,+}+impl RankingPipeline {+ pub(crate) fn try_merge(&mut self, other_pipeline: RankingPipeline) -> ModResult<()> {+ if self.stages.len() != other_pipeline.stages.len() {+ return Err(Error::RankingStagesMismatch);+ }++ for (t, mut o) in self+ .stages+ .iter_mut()+ .zip_eq(other_pipeline.stages.into_iter())+ {+ t.coefficients.append(&mut o.coefficients)+ }++ Ok(())+ }+}++#[derive(Debug, PartialEq, Clone)]+pub struct RankingStage {+ pub coefficients: Vec<RankingCoeff>,+}+impl From<Vec<RawOpticBlock>> for RawOptic {fn from(blocks: Vec<RawOpticBlock>) -> Self {let mut rules = Vec::new();@@ -74,7 +102,7 @@ impl From<Vec<RawOpticBlock>> for RawOptic {pub enum RawOpticBlock {Rule(RawRule),SitePreference(RawSitePreference),- Ranking(RankingCoeff),+ Ranking(RankingPipeline),DiscardNonMatching,}@@ -148,12 +176,17 @@ mod tests {let optic = parse(r#"// this is a normal comment- Ranking{Signal("host_centrality"), 3};/*this is a block comment*/- Ranking{Signal("bm25"), 100};- Ranking{Field("url"), 2};+ RankingPipeline {+ Stage {+ Ranking{Signal("host_centrality"), 3},+ Ranking{Signal("bm25"), 100},+ Ranking{Field("url"), 2},+ }+ };+Rule {Matches {Url("/this/is/a/*/pattern")@@ -187,20 +220,24 @@ mod tests {action: None,},],- rankings: vec![- RankingCoeff {- target: RankingTarget::Signal("host_centrality".to_string()),- score: 3.0,- },- RankingCoeff {- target: RankingTarget::Signal("bm25".to_string()),- score: 100.0,- },- RankingCoeff {- target: RankingTarget::Field("url".to_string()),- score: 2.0,- },- ],+ rankings: vec![RankingPipeline {+ stages: vec![RankingStage {+ coefficients: vec![+ RankingCoeff {+ target: RankingTarget::Signal("host_centrality".to_string()),+ score: 3.0,+ },+ RankingCoeff {+ target: RankingTarget::Signal("bm25".to_string()),+ score: 100.0,+ },+ RankingCoeff {+ target: RankingTarget::Field("url".to_string()),+ score: 2.0,+ },+ ]+ }]+ }],site_preferences: vec![],discard_non_matching: false,}diff --git a/optics/src/lexer.rs b/optics/src/lexer.rsindex ff95aaa5..eb76060b 100644--- a/optics/src/lexer.rs+++ b/optics/src/lexer.rs@@ -29,7 +29,9 @@ pub enum Token<'a> {DiscardNonMatching,Rule,+ RankingPipeline,Ranking,+ Stage,Signal,Field,Matches,@@ -62,7 +64,9 @@ impl<'a> ToString for Token<'a> {Token::CloseParenthesis => ")".to_string(),Token::DiscardNonMatching => "DiscardNonMatching".to_string(),Token::Rule => "Rule".to_string(),+ Token::RankingPipeline => "RankingPipeline".to_string(),Token::Ranking => "Ranking".to_string(),+ Token::Stage => "Stage".to_string(),Token::Signal => "Signal".to_string(),Token::Field => "Field".to_string(),Token::Matches => "Matches".to_string(),@@ -115,8 +119,12 @@ enum Outer<'a> {DiscardNonMatching,#[token("Rule")]Rule,+ #[token("RankingPipeline")]+ RankingPipeline,#[token("Ranking")]Ranking,+ #[token("Stage")]+ Stage,#[token("Signal")]Signal,#[token("Field")]@@ -295,6 +303,8 @@ impl<'source> Iterator for LexerBridge<'source> {Outer::CloseParenthesis => Some(Ok((s.start, Token::CloseParenthesis, s.end))),Outer::Rule => Some(Ok((s.start, Token::Rule, s.end))),Outer::Ranking => Some(Ok((s.start, Token::Ranking, s.end))),+ Outer::Stage => Some(Ok((s.start, Token::Stage, s.end))),+ Outer::RankingPipeline => Some(Ok((s.start, Token::RankingPipeline, s.end))),Outer::Signal => Some(Ok((s.start, Token::Signal, s.end))),Outer::Field => Some(Ok((s.start, Token::Field, s.end))),Outer::Matches => Some(Ok((s.start, Token::Matches, s.end))),diff --git a/optics/src/lib.rs b/optics/src/lib.rsindex da5b212a..05594128 100644--- a/optics/src/lib.rs+++ b/optics/src/lib.rs@@ -17,17 +17,18 @@pub mod ast;mod lexer;-use std::{collections::HashMap, convert::TryFrom};+use std::convert::TryFrom;+use ast::RankingPipeline;use logos::Logos;use serde::{Deserialize, Serialize};use thiserror::Error;-use self::ast::{RankingTarget, RawAction, RawMatchPart, RawOptic, RawRule};+use self::ast::{RawAction, RawMatchPart, RawOptic, RawRule};pub use lexer::lex;pub use lexer::Token;-type Result<T> = std::result::Result<T, Error>;+pub(crate) type Result<T> = std::result::Result<T, Error>;#[derive(Error, Debug, Clone)]pub enum Error {@@ -57,6 +58,9 @@ pub enum Error {#[error("Unknown parse error")]Unknown(usize, usize),+ #[error("Ranking stages mismatch")]+ RankingStagesMismatch,+#[error("Unsupported pattern")]Pattern,}@@ -77,20 +81,6 @@ impl TryFrom<RawOptic> for Optic {rules.push(Rule::try_from(rule)?);}- let mut boosts = HashMap::new();- let mut coefficients = HashMap::new();-- for ranking in raw.rankings {- match ranking.target {- RankingTarget::Signal(name) => {- coefficients.insert(name, ranking.score);- }- RankingTarget::Field(name) => {- boosts.insert(name, ranking.score);- }- }- }-let mut liked_sites = Vec::new();let mut disliked_sites = Vec::new();@@ -103,8 +93,7 @@ impl TryFrom<RawOptic> for Optic {Ok(Self {rules,- coefficients,- boosts,+ pipeline: raw.rankings.into_iter().next(),discard_non_matching: raw.discard_non_matching,site_rankings: SiteRankings {liked: liked_sites,@@ -233,18 +222,25 @@ pub enum Action {#[derive(Debug, Default, Clone)]pub struct Optic {- pub coefficients: HashMap<String, f64>,- pub boosts: HashMap<String, f64>,+ pub pipeline: Option<RankingPipeline>,pub site_rankings: SiteRankings,pub rules: Vec<Rule>,pub discard_non_matching: bool,}impl Optic {- pub fn merge(mut self, mut other: Self) -> Self {+ pub fn try_merge(mut self, mut other: Self) -> Result<Self> {self.rules.append(&mut other.rules);- self.coefficients.extend(other.coefficients.into_iter());- self.boosts.extend(other.boosts.into_iter());++ self.pipeline = match (self.pipeline, other.pipeline) {+ (None, None) => None,+ (None, Some(pipeline)) => Some(pipeline),+ (Some(pipeline), None) => Some(pipeline),+ (Some(mut self_pipeline), Some(other_pipeline)) => {+ self_pipeline.try_merge(other_pipeline)?;+ Some(self_pipeline)+ }+ };self.discard_non_matching |= other.discard_non_matching;@@ -260,7 +256,7 @@ impl Optic {.blocked.append(&mut other.site_rankings.blocked);- self+ Ok(self)}}diff --git a/optics/src/parser.lalrpop b/optics/src/parser.lalrpopindex f03f7bc2..74a992f2 100644--- a/optics/src/parser.lalrpop+++ b/optics/src/parser.lalrpop@@ -18,7 +18,7 @@ pub Blocks: Vec<RawOpticBlock> = <Sep<";", Block>>;Block: RawOpticBlock = {<Rule> => RawOpticBlock::Rule(<>),- <Ranking> => RawOpticBlock::Ranking(<>),+ <RankingPipeline> => RawOpticBlock::Ranking(<>),<SitePreference> => RawOpticBlock::SitePreference(<>),"DiscardNonMatching" => RawOpticBlock::DiscardNonMatching,}@@ -68,6 +68,18 @@ InnerAction: RawAction= {"Discard" => RawAction::Discard,}+RankingPipeline: RankingPipeline = {+ "RankingPipeline" "{" <Sep<",", RankingStage>> "}" => RankingPipeline {+ stages: <>+ }+}++RankingStage: RankingStage = {+ "Stage" "{" <Sep<",", Ranking>> "}" => RankingStage {+ coefficients: <>+ }+}+Ranking: RankingCoeff = {"Ranking" "{" <target:RankingTarget> "," <l:@L> <score:Number> <r:@R> "}" =>? {@@ -107,7 +119,9 @@ extern {"DiscardNonMatching" => Token::DiscardNonMatching,"Rule" => Token::Rule,+ "RankingPipeline" => Token::RankingPipeline,"Ranking" => Token::Ranking,+ "Stage" => Token::Stage,"Signal" => Token::Signal,"Field" => Token::Field,"Matches" => Token::Matches,