web search done right
got proximity scoring back
Mikkel Denker0401bd4cfa2a
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 6cffce1d..6f9265a1 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -471,7 +471,7 @@ mod tests {"https://www.old.com",),backlinks: vec![],- host_centrality: 0.092,+ host_centrality: 0.0091,fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,@@ -501,7 +501,7 @@ mod tests {"https://www.new.com",),backlinks: vec![],- host_centrality: 0.09,+ host_centrality: 0.009,fetch_time_ms: 500,pre_computed_score: 0.0,crawl_stability: 0.0,@@ -584,7 +584,7 @@ mod tests {"#,"https://www.second.com"),backlinks: vec![],- host_centrality: 0.003,+ host_centrality: 0.00003,fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 117b9d5e..c8438a72 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -32,7 +32,7 @@ use std::str::FromStr;use std::sync::Arc;use tantivy::fieldnorm::FieldNormReader;use tantivy::postings::SegmentPostings;-use tantivy::query::Scorer;+use tantivy::query::{PhraseQuery, Query as _, Scorer};use tantivy::tokenizer::Tokenizer;use thiserror::Error;@@ -75,6 +75,11 @@ pub enum Signal {Bm25TitleIfHomepage,Bm25BacklinkText,Bm25Description,+ ProximitySlop0,+ ProximitySlop1,+ ProximitySlop2,+ ProximitySlop4,+ ProximitySlop8,CrossEncoder,HostCentrality,PageCentrality,@@ -97,7 +102,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 34] = [+pub const ALL_SIGNALS: [Signal; 39] = [Signal::Bm25,Signal::Bm25Title,Signal::Bm25TitleBigrams,@@ -118,6 +123,11 @@ pub const ALL_SIGNALS: [Signal; 34] = [Signal::Bm25TitleIfHomepage,Signal::Bm25BacklinkText,Signal::Bm25Description,+ Signal::ProximitySlop0,+ Signal::ProximitySlop1,+ Signal::ProximitySlop2,+ Signal::ProximitySlop4,+ Signal::ProximitySlop8,Signal::CrossEncoder,Signal::HostCentrality,Signal::PageCentrality,@@ -196,8 +206,13 @@ impl Signal {Signal::Bm25TitleIfHomepage => 3.0,Signal::Bm25BacklinkText => 4.0,Signal::Bm25Description => 1.0,+ Signal::ProximitySlop0 => 32.0,+ Signal::ProximitySlop1 => 16.0,+ Signal::ProximitySlop2 => 8.0,+ Signal::ProximitySlop4 => 4.0,+ Signal::ProximitySlop8 => 2.0,Signal::CrossEncoder => 100.0,- Signal::HostCentrality => 10_000.0,+ Signal::HostCentrality => 6_000.0,Signal::PageCentrality => 4_500.0,Signal::TopicCentrality => 2_500.0,Signal::QueryCentrality => 1_000.0,@@ -347,6 +362,28 @@ impl Signal {.map(|field| bm25(field, doc)).unwrap_or(0.0)}),++ Signal::ProximitySlop0+ | Signal::ProximitySlop1+ | Signal::ProximitySlop2+ | Signal::ProximitySlop4+ | Signal::ProximitySlop8 => {+ signal_aggregator+ .segment_reader+ .as_mut()+ .and_then(|reader| {+ reader.proximity_scorers.get_mut(self).map(|scorer| {+ let docset = &mut scorer.docset;+ if docset.doc() == doc+ || (docset.doc() < doc && docset.seek(doc) == doc)+ {+ docset.score() as f64+ } else {+ 0.0+ }+ })+ })+ }Signal::CrossEncoder => None, // this is calculated in a later stepSignal::LambdaMART => None,};@@ -417,6 +454,11 @@ impl Signal {| Signal::Bm25TitleIfHomepage| Signal::Bm25BacklinkText| Signal::Bm25Description+ | Signal::ProximitySlop0+ | Signal::ProximitySlop1+ | Signal::ProximitySlop2+ | Signal::ProximitySlop4+ | Signal::ProximitySlop8| Signal::CrossEncoder| Signal::PersonalCentrality| Signal::TopicCentrality@@ -435,6 +477,17 @@ impl Signal {})}+ fn proximity_slop(&self) -> Option<u32> {+ match self {+ Signal::ProximitySlop0 => Some(0),+ Signal::ProximitySlop1 => Some(1),+ Signal::ProximitySlop2 => Some(2),+ Signal::ProximitySlop4 => Some(4),+ Signal::ProximitySlop8 => Some(8),+ _ => None,+ }+ }+fn as_fastfield(&self) -> Option<FastField> {match self {Signal::HostCentrality => Some(FastField::HostCentrality),@@ -585,8 +638,13 @@ struct OpticBoosts {rules: Vec<RuleBoost>,}+struct ProximityScorer {+ docset: Box<dyn Scorer>,+}+struct SegmentReader {text_fields: EnumMap<TextField, TextFieldData>,+ proximity_scorers: EnumMap<Signal, ProximityScorer>,optic_boosts: OpticBoosts,fastfield_reader: Arc<fastfield_reader::SegmentReader>,}@@ -664,16 +722,13 @@ impl SignalAggregator {}}- pub fn register_segment(- &mut self,+ fn prepare_textfields(+ &self,tv_searcher: &tantivy::Searcher,segment_reader: &tantivy::SegmentReader,- fastfield_reader: &fastfield_reader::FastFieldReader,- ) -> Result<()> {- let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());+ ) -> Result<EnumMap<TextField, TextFieldData>> {let mut text_fields = EnumMap::new();let schema = tv_searcher.schema();- let mut optic_rule_boosts = Vec::new();if let Some(query) = &self.query {if !query.simple_terms().is_empty() {@@ -723,7 +778,19 @@ impl SignalAggregator {}}}+ }++ Ok(text_fields)+ }+ fn prepare_optic(+ &self,+ tv_searcher: &tantivy::Searcher,+ segment_reader: &tantivy::SegmentReader,+ fastfield_reader: &fastfield_reader::FastFieldReader,+ ) -> Vec<RuleBoost> {+ let mut optic_rule_boosts = Vec::new();+ if let Some(query) = &self.query {if let Some(optic) = query.optic() {optic_rule_boosts = optic.rules@@ -744,12 +811,72 @@ impl SignalAggregator {}}+ optic_rule_boosts+ }++ fn prepare_proximity_scorers(+ &self,+ tv_searcher: &tantivy::Searcher,+ segment_reader: &tantivy::SegmentReader,+ ) -> EnumMap<Signal, ProximityScorer> {+ let mut proximity_scorers = EnumMap::new();+ let proximity_fields = [TextField::Title, TextField::CleanBody];+ let schema = tv_searcher.schema();++ if let Some(query) = &self.query {+ let simple_terms = query.simple_terms().to_vec();++ if simple_terms.len() < 2 {+ return proximity_scorers;+ }++ for signal in ALL_SIGNALS {+ if let Some(slop) = signal.proximity_slop() {+ for field in proximity_fields {+ let tv_field = schema.get_field(field.name()).unwrap();+ let mut terms = Vec::with_capacity(simple_terms.len());++ for term in &simple_terms {+ let term = tantivy::Term::from_field_text(tv_field, term);+ terms.push(term);+ }++ let mut phrase_query = PhraseQuery::new(terms);+ phrase_query.set_slop(slop);++ let docset = phrase_query+ .weight(tantivy::query::EnableScoring::Enabled(tv_searcher))+ .unwrap()+ .scorer(segment_reader, 1.0)+ .unwrap();++ proximity_scorers.insert(signal, ProximityScorer { docset });+ }+ }+ }+ }++ proximity_scorers+ }++ pub fn register_segment(+ &mut self,+ tv_searcher: &tantivy::Searcher,+ segment_reader: &tantivy::SegmentReader,+ fastfield_reader: &fastfield_reader::FastFieldReader,+ ) -> Result<()> {+ let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());+ let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;+ let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, fastfield_reader);+ let proximity_scorers = self.prepare_proximity_scorers(tv_searcher, segment_reader);+self.segment_reader = Some(SegmentReader {text_fields,fastfield_reader: fastfield_segment_reader,optic_boosts: OpticBoosts {rules: optic_rule_boosts,},+ proximity_scorers,});Ok(())diff --git a/ltr/generate_data.py b/ltr/generate_data.pyindex 11ca6ae3..89f42af2 100644--- a/ltr/generate_data.py+++ b/ltr/generate_data.py@@ -17,10 +17,8 @@ openai.api_key = os.getenv("OPENAI_API_KEY")def search(q, top_n, optic=None):url = f"http://{STRACT_ENDPOINT}/beta/api/search"- print(q)r = requests.post(url, json={"query": q, "page": 0, "num_results": top_n, "return_ranking_signals": True, "optic_program": optic})- print(r)search_result = r.json()['webpages']res = []