web search done right
split signal.rs into more manageable parts
Mikkel Denkerbdf93a63166f
diff --git a/crates/core/src/ranking/signal.rs b/crates/core/src/ranking/signal.rsdeleted file mode 100644index 16e2b372..00000000--- a/crates/core/src/ranking/signal.rs+++ /dev/null@@ -1,1946 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use crate::enum_map::InsertEnumMapKey;-use crate::fastfield_reader::FieldReader;-use crate::query::optic::AsSearchableRule;-use crate::query::Query;-use crate::schema::text_field::TextField;-use crate::schema::{self, Field};-use crate::{enum_dispatch_from_discriminant, Result};-use crate::{- enum_map::EnumMap,- fastfield_reader,- schema::{FastFieldEnum, TextFieldEnum},- webgraph::NodeID,- webpage::Webpage,-};-use enum_dispatch::enum_dispatch;-use itertools::Itertools;-use optics::ast::RankingTarget;-use optics::Optic;-use serde::{Deserialize, Serialize};-use std::cell::RefCell;-use std::str::FromStr;-use std::sync::Arc;-use strum::{EnumDiscriminants, VariantArray};-use tantivy::fieldnorm::FieldNormReader;-use tantivy::postings::SegmentPostings;-use tantivy::query::{Query as _, Scorer};-use tantivy::tokenizer::Tokenizer as _;-use thiserror::Error;-use utoipa::ToSchema;--use tantivy::DocSet;-use tantivy::{DocId, Postings};--use crate::{schema::FLOAT_SCALING, webpage::region::RegionCount};--use super::bm25::MultiBm25Weight;-use super::models::linear::LinearRegression;-use super::{inbound_similarity, query_centrality};--#[derive(Debug, Error)]-pub enum Error {- #[error("unknown signal: {0}")]- UnknownSignal(#[from] serde_json::Error),-}--#[enum_dispatch]-pub trait Signal:- Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash + Into<SignalEnum>-{- fn default_coefficient(&self) -> f64;- fn as_field(&self) -> Option<Field>;- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64>;-- fn precompute(self, _webpage: &Webpage, _signal_computer: &SignalComputer) -> Option<f64> {- None- }-- fn as_textfield(&self) -> Option<TextFieldEnum> {- self.as_field().and_then(|field| field.as_text())- }-- fn as_fastfield(&self) -> Option<FastFieldEnum> {- self.as_field().and_then(|field| field.as_fast())- }-}--#[enum_dispatch(Signal)]-#[derive(- Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants,-)]-#[strum_discriminants(derive(VariantArray, serde::Serialize, serde::Deserialize, Hash))]-#[strum_discriminants(serde(rename_all = "snake_case"))]-pub enum SignalEnum {- Bm25Title,- Bm25TitleBigrams,- Bm25TitleTrigrams,- Bm25CleanBody,- Bm25CleanBodyBigrams,- Bm25CleanBodyTrigrams,- Bm25StemmedTitle,- Bm25StemmedCleanBody,- Bm25AllBody,- Bm25Keywords,- Bm25BacklinkText,- IdfSumUrl,- IdfSumSite,- IdfSumDomain,- IdfSumSiteNoTokenizer,- IdfSumDomainNoTokenizer,- IdfSumDomainNameNoTokenizer,- IdfSumDomainIfHomepage,- IdfSumDomainNameIfHomepageNoTokenizer,- IdfSumDomainIfHomepageNoTokenizer,- IdfSumTitleIfHomepage,- CrossEncoderSnippet,- CrossEncoderTitle,- HostCentrality,- HostCentralityRank,- PageCentrality,- PageCentralityRank,- IsHomepage,- FetchTimeMs,- UpdateTimestamp,- TrackerScore,- Region,- QueryCentrality,- InboundSimilarity,- LambdaMart,- UrlDigits,- UrlSlashes,- LinkDensity,- TitleEmbeddingSimilarity,- KeywordEmbeddingSimilarity,-}--enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,-[- Bm25Title,- Bm25TitleBigrams,- Bm25TitleTrigrams,- Bm25CleanBody,- Bm25CleanBodyBigrams,- Bm25CleanBodyTrigrams,- Bm25StemmedTitle,- Bm25StemmedCleanBody,- Bm25AllBody,- Bm25Keywords,- Bm25BacklinkText,- IdfSumUrl,- IdfSumSite,- IdfSumDomain,- IdfSumSiteNoTokenizer,- IdfSumDomainNoTokenizer,- IdfSumDomainNameNoTokenizer,- IdfSumDomainIfHomepage,- IdfSumDomainNameIfHomepageNoTokenizer,- IdfSumDomainIfHomepageNoTokenizer,- IdfSumTitleIfHomepage,- CrossEncoderSnippet,- CrossEncoderTitle,- HostCentrality,- HostCentralityRank,- PageCentrality,- PageCentralityRank,- IsHomepage,- FetchTimeMs,- UpdateTimestamp,- TrackerScore,- Region,- QueryCentrality,- InboundSimilarity,- LambdaMart,- UrlDigits,- UrlSlashes,- LinkDensity,- TitleEmbeddingSimilarity,- KeywordEmbeddingSimilarity,-]);--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25Title;-impl Signal for Bm25Title {- fn default_coefficient(&self) -> f64 {- 0.0063- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::Title.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25TitleBigrams;-impl Signal for Bm25TitleBigrams {- fn default_coefficient(&self) -> f64 {- 0.01- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::TitleBigrams.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25TitleTrigrams;-impl Signal for Bm25TitleTrigrams {- fn default_coefficient(&self) -> f64 {- 0.01- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::TitleTrigrams.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25CleanBody;-impl Signal for Bm25CleanBody {- fn default_coefficient(&self) -> f64 {- 0.0063- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::CleanBody.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25CleanBodyBigrams;-impl Signal for Bm25CleanBodyBigrams {- fn default_coefficient(&self) -> f64 {- 0.005- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::CleanBodyBigrams.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25CleanBodyTrigrams;-impl Signal for Bm25CleanBodyTrigrams {- fn default_coefficient(&self) -> f64 {- 0.005- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::CleanBodyTrigrams.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25StemmedTitle;-impl Signal for Bm25StemmedTitle {- fn default_coefficient(&self) -> f64 {- 0.003- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::StemmedTitle.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25StemmedCleanBody;-impl Signal for Bm25StemmedCleanBody {- fn default_coefficient(&self) -> f64 {- 0.001- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::StemmedCleanBody.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25AllBody;-impl Signal for Bm25AllBody {- fn default_coefficient(&self) -> f64 {- 0.0- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::AllBody.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25Keywords;-impl Signal for Bm25Keywords {- fn default_coefficient(&self) -> f64 {- 0.001- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::Keywords.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Bm25BacklinkText;-impl Signal for Bm25BacklinkText {- fn default_coefficient(&self) -> f64 {- 0.003- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::BacklinkText.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| bm25(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumUrl;-impl Signal for IdfSumUrl {- fn default_coefficient(&self) -> f64 {- 0.0003- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::Url.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumSite;-impl Signal for IdfSumSite {- fn default_coefficient(&self) -> f64 {- 0.00015- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::SiteWithout.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumDomain;-impl Signal for IdfSumDomain {- fn default_coefficient(&self) -> f64 {- 0.0003- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::Domain.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumSiteNoTokenizer;-impl Signal for IdfSumSiteNoTokenizer {- fn default_coefficient(&self) -> f64 {- 0.00015- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::SiteNoTokenizer.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumDomainNoTokenizer;-impl Signal for IdfSumDomainNoTokenizer {- fn default_coefficient(&self) -> f64 {- 0.0002- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::DomainNoTokenizer.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumDomainNameNoTokenizer;-impl Signal for IdfSumDomainNameNoTokenizer {- fn default_coefficient(&self) -> f64 {- 0.0002- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(- schema::text_field::DomainNameNoTokenizer.into(),- ))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumDomainIfHomepage;-impl Signal for IdfSumDomainIfHomepage {- fn default_coefficient(&self) -> f64 {- 0.0004- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::DomainIfHomepage.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumDomainNameIfHomepageNoTokenizer;-impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {- fn default_coefficient(&self) -> f64 {- 0.0036- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(- schema::text_field::DomainNameIfHomepageNoTokenizer.into(),- ))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumDomainIfHomepageNoTokenizer;-impl Signal for IdfSumDomainIfHomepageNoTokenizer {- fn default_coefficient(&self) -> f64 {- 0.0036- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(- schema::text_field::DomainIfHomepageNoTokenizer.into(),- ))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IdfSumTitleIfHomepage;-impl Signal for IdfSumTitleIfHomepage {- fn default_coefficient(&self) -> f64 {- 0.00022- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Text(schema::text_field::TitleIfHomepage.into()))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let mut seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();-- seg_reader- .text_fields- .get_mut(self.as_textfield().unwrap())- .map(|field| idf_sum(field, doc))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct CrossEncoderSnippet;-impl Signal for CrossEncoderSnippet {- fn default_coefficient(&self) -> f64 {- 0.17- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {- None // computed in later ranking stage- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct CrossEncoderTitle;-impl Signal for CrossEncoderTitle {- fn default_coefficient(&self) -> f64 {- 0.17- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {- None // computed in later ranking stage- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct HostCentrality;-impl Signal for HostCentrality {- fn default_coefficient(&self) -> f64 {- 0.5- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::HostCentrality.into()))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(webpage.host_centrality)- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(val as f64 / FLOAT_SCALING as f64)- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct HostCentralityRank;-impl Signal for HostCentralityRank {- fn default_coefficient(&self) -> f64 {- 0.0- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::HostCentralityRank.into()))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(score_rank(webpage.host_centrality_rank as f64))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_rank(val as f64))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct PageCentrality;-impl Signal for PageCentrality {- fn default_coefficient(&self) -> f64 {- 0.25- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::PageCentrality.into()))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(webpage.page_centrality)- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(val as f64 / FLOAT_SCALING as f64)- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct PageCentralityRank;-impl Signal for PageCentralityRank {- fn default_coefficient(&self) -> f64 {- 0.0- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::PageCentralityRank.into()))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(score_rank(webpage.page_centrality_rank as f64))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_rank(val as f64))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct IsHomepage;-impl Signal for IsHomepage {- fn default_coefficient(&self) -> f64 {- 0.0005- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::IsHomepage.into()))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- Some(webpage.html.is_homepage().into())- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(val as f64)- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct FetchTimeMs;-impl Signal for FetchTimeMs {- fn default_coefficient(&self) -> f64 {- 0.001- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::FetchTimeMs.into()))- }-- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {- let fetch_time_ms = webpage.fetch_time_ms as usize;- if fetch_time_ms >= signal_computer.fetch_time_ms_cache.len() {- Some(0.0)- } else {- Some(signal_computer.fetch_time_ms_cache[fetch_time_ms])- }- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let fetch_time_ms = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap() as usize;-- if fetch_time_ms >= signal_computer.fetch_time_ms_cache.len() {- Some(0.0)- } else {- Some(signal_computer.fetch_time_ms_cache[fetch_time_ms])- }- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct UpdateTimestamp;-impl Signal for UpdateTimestamp {- fn default_coefficient(&self) -> f64 {- 0.001- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::LastUpdated.into()))- }-- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {- let update_timestamp = webpage- .html- .updated_time()- .map(|date| date.timestamp().max(0))- .unwrap_or(0) as usize;-- Some(score_timestamp(update_timestamp, signal_computer))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap() as usize;-- Some(score_timestamp(val, signal_computer))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct TrackerScore;-impl Signal for TrackerScore {- fn default_coefficient(&self) -> f64 {- 0.05- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::TrackerScore.into()))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- let num_trackers = webpage.html.trackers().len() as f64;- Some(score_trackers(num_trackers))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_trackers(val as f64))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct Region;-impl Signal for Region {- fn default_coefficient(&self) -> f64 {- 0.15- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::Region.into()))- }-- fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {- let region =- crate::webpage::Region::guess_from(webpage).unwrap_or(crate::webpage::Region::All);- Some(score_region(region, signal_computer))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- let region = crate::webpage::Region::from_id(val);- Some(score_region(region, signal_computer))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct QueryCentrality;-impl Signal for QueryCentrality {- fn default_coefficient(&self) -> f64 {- 0.0- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);- let host_id = host_id(&fastfield_reader);-- host_id.and_then(|host_id| signal_computer.query_centrality(host_id))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct InboundSimilarity;-impl Signal for InboundSimilarity {- fn default_coefficient(&self) -> f64 {- 0.25- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);- let host_id = host_id(&fastfield_reader);-- host_id.map(|host_id| signal_computer.inbound_similarity(host_id))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct LambdaMart;-impl Signal for LambdaMart {- fn default_coefficient(&self) -> f64 {- 10.0- }-- fn as_field(&self) -> Option<Field> {- None- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {- None // computed in later ranking stage- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct UrlDigits;-impl Signal for UrlDigits {- fn default_coefficient(&self) -> f64 {- 0.01- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(- schema::fast_field::NumPathAndQueryDigits.into(),- ))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- let num_digits = (webpage- .html- .url()- .path()- .chars()- .filter(|c| c.is_ascii_digit())- .count()- + webpage- .html- .url()- .query()- .unwrap_or_default()- .chars()- .filter(|c| c.is_ascii_digit())- .count()) as f64;-- Some(score_digits(num_digits))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_digits(val as f64))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct UrlSlashes;-impl Signal for UrlSlashes {- fn default_coefficient(&self) -> f64 {- 0.01- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(- schema::fast_field::NumPathAndQuerySlashes.into(),- ))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- let num_slashes = webpage- .html- .url()- .path()- .chars()- .filter(|c| c == &'/')- .count() as f64;- Some(score_slashes(num_slashes))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_slashes(val as f64))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct LinkDensity;-impl Signal for LinkDensity {- fn default_coefficient(&self) -> f64 {- 0.0- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::LinkDensity.into()))- }-- fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {- let link_density = webpage.html.link_density();- Some(score_link_density(link_density))- }-- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer- .segment_reader- .as_ref()- .unwrap()- .borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);-- let val = fastfield_reader- .get(self.as_fastfield().unwrap())- .and_then(|v| v.as_u64())- .unwrap();- Some(score_link_density(val as f64 / FLOAT_SCALING as f64))- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct TitleEmbeddingSimilarity;-impl Signal for TitleEmbeddingSimilarity {- fn default_coefficient(&self) -> f64 {- 0.01- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::TitleEmbeddings.into()))- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {- None // computed in later ranking stage- }-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]-pub struct KeywordEmbeddingSimilarity;-impl Signal for KeywordEmbeddingSimilarity {- fn default_coefficient(&self) -> f64 {- 0.01- }-- fn as_field(&self) -> Option<Field> {- Some(Field::Fast(schema::fast_field::KeywordEmbeddings.into()))- }-- fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {- None // computed in later ranking stage- }-}--impl SignalEnum {- pub fn num_variants() -> usize {- SignalEnumDiscriminants::VARIANTS.len()- }-- pub fn all() -> impl Iterator<Item = SignalEnum> {- SignalEnumDiscriminants::VARIANTS- .iter()- .copied()- .map(|v| v.into())- }-- pub fn get(field_id: usize) -> Option<SignalEnum> {- SignalEnumDiscriminants::VARIANTS- .get(field_id)- .copied()- .map(SignalEnum::from)- }-}--impl InsertEnumMapKey for SignalEnum {- fn into_usize(self) -> usize {- SignalEnumDiscriminants::from(self) as usize- }-}--fn score_timestamp(timestamp: usize, signal_computer: &SignalComputer) -> f64 {- if timestamp >= signal_computer.current_timestamp.unwrap_or(0) {- return 0.0;- }-- let hours_since_update = (signal_computer.current_timestamp.unwrap() - timestamp).max(1) / 3600;-- if hours_since_update < signal_computer.update_time_cache.len() {- signal_computer.update_time_cache[hours_since_update]- } else {- 0.0- }-}--#[inline]-fn score_rank(rank: f64) -> f64 {- 1.0 / (rank + 1.0)-}--#[inline]-fn score_trackers(num_trackers: f64) -> f64 {- 1.0 / (num_trackers + 1.0)-}--#[inline]-fn score_digits(num_digits: f64) -> f64 {- 1.0 / (num_digits + 1.0)-}--#[inline]-fn score_slashes(num_slashes: f64) -> f64 {- 1.0 / (num_slashes + 1.0)-}--#[inline]-fn score_link_density(link_density: f64) -> f64 {- if link_density > 0.5 {- 0.0- } else {- 1.0 - link_density- }-}--fn score_region(webpage_region: crate::webpage::Region, computer: &SignalComputer) -> f64 {- match computer.region_count.as_ref() {- Some(region_count) => {- let boost = computer- .query_data- .as_ref()- .and_then(|q| q.selected_region)- .map_or(0.0, |region| {- if region != crate::webpage::Region::All && region == webpage_region {- 50.0- } else {- 0.0- }- });-- boost + region_count.score(&webpage_region)- }- None => 0.0,- }-}--fn bm25(field: &mut TextFieldData, doc: DocId) -> f64 {- if field.postings.is_empty() {- return 0.0;- }-- let fieldnorm_id = field.fieldnorm_reader.fieldnorm_id(doc);-- field- .weight- .score(field.postings.iter_mut().map(move |posting| {- if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {- (fieldnorm_id, posting.term_freq())- } else {- (fieldnorm_id, 0)- }- })) as f64-}--fn idf_sum(field: &mut TextFieldData, doc: DocId) -> f64 {- if field.postings.is_empty() {- return 0.0;- }-- field- .postings- .iter_mut()- .zip_eq(field.weight.idf())- .filter_map(|(posting, idf)| {- if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {- Some(idf)- } else {- None- }- })- .sum::<f32>() as f64-}--fn host_id<'a>(fastfield_reader: &FieldReader<'a>) -> Option<NodeID> {- let node_id = fastfield_reader- .get(schema::fast_field::HostNodeID.into())- .and_then(|n| n.as_u64())- .unwrap();-- if node_id == u64::MAX {- None- } else {- Some(node_id.into())- }-}--impl FromStr for SignalEnumDiscriminants {- type Err = Error;-- fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {- let s = "\"".to_string() + name + "\"";- let signal = serde_json::from_str(&s)?;- Ok(signal)- }-}--#[derive(Debug, Clone, Default)]-pub struct SignalCoefficient {- map: EnumMap<SignalEnum, f64>,-}--impl SignalCoefficient {- pub fn get(&self, signal: &SignalEnum) -> f64 {- self.map- .get(*signal)- .copied()- .unwrap_or(signal.default_coefficient())- }-- pub fn new(coefficients: impl Iterator<Item = (SignalEnum, f64)>) -> Self {- let mut map = EnumMap::default();-- for (signal, coefficient) in coefficients {- map.insert(signal, coefficient);- }-- Self { map }- }-- pub fn from_optic(optic: &Optic) -> Self {- SignalCoefficient::new(optic.rankings.iter().filter_map(|coeff| {- match &coeff.target {- RankingTarget::Signal(signal) => SignalEnumDiscriminants::from_str(signal)- .ok()- .map(|signal| (signal.into(), coeff.value)),- }- }))- }-- pub fn merge_into(&mut self, coeffs: SignalCoefficient) {- for signal in SignalEnum::all() {- if let Some(coeff) = coeffs.map.get(signal).copied() {- match self.map.get_mut(signal) {- Some(existing_coeff) => *existing_coeff += coeff,- None => {- self.map.insert(signal, coeff);- }- }- }- }- }-}--#[derive(Clone)]-struct TextFieldData {- postings: Vec<SegmentPostings>,- weight: MultiBm25Weight,- fieldnorm_reader: FieldNormReader,-}--struct RuleBoost {- docset: Box<dyn Scorer>,- boost: f64,-}--struct OpticBoosts {- rules: Vec<RuleBoost>,-}--struct SegmentReader {- text_fields: EnumMap<TextFieldEnum, TextFieldData>,- optic_boosts: OpticBoosts,- fastfield_reader: Arc<fastfield_reader::SegmentReader>,-}--#[derive(Clone)]-struct QueryData {- simple_terms: Vec<String>,- optic_rules: Vec<optics::Rule>,- selected_region: Option<crate::webpage::Region>,-}--pub struct SignalComputer {- query_data: Option<QueryData>,- query_signal_coefficients: Option<SignalCoefficient>,- segment_reader: Option<RefCell<SegmentReader>>,- inbound_similarity: Option<RefCell<inbound_similarity::Scorer>>,- fetch_time_ms_cache: Vec<f64>,- update_time_cache: Vec<f64>,- query_centrality: Option<RefCell<query_centrality::Scorer>>,- region_count: Option<Arc<RegionCount>>,- current_timestamp: Option<usize>,- linear_regression: Option<Arc<LinearRegression>>,- order: SignalOrder,-}--impl Clone for SignalComputer {- fn clone(&self) -> Self {- let inbound_similarity = self- .inbound_similarity- .as_ref()- .map(|scorer| RefCell::new(scorer.borrow().clone()));-- let query_centrality = self- .query_centrality- .as_ref()- .map(|scorer| RefCell::new(scorer.borrow().clone()));-- Self {- query_data: self.query_data.clone(),- query_signal_coefficients: self.query_signal_coefficients.clone(),- segment_reader: None,- inbound_similarity,- fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),- update_time_cache: self.update_time_cache.clone(),- query_centrality,- region_count: self.region_count.clone(),- current_timestamp: self.current_timestamp,- linear_regression: self.linear_regression.clone(),- order: self.order.clone(),- }- }-}--impl std::fmt::Debug for SignalComputer {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.debug_struct("SignalComputer")- .field(- "query",- &self- .query_data- .as_ref()- .map(|q| q.simple_terms.clone())- .unwrap_or_default(),- )- .finish()- }-}--impl SignalComputer {- pub fn new(query: Option<&Query>) -> Self {- let query_signal_coefficients = query.as_ref().and_then(|q| q.signal_coefficients());-- let fetch_time_ms_cache: Vec<_> = (0..1000)- .map(|fetch_time| 1.0 / (fetch_time as f64 + 1.0))- .collect();-- let update_time_cache = (0..(3 * 365 * 24))- .map(|hours_since_update| 1.0 / ((hours_since_update as f64 + 1.0).log2()))- .collect();-- let query = query.as_ref().map(|q| QueryData {- simple_terms: q.simple_terms().to_vec(),- optic_rules: q- .optics()- .iter()- .flat_map(|o| o.rules.iter())- .filter(|rule| match rule.action {- optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,- optics::Action::Discard => false,- })- .cloned()- .collect(),- selected_region: q.region().cloned(),- });-- let mut s = Self {- segment_reader: None,- inbound_similarity: None,- query_signal_coefficients,- fetch_time_ms_cache,- update_time_cache,- query_centrality: None,- region_count: None,- current_timestamp: None,- linear_regression: None,- query_data: query,- order: SignalOrder::empty(),- };-- s.order = SignalOrder::new(&s);- s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);-- s- }-- fn prepare_textfields(- &self,- tv_searcher: &tantivy::Searcher,- segment_reader: &tantivy::SegmentReader,- ) -> Result<EnumMap<TextFieldEnum, TextFieldData>> {- let mut text_fields = EnumMap::new();- let schema = tv_searcher.schema();-- if let Some(query) = &self.query_data {- if !query.simple_terms.is_empty() {- for signal in SignalEnum::all() {- if let Some(text_field) = signal.as_textfield() {- let tv_field = schema.get_field(text_field.name()).unwrap();- let simple_query = itertools::intersperse(- query.simple_terms.iter().map(|s| s.as_str()),- " ",- )- .collect::<String>();-- let mut terms = Vec::new();- let mut tokenizer = text_field.indexing_tokenizer();- let mut stream = tokenizer.token_stream(&simple_query);-- while let Some(token) = stream.next() {- let term = tantivy::Term::from_field_text(tv_field, &token.text);- terms.push(term);- }-- if terms.is_empty() {- continue;- }-- let fieldnorm_reader = segment_reader.get_fieldnorms_reader(tv_field)?;- let inverted_index = segment_reader.inverted_index(tv_field)?;-- let mut matching_terms = Vec::with_capacity(terms.len());- let mut postings = Vec::with_capacity(terms.len());- for term in &terms {- if let Some(p) =- inverted_index.read_postings(term, text_field.record_option())?- {- postings.push(p);- matching_terms.push(term.clone());- }- }- let weight = MultiBm25Weight::for_terms(tv_searcher, &matching_terms)?;-- text_fields.insert(- text_field,- TextFieldData {- postings,- weight,- fieldnorm_reader,- },- );- }- }- }- }-- Ok(text_fields)- }-- fn prepare_optic(- &self,- tv_searcher: &tantivy::Searcher,- segment_reader: &tantivy::SegmentReader,- fastfield_reader: &fastfield_reader::FastFieldReader,- ) -> Vec<RuleBoost> {- let mut optic_rule_boosts = Vec::new();-- if let Some(query) = &self.query_data {- optic_rule_boosts = query- .optic_rules- .iter()- .filter_map(|rule| rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader))- .map(|(_, rule)| RuleBoost {- docset: rule- .query- .weight(tantivy::query::EnableScoring::Enabled {- searcher: tv_searcher,- statistics_provider: tv_searcher,- })- .unwrap()- .scorer(segment_reader, 0.0)- .unwrap(),- boost: rule.boost,- })- .collect();- }-- optic_rule_boosts- }-- pub fn register_segment(- &mut self,- tv_searcher: &tantivy::Searcher,- segment_reader: &tantivy::SegmentReader,- fastfield_reader: &fastfield_reader::FastFieldReader,- ) -> Result<()> {- let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());- let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;- let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, fastfield_reader);-- self.segment_reader = Some(RefCell::new(SegmentReader {- text_fields,- fastfield_reader: fastfield_segment_reader,- optic_boosts: OpticBoosts {- rules: optic_rule_boosts,- },- }));-- Ok(())- }-- pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {- self.query_centrality = Some(RefCell::new(query_centrality));- }-- pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {- let mut scorer = scorer;- scorer.set_default_if_precalculated(true);-- self.inbound_similarity = Some(RefCell::new(scorer));- }-- pub fn set_region_count(&mut self, region_count: RegionCount) {- self.region_count = Some(Arc::new(region_count));- }-- pub fn set_current_timestamp(&mut self, current_timestamp: usize) {- self.current_timestamp = Some(current_timestamp);- }-- pub fn set_linear_model(&mut self, linear_model: Arc<LinearRegression>) {- self.linear_regression = Some(linear_model);- }-- pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {- self.query_centrality- .as_ref()- .map(|scorer| scorer.borrow_mut().score(host_id))- }-- pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {- self.inbound_similarity- .as_ref()- .map(|scorer| scorer.borrow_mut().score(&host_id))- .unwrap_or_default()- }-- /// Computes the scored signals for a given document.- ///- /// Important: This function assues that the docs a scored in ascending order of docid- /// within their segment. If this invariant is not upheld, the documents will not have- /// scores calculated for their text related signals. The wrong ranking will most likely- /// be returned.- /// This function also assumes that the segment reader has been set.- pub fn compute_signals(&self, doc: DocId) -> impl Iterator<Item = Option<ComputedSignal>> + '_ {- self.order.compute(doc, self)- }-- pub fn boosts(&mut self, doc: DocId) -> Option<f64> {- self.segment_reader.as_ref().map(|segment_reader| {- let mut downrank = 0.0;- let mut boost = 0.0;-- for rule in &mut segment_reader.borrow_mut().optic_boosts.rules {- if rule.docset.doc() > doc {- continue;- }-- if rule.docset.doc() == doc || rule.docset.seek(doc) == doc {- if rule.boost < 0.0 {- downrank += rule.boost.abs();- } else {- boost += rule.boost;- }- }- }-- if downrank > boost {- let diff = downrank - boost;- 1.0 / (1.0 + diff)- } else {- boost - downrank + 1.0- }- })- }-- pub fn precompute_score(&self, webpage: &Webpage) -> f64 {- SignalEnum::all()- .filter_map(|signal| {- signal- .precompute(webpage, self)- .map(|value| ComputedSignal {- signal,- score: SignalScore {- coefficient: self.coefficient(&signal),- value,- },- })- })- .map(|computed| computed.score.coefficient * computed.score.value)- .sum()- }-- pub fn coefficient(&self, signal: &SignalEnum) -> f64 {- self.query_signal_coefficients- .as_ref()- .map(|coefficients| coefficients.get(signal))- .or_else(|| {- self.linear_regression- .as_ref()- .and_then(|model| model.weights.get(*signal).copied())- })- .unwrap_or(signal.default_coefficient())- }-}--#[derive(Debug, Clone, Copy)]-pub struct ComputedSignal {- pub signal: SignalEnum,- pub score: SignalScore,-}--#[derive(Debug, Clone, Copy, Serialize, Deserialize, ToSchema)]-#[serde(rename_all = "camelCase")]-pub struct SignalScore {- pub coefficient: f64,- pub value: f64,-}--#[derive(Clone)]-pub struct SignalOrder {- text_signals: EnumMap<TextFieldEnum, NGramSignalOrder>,- other_signals: Vec<SignalEnum>,-}--impl SignalOrder {- pub fn empty() -> Self {- Self {- text_signals: EnumMap::new(),- other_signals: Vec::new(),- }- }-- pub fn new(signal_computer: &SignalComputer) -> Self {- let mut text_signals = EnumMap::new();- let mut other_signals = Vec::new();-- for signal in SignalEnum::all() {- if signal_computer.coefficient(&signal) == 0.0 {- continue;- }-- if let Some(text_field) = signal.as_textfield() {- let mono = text_field.monogram_field();-- if !text_signals.contains_key(mono) {- text_signals.insert(mono, NGramSignalOrder::default());- }-- let ngram = text_field.ngram_size();- text_signals.get_mut(mono).unwrap().push(signal, ngram);- } else {- other_signals.push(signal);- }- }-- Self {- text_signals,- other_signals,- }- }-- fn compute<'a>(- &'a self,- doc: DocId,- signal_computer: &'a SignalComputer,- ) -> impl Iterator<Item = Option<ComputedSignal>> + 'a {- self.text_signals- .values()- .flat_map(move |ngram| ngram.compute(doc, signal_computer))- .map(Some)- .chain(- self.other_signals- .iter()- .filter_map(|signal| {- let coefficient = signal_computer.coefficient(signal);-- if coefficient > 0.0 {- Some((signal, coefficient))- } else {- None- }- })- .map(move |(signal, coefficient)| {- signal- .compute(doc, signal_computer)- .map(|value| ComputedSignal {- signal: *signal,- score: SignalScore { coefficient, value },- })- }),- )- }-}--/// If an ngram of size n matches the query for a given document in a given field,-/// the score of all ngrams where n' < n is dampened by NGRAM_DAMPENING.-///-/// A dampening factor of 0.0 means that we ignore all ngrams where n' < n. A dampening factor of 1.0-/// does not dampen any ngrams.-const NGRAM_DAMPENING: f64 = 0.4;--#[derive(Debug, Default, Clone)]-pub struct NGramSignalOrder {- /// ordered by descending ngram size. e.g. [title_bm25_trigram, title_bm25_bigram, title_bm25]- signals: Vec<(usize, SignalEnum)>,-}--impl NGramSignalOrder {- fn push(&mut self, signal: SignalEnum, ngram: usize) {- self.signals.push((ngram, signal));- self.signals.sort_unstable_by(|(a, _), (b, _)| b.cmp(a));- }-- fn compute<'a>(- &'a self,- doc: DocId,- signal_computer: &'a SignalComputer,- ) -> impl Iterator<Item = ComputedSignal> + 'a {- let mut hits = 0;-- self.signals- .iter()- .map(|(_, s)| s)- .filter_map(move |signal| {- signal- .compute(doc, signal_computer)- .map(|value| {- let coefficient = signal_computer.coefficient(signal);-- ComputedSignal {- signal: *signal,- score: SignalScore { coefficient, value },- }- })- .map(|mut c| {- c.score.coefficient *= NGRAM_DAMPENING.powi(hits);-- if c.score.value > 0.0 {- hits += 1;- }-- c- })- })- }-}diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsnew file mode 100644index 00000000..75dd423f--- /dev/null+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -0,0 +1,440 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::query::optic::AsSearchableRule;+use crate::query::Query;+use crate::schema::text_field::TextField;+use crate::Result;+use crate::{+ enum_map::EnumMap, fastfield_reader, schema::TextFieldEnum, webgraph::NodeID, webpage::Webpage,+};++use std::cell::RefCell;++use std::sync::Arc;++use tantivy::fieldnorm::FieldNormReader;+use tantivy::postings::SegmentPostings;+use tantivy::query::{Query as _, Scorer};+use tantivy::tokenizer::Tokenizer as _;++use tantivy::DocId;+use tantivy::DocSet;++use crate::webpage::region::RegionCount;++use crate::ranking::bm25::MultiBm25Weight;+use crate::ranking::models::linear::LinearRegression;+use crate::ranking::{inbound_similarity, query_centrality};++use super::{ComputedSignal, Signal, SignalCoefficient, SignalEnum, SignalScore};++mod order;+pub use order::SignalComputeOrder;++#[derive(Clone)]+pub struct TextFieldData {+ pub(super) postings: Vec<SegmentPostings>,+ pub(super) weight: MultiBm25Weight,+ pub(super) fieldnorm_reader: FieldNormReader,+}++pub struct RuleBoost {+ docset: Box<dyn Scorer>,+ boost: f64,+}++pub struct OpticBoosts {+ rules: Vec<RuleBoost>,+}++pub struct SegmentReader {+ text_fields: EnumMap<TextFieldEnum, TextFieldData>,+ optic_boosts: OpticBoosts,+ fastfield_reader: Arc<fastfield_reader::SegmentReader>,+}++impl SegmentReader {+ pub fn text_fields_mut(&mut self) -> &mut EnumMap<TextFieldEnum, TextFieldData> {+ &mut self.text_fields+ }++ pub fn fastfield_reader(&self) -> &fastfield_reader::SegmentReader {+ &self.fastfield_reader+ }+}++#[derive(Clone)]+pub struct QueryData {+ simple_terms: Vec<String>,+ optic_rules: Vec<optics::Rule>,+ selected_region: Option<crate::webpage::Region>,+}+impl QueryData {+ pub fn selected_region(&self) -> Option<crate::webpage::Region> {+ self.selected_region+ }+}++pub struct SignalComputer {+ query_data: Option<QueryData>,+ query_signal_coefficients: Option<SignalCoefficient>,+ segment_reader: Option<RefCell<SegmentReader>>,+ inbound_similarity: Option<RefCell<inbound_similarity::Scorer>>,+ fetch_time_ms_cache: Vec<f64>,+ update_time_cache: Vec<f64>,+ query_centrality: Option<RefCell<query_centrality::Scorer>>,+ region_count: Option<Arc<RegionCount>>,+ current_timestamp: Option<usize>,+ linear_regression: Option<Arc<LinearRegression>>,+ order: SignalComputeOrder,+}++impl Clone for SignalComputer {+ fn clone(&self) -> Self {+ let inbound_similarity = self+ .inbound_similarity+ .as_ref()+ .map(|scorer| RefCell::new(scorer.borrow().clone()));++ let query_centrality = self+ .query_centrality+ .as_ref()+ .map(|scorer| RefCell::new(scorer.borrow().clone()));++ Self {+ query_data: self.query_data.clone(),+ query_signal_coefficients: self.query_signal_coefficients.clone(),+ segment_reader: None,+ inbound_similarity,+ fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),+ update_time_cache: self.update_time_cache.clone(),+ query_centrality,+ region_count: self.region_count.clone(),+ current_timestamp: self.current_timestamp,+ linear_regression: self.linear_regression.clone(),+ order: self.order.clone(),+ }+ }+}++impl std::fmt::Debug for SignalComputer {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("SignalComputer")+ .field(+ "query",+ &self+ .query_data+ .as_ref()+ .map(|q| q.simple_terms.clone())+ .unwrap_or_default(),+ )+ .finish()+ }+}++impl SignalComputer {+ pub fn new(query: Option<&Query>) -> Self {+ let query_signal_coefficients = query.as_ref().and_then(|q| q.signal_coefficients());++ let fetch_time_ms_cache: Vec<_> = (0..1000)+ .map(|fetch_time| 1.0 / (fetch_time as f64 + 1.0))+ .collect();++ let update_time_cache = (0..(3 * 365 * 24))+ .map(|hours_since_update| 1.0 / ((hours_since_update as f64 + 1.0).log2()))+ .collect();++ let query = query.as_ref().map(|q| QueryData {+ simple_terms: q.simple_terms().to_vec(),+ optic_rules: q+ .optics()+ .iter()+ .flat_map(|o| o.rules.iter())+ .filter(|rule| match rule.action {+ optics::Action::Downrank(b) | optics::Action::Boost(b) => b != 0,+ optics::Action::Discard => false,+ })+ .cloned()+ .collect(),+ selected_region: q.region().cloned(),+ });++ let mut s = Self {+ segment_reader: None,+ inbound_similarity: None,+ query_signal_coefficients,+ fetch_time_ms_cache,+ update_time_cache,+ query_centrality: None,+ region_count: None,+ current_timestamp: None,+ linear_regression: None,+ query_data: query,+ order: SignalComputeOrder::empty(),+ };++ s.order = SignalComputeOrder::new(&s);+ s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);++ s+ }++ fn prepare_textfields(+ &self,+ tv_searcher: &tantivy::Searcher,+ segment_reader: &tantivy::SegmentReader,+ ) -> Result<EnumMap<TextFieldEnum, TextFieldData>> {+ let mut text_fields = EnumMap::new();+ let schema = tv_searcher.schema();++ if let Some(query) = &self.query_data {+ if !query.simple_terms.is_empty() {+ for signal in SignalEnum::all() {+ if let Some(text_field) = signal.as_textfield() {+ let tv_field = schema.get_field(text_field.name()).unwrap();+ let simple_query = itertools::intersperse(+ query.simple_terms.iter().map(|s| s.as_str()),+ " ",+ )+ .collect::<String>();++ let mut terms = Vec::new();+ let mut tokenizer = text_field.indexing_tokenizer();+ let mut stream = tokenizer.token_stream(&simple_query);++ while let Some(token) = stream.next() {+ let term = tantivy::Term::from_field_text(tv_field, &token.text);+ terms.push(term);+ }++ if terms.is_empty() {+ continue;+ }++ let fieldnorm_reader = segment_reader.get_fieldnorms_reader(tv_field)?;+ let inverted_index = segment_reader.inverted_index(tv_field)?;++ let mut matching_terms = Vec::with_capacity(terms.len());+ let mut postings = Vec::with_capacity(terms.len());+ for term in &terms {+ if let Some(p) =+ inverted_index.read_postings(term, text_field.record_option())?+ {+ postings.push(p);+ matching_terms.push(term.clone());+ }+ }+ let weight = MultiBm25Weight::for_terms(tv_searcher, &matching_terms)?;++ text_fields.insert(+ text_field,+ TextFieldData {+ postings,+ weight,+ fieldnorm_reader,+ },+ );+ }+ }+ }+ }++ Ok(text_fields)+ }++ fn prepare_optic(+ &self,+ tv_searcher: &tantivy::Searcher,+ segment_reader: &tantivy::SegmentReader,+ fastfield_reader: &fastfield_reader::FastFieldReader,+ ) -> Vec<RuleBoost> {+ let mut optic_rule_boosts = Vec::new();++ if let Some(query) = &self.query_data {+ optic_rule_boosts = query+ .optic_rules+ .iter()+ .filter_map(|rule| rule.as_searchable_rule(tv_searcher.schema(), fastfield_reader))+ .map(|(_, rule)| RuleBoost {+ docset: rule+ .query+ .weight(tantivy::query::EnableScoring::Enabled {+ searcher: tv_searcher,+ statistics_provider: tv_searcher,+ })+ .unwrap()+ .scorer(segment_reader, 0.0)+ .unwrap(),+ boost: rule.boost,+ })+ .collect();+ }++ optic_rule_boosts+ }++ pub fn register_segment(+ &mut self,+ tv_searcher: &tantivy::Searcher,+ segment_reader: &tantivy::SegmentReader,+ fastfield_reader: &fastfield_reader::FastFieldReader,+ ) -> Result<()> {+ let fastfield_segment_reader = fastfield_reader.get_segment(&segment_reader.segment_id());+ let text_fields = self.prepare_textfields(tv_searcher, segment_reader)?;+ let optic_rule_boosts = self.prepare_optic(tv_searcher, segment_reader, fastfield_reader);++ self.segment_reader = Some(RefCell::new(SegmentReader {+ text_fields,+ fastfield_reader: fastfield_segment_reader,+ optic_boosts: OpticBoosts {+ rules: optic_rule_boosts,+ },+ }));++ Ok(())+ }++ pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {+ self.query_centrality = Some(RefCell::new(query_centrality));+ }++ pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {+ let mut scorer = scorer;+ scorer.set_default_if_precalculated(true);++ self.inbound_similarity = Some(RefCell::new(scorer));+ }++ pub fn set_region_count(&mut self, region_count: RegionCount) {+ self.region_count = Some(Arc::new(region_count));+ }++ pub fn set_current_timestamp(&mut self, current_timestamp: usize) {+ self.current_timestamp = Some(current_timestamp);+ }++ pub fn set_linear_model(&mut self, linear_model: Arc<LinearRegression>) {+ self.linear_regression = Some(linear_model);+ }++ pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {+ self.query_centrality+ .as_ref()+ .map(|scorer| scorer.borrow_mut().score(host_id))+ }++ pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {+ self.inbound_similarity+ .as_ref()+ .map(|scorer| scorer.borrow_mut().score(&host_id))+ .unwrap_or_default()+ }++ /// Computes the scored signals for a given document.+ ///+ /// Important: This function assues that the docs a scored in ascending order of docid+ /// within their segment. If this invariant is not upheld, the documents will not have+ /// scores calculated for their text related signals. The wrong ranking will most likely+ /// be returned.+ /// This function also assumes that the segment reader has been set.+ pub fn compute_signals(&self, doc: DocId) -> impl Iterator<Item = Option<ComputedSignal>> + '_ {+ self.order.compute(doc, self)+ }++ pub fn boosts(&mut self, doc: DocId) -> Option<f64> {+ self.segment_reader.as_ref().map(|segment_reader| {+ let mut downrank = 0.0;+ let mut boost = 0.0;++ for rule in &mut segment_reader.borrow_mut().optic_boosts.rules {+ if rule.docset.doc() > doc {+ continue;+ }++ if rule.docset.doc() == doc || rule.docset.seek(doc) == doc {+ if rule.boost < 0.0 {+ downrank += rule.boost.abs();+ } else {+ boost += rule.boost;+ }+ }+ }++ if downrank > boost {+ let diff = downrank - boost;+ 1.0 / (1.0 + diff)+ } else {+ boost - downrank + 1.0+ }+ })+ }++ pub fn precompute_score(&self, webpage: &Webpage) -> f64 {+ SignalEnum::all()+ .filter_map(|signal| {+ signal+ .precompute(webpage, self)+ .map(|value| ComputedSignal {+ signal,+ score: SignalScore {+ coefficient: self.coefficient(&signal),+ value,+ },+ })+ })+ .map(|computed| computed.score.coefficient * computed.score.value)+ .sum()+ }++ pub fn coefficient(&self, signal: &SignalEnum) -> f64 {+ self.query_signal_coefficients+ .as_ref()+ .map(|coefficients| coefficients.get(signal))+ .or_else(|| {+ self.linear_regression+ .as_ref()+ .and_then(|model| model.weights.get(*signal).copied())+ })+ .unwrap_or(signal.default_coefficient())+ }++ pub fn segment_reader(&self) -> Option<&RefCell<SegmentReader>> {+ self.segment_reader.as_ref()+ }++ pub fn fetch_time_ms_cache(&self) -> &[f64] {+ &self.fetch_time_ms_cache+ }++ pub fn current_timestamp(&self) -> Option<usize> {+ self.current_timestamp+ }++ pub fn update_time_cache(&self) -> &[f64] {+ &self.update_time_cache+ }++ pub fn region_count(&self) -> Option<&RegionCount> {+ self.region_count+ .as_ref()+ .map(|region_count| &**region_count)+ }++ pub fn query_data(&self) -> Option<&QueryData> {+ self.query_data.as_ref()+ }+}diff --git a/crates/core/src/ranking/signal/computer/order.rs b/crates/core/src/ranking/signal/computer/order.rsnew file mode 100644index 00000000..64ae2d21--- /dev/null+++ b/crates/core/src/ranking/signal/computer/order.rs@@ -0,0 +1,154 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::DocId;++use crate::{+ enum_map::EnumMap,+ ranking::{ComputedSignal, Signal, SignalEnum, SignalScore},+ schema::{text_field::TextField, TextFieldEnum},+};++use super::SignalComputer;++#[derive(Clone)]+pub struct SignalComputeOrder {+ text_signals: EnumMap<TextFieldEnum, NGramComputeOrder>,+ other_signals: Vec<SignalEnum>,+}++impl SignalComputeOrder {+ pub fn empty() -> Self {+ Self {+ text_signals: EnumMap::new(),+ other_signals: Vec::new(),+ }+ }++ pub fn new(signal_computer: &SignalComputer) -> Self {+ let mut text_signals = EnumMap::new();+ let mut other_signals = Vec::new();++ for signal in SignalEnum::all() {+ if signal_computer.coefficient(&signal) == 0.0 {+ continue;+ }++ if let Some(text_field) = signal.as_textfield() {+ let mono = text_field.monogram_field();++ if !text_signals.contains_key(mono) {+ text_signals.insert(mono, NGramComputeOrder::default());+ }++ let ngram = text_field.ngram_size();+ text_signals.get_mut(mono).unwrap().push(signal, ngram);+ } else {+ other_signals.push(signal);+ }+ }++ Self {+ text_signals,+ other_signals,+ }+ }++ pub fn compute<'a>(+ &'a self,+ doc: DocId,+ signal_computer: &'a SignalComputer,+ ) -> impl Iterator<Item = Option<ComputedSignal>> + 'a {+ self.text_signals+ .values()+ .flat_map(move |ngram| ngram.compute(doc, signal_computer))+ .map(Some)+ .chain(+ self.other_signals+ .iter()+ .filter_map(|signal| {+ let coefficient = signal_computer.coefficient(signal);++ if coefficient > 0.0 {+ Some((signal, coefficient))+ } else {+ None+ }+ })+ .map(move |(signal, coefficient)| {+ signal+ .compute(doc, signal_computer)+ .map(|value| ComputedSignal {+ signal: *signal,+ score: SignalScore { coefficient, value },+ })+ }),+ )+ }+}++/// If an ngram of size n matches the query for a given document in a given field,+/// the score of all ngrams where n' < n is dampened by NGRAM_DAMPENING.+///+/// A dampening factor of 0.0 means that we ignore all ngrams where n' < n. A dampening factor of 1.0+/// does not dampen any ngrams.+const NGRAM_DAMPENING: f64 = 0.4;++#[derive(Debug, Default, Clone)]+pub struct NGramComputeOrder {+ /// ordered by descending ngram size. e.g. [title_bm25_trigram, title_bm25_bigram, title_bm25]+ signals: Vec<(usize, SignalEnum)>,+}++impl NGramComputeOrder {+ fn push(&mut self, signal: SignalEnum, ngram: usize) {+ self.signals.push((ngram, signal));+ self.signals.sort_unstable_by(|(a, _), (b, _)| b.cmp(a));+ }++ fn compute<'a>(+ &'a self,+ doc: DocId,+ signal_computer: &'a SignalComputer,+ ) -> impl Iterator<Item = ComputedSignal> + 'a {+ let mut hits = 0;++ self.signals+ .iter()+ .map(|(_, s)| s)+ .filter_map(move |signal| {+ signal+ .compute(doc, signal_computer)+ .map(|value| {+ let coefficient = signal_computer.coefficient(signal);++ ComputedSignal {+ signal: *signal,+ score: SignalScore { coefficient, value },+ }+ })+ .map(|mut c| {+ c.score.coefficient *= NGRAM_DAMPENING.powi(hits);++ if c.score.value > 0.0 {+ hits += 1;+ }++ c+ })+ })+ }+}diff --git a/crates/core/src/ranking/signal/mod.rs b/crates/core/src/ranking/signal/mod.rsnew file mode 100644index 00000000..f1954a37--- /dev/null+++ b/crates/core/src/ranking/signal/mod.rs@@ -0,0 +1,110 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::enum_map::EnumMap;+use optics::ast::RankingTarget;+use optics::Optic;+use serde::{Deserialize, Serialize};+use std::str::FromStr;++use thiserror::Error;+use utoipa::ToSchema;++mod computer;+mod non_text;+mod prelude;+mod text;++pub use computer::SignalComputer;+pub use non_text::*;+pub use prelude::*;+pub use text::*;++#[derive(Debug, Error)]+pub enum Error {+ #[error("unknown signal: {0}")]+ UnknownSignal(#[from] serde_json::Error),+}++impl FromStr for SignalEnumDiscriminants {+ type Err = Error;++ fn from_str(name: &str) -> std::result::Result<Self, Self::Err> {+ let s = "\"".to_string() + name + "\"";+ let signal = serde_json::from_str(&s)?;+ Ok(signal)+ }+}++#[derive(Debug, Clone, Default)]+pub struct SignalCoefficient {+ map: EnumMap<SignalEnum, f64>,+}++impl SignalCoefficient {+ pub fn get(&self, signal: &SignalEnum) -> f64 {+ self.map+ .get(*signal)+ .copied()+ .unwrap_or(signal.default_coefficient())+ }++ pub fn new(coefficients: impl Iterator<Item = (SignalEnum, f64)>) -> Self {+ let mut map = EnumMap::default();++ for (signal, coefficient) in coefficients {+ map.insert(signal, coefficient);+ }++ Self { map }+ }++ pub fn from_optic(optic: &Optic) -> Self {+ SignalCoefficient::new(optic.rankings.iter().filter_map(|coeff| {+ match &coeff.target {+ RankingTarget::Signal(signal) => SignalEnumDiscriminants::from_str(signal)+ .ok()+ .map(|signal| (signal.into(), coeff.value)),+ }+ }))+ }++ pub fn merge_into(&mut self, coeffs: SignalCoefficient) {+ for signal in SignalEnum::all() {+ if let Some(coeff) = coeffs.map.get(signal).copied() {+ match self.map.get_mut(signal) {+ Some(existing_coeff) => *existing_coeff += coeff,+ None => {+ self.map.insert(signal, coeff);+ }+ }+ }+ }+ }+}++#[derive(Debug, Clone, Copy)]+pub struct ComputedSignal {+ pub signal: SignalEnum,+ pub score: SignalScore,+}++#[derive(Debug, Clone, Copy, Serialize, Deserialize, ToSchema)]+#[serde(rename_all = "camelCase")]+pub struct SignalScore {+ pub coefficient: f64,+ pub value: f64,+}diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsnew file mode 100644index 00000000..098d1e99--- /dev/null+++ b/crates/core/src/ranking/signal/non_text.rs@@ -0,0 +1,531 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use tantivy::DocId;++use super::{Signal, SignalComputer};+use crate::{+ fastfield_reader::FieldReader,+ schema::{self, Field, FLOAT_SCALING},+ webgraph::NodeID,+ webpage::Webpage,+};++fn score_timestamp(page_timestamp: usize, signal_computer: &SignalComputer) -> f64 {+ if page_timestamp >= signal_computer.current_timestamp().unwrap_or(0) {+ return 0.0;+ }++ let hours_since_update =+ (signal_computer.current_timestamp().unwrap() - page_timestamp).max(1) / 3600;++ signal_computer+ .update_time_cache()+ .get(hours_since_update)+ .copied()+ .unwrap_or(0.0)+}++#[inline]+fn score_rank(rank: f64) -> f64 {+ 1.0 / (rank + 1.0)+}++#[inline]+fn score_trackers(num_trackers: f64) -> f64 {+ 1.0 / (num_trackers + 1.0)+}++#[inline]+fn score_digits(num_digits: f64) -> f64 {+ 1.0 / (num_digits + 1.0)+}++#[inline]+fn score_slashes(num_slashes: f64) -> f64 {+ 1.0 / (num_slashes + 1.0)+}++#[inline]+fn score_link_density(link_density: f64) -> f64 {+ if link_density > 0.5 {+ 0.0+ } else {+ 1.0 - link_density+ }+}++fn score_region(webpage_region: crate::webpage::Region, computer: &SignalComputer) -> f64 {+ match computer.region_count() {+ Some(region_count) => {+ let boost = computer+ .query_data()+ .and_then(|q| q.selected_region())+ .map_or(0.0, |region| {+ if region != crate::webpage::Region::All && region == webpage_region {+ 50.0+ } else {+ 0.0+ }+ });++ boost + region_count.score(&webpage_region)+ }+ None => 0.0,+ }+}++fn host_id<'a>(fastfield_reader: &FieldReader<'a>) -> Option<NodeID> {+ let node_id = fastfield_reader+ .get(schema::fast_field::HostNodeID.into())+ .and_then(|n| n.as_u64())+ .unwrap();++ if node_id == u64::MAX {+ None+ } else {+ Some(node_id.into())+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct HostCentrality;+impl Signal for HostCentrality {+ fn default_coefficient(&self) -> f64 {+ 0.5+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::HostCentrality.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ Some(webpage.host_centrality)+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(val as f64 / FLOAT_SCALING as f64)+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct HostCentralityRank;+impl Signal for HostCentralityRank {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::HostCentralityRank.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ Some(score_rank(webpage.host_centrality_rank as f64))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_rank(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct PageCentrality;+impl Signal for PageCentrality {+ fn default_coefficient(&self) -> f64 {+ 0.25+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::PageCentrality.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ Some(webpage.page_centrality)+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(val as f64 / FLOAT_SCALING as f64)+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct PageCentralityRank;+impl Signal for PageCentralityRank {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::PageCentralityRank.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ Some(score_rank(webpage.page_centrality_rank as f64))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_rank(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IsHomepage;+impl Signal for IsHomepage {+ fn default_coefficient(&self) -> f64 {+ 0.0005+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::IsHomepage.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ Some(webpage.html.is_homepage().into())+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(val as f64)+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct FetchTimeMs;+impl Signal for FetchTimeMs {+ fn default_coefficient(&self) -> f64 {+ 0.001+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::FetchTimeMs.into()))+ }++ fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {+ let fetch_time_ms = webpage.fetch_time_ms as usize;+ if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {+ Some(0.0)+ } else {+ Some(signal_computer.fetch_time_ms_cache()[fetch_time_ms])+ }+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let fetch_time_ms = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap() as usize;++ if fetch_time_ms >= signal_computer.fetch_time_ms_cache().len() {+ Some(0.0)+ } else {+ Some(signal_computer.fetch_time_ms_cache()[fetch_time_ms])+ }+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct UpdateTimestamp;+impl Signal for UpdateTimestamp {+ fn default_coefficient(&self) -> f64 {+ 0.001+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::LastUpdated.into()))+ }++ fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {+ let update_timestamp = webpage+ .html+ .updated_time()+ .map(|date| date.timestamp().max(0))+ .unwrap_or(0) as usize;++ Some(score_timestamp(update_timestamp, signal_computer))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap() as usize;++ Some(score_timestamp(val, signal_computer))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct TrackerScore;+impl Signal for TrackerScore {+ fn default_coefficient(&self) -> f64 {+ 0.05+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::TrackerScore.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ let num_trackers = webpage.html.trackers().len() as f64;+ Some(score_trackers(num_trackers))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_trackers(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Region;+impl Signal for Region {+ fn default_coefficient(&self) -> f64 {+ 0.15+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::Region.into()))+ }++ fn precompute(self, webpage: &Webpage, signal_computer: &SignalComputer) -> Option<f64> {+ let region =+ crate::webpage::Region::guess_from(webpage).unwrap_or(crate::webpage::Region::All);+ Some(score_region(region, signal_computer))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ let region = crate::webpage::Region::from_id(val);+ Some(score_region(region, signal_computer))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct QueryCentrality;+impl Signal for QueryCentrality {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let host_id = host_id(&fastfield_reader);++ host_id.and_then(|host_id| signal_computer.query_centrality(host_id))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct InboundSimilarity;+impl Signal for InboundSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.25+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);+ let host_id = host_id(&fastfield_reader);++ host_id.map(|host_id| signal_computer.inbound_similarity(host_id))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct LambdaMart;+impl Signal for LambdaMart {+ fn default_coefficient(&self) -> f64 {+ 10.0+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ None // computed in later ranking stage+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct UrlDigits;+impl Signal for UrlDigits {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(+ schema::fast_field::NumPathAndQueryDigits.into(),+ ))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ let num_digits = (webpage+ .html+ .url()+ .path()+ .chars()+ .filter(|c| c.is_ascii_digit())+ .count()+ + webpage+ .html+ .url()+ .query()+ .unwrap_or_default()+ .chars()+ .filter(|c| c.is_ascii_digit())+ .count()) as f64;++ Some(score_digits(num_digits))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_digits(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct UrlSlashes;+impl Signal for UrlSlashes {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(+ schema::fast_field::NumPathAndQuerySlashes.into(),+ ))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ let num_slashes = webpage+ .html+ .url()+ .path()+ .chars()+ .filter(|c| c == &'/')+ .count() as f64;+ Some(score_slashes(num_slashes))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_slashes(val as f64))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct LinkDensity;+impl Signal for LinkDensity {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::LinkDensity.into()))+ }++ fn precompute(self, webpage: &Webpage, _: &SignalComputer) -> Option<f64> {+ let link_density = webpage.html.link_density();+ Some(score_link_density(link_density))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();+ let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);++ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();+ Some(score_link_density(val as f64 / FLOAT_SCALING as f64))+ }+}diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/signal/prelude.rsnew file mode 100644index 00000000..20cf8a8d--- /dev/null+++ b/crates/core/src/ranking/signal/prelude.rs@@ -0,0 +1,171 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>+use crate::enum_map::InsertEnumMapKey;++use crate::enum_dispatch_from_discriminant;+use crate::schema::Field;+use crate::{+ schema::{FastFieldEnum, TextFieldEnum},+ webpage::Webpage,+};+use enum_dispatch::enum_dispatch;++use strum::{EnumDiscriminants, VariantArray};++use super::non_text::*;+use super::text::*;+use super::SignalComputer;+use tantivy::DocId;++#[enum_dispatch]+pub trait Signal:+ Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash + Into<SignalEnum>+{+ fn default_coefficient(&self) -> f64;+ fn as_field(&self) -> Option<Field>;+ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64>;++ fn precompute(self, _webpage: &Webpage, _signal_computer: &SignalComputer) -> Option<f64> {+ None+ }++ fn as_textfield(&self) -> Option<TextFieldEnum> {+ self.as_field().and_then(|field| field.as_text())+ }++ fn as_fastfield(&self) -> Option<FastFieldEnum> {+ self.as_field().and_then(|field| field.as_fast())+ }+}++#[enum_dispatch(Signal)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants,+)]+#[strum_discriminants(derive(VariantArray, serde::Serialize, serde::Deserialize, Hash))]+#[strum_discriminants(serde(rename_all = "snake_case"))]+pub enum SignalEnum {+ Bm25Title,+ Bm25TitleBigrams,+ Bm25TitleTrigrams,+ Bm25CleanBody,+ Bm25CleanBodyBigrams,+ Bm25CleanBodyTrigrams,+ Bm25StemmedTitle,+ Bm25StemmedCleanBody,+ Bm25AllBody,+ Bm25Keywords,+ Bm25BacklinkText,+ IdfSumUrl,+ IdfSumSite,+ IdfSumDomain,+ IdfSumSiteNoTokenizer,+ IdfSumDomainNoTokenizer,+ IdfSumDomainNameNoTokenizer,+ IdfSumDomainIfHomepage,+ IdfSumDomainNameIfHomepageNoTokenizer,+ IdfSumDomainIfHomepageNoTokenizer,+ IdfSumTitleIfHomepage,+ CrossEncoderSnippet,+ CrossEncoderTitle,+ HostCentrality,+ HostCentralityRank,+ PageCentrality,+ PageCentralityRank,+ IsHomepage,+ FetchTimeMs,+ UpdateTimestamp,+ TrackerScore,+ Region,+ QueryCentrality,+ InboundSimilarity,+ LambdaMart,+ UrlDigits,+ UrlSlashes,+ LinkDensity,+ TitleEmbeddingSimilarity,+ KeywordEmbeddingSimilarity,+}++enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,+[+ Bm25Title,+ Bm25TitleBigrams,+ Bm25TitleTrigrams,+ Bm25CleanBody,+ Bm25CleanBodyBigrams,+ Bm25CleanBodyTrigrams,+ Bm25StemmedTitle,+ Bm25StemmedCleanBody,+ Bm25AllBody,+ Bm25Keywords,+ Bm25BacklinkText,+ IdfSumUrl,+ IdfSumSite,+ IdfSumDomain,+ IdfSumSiteNoTokenizer,+ IdfSumDomainNoTokenizer,+ IdfSumDomainNameNoTokenizer,+ IdfSumDomainIfHomepage,+ IdfSumDomainNameIfHomepageNoTokenizer,+ IdfSumDomainIfHomepageNoTokenizer,+ IdfSumTitleIfHomepage,+ CrossEncoderSnippet,+ CrossEncoderTitle,+ HostCentrality,+ HostCentralityRank,+ PageCentrality,+ PageCentralityRank,+ IsHomepage,+ FetchTimeMs,+ UpdateTimestamp,+ TrackerScore,+ Region,+ QueryCentrality,+ InboundSimilarity,+ LambdaMart,+ UrlDigits,+ UrlSlashes,+ LinkDensity,+ TitleEmbeddingSimilarity,+ KeywordEmbeddingSimilarity,+]);++impl SignalEnum {+ pub fn num_variants() -> usize {+ SignalEnumDiscriminants::VARIANTS.len()+ }++ pub fn all() -> impl Iterator<Item = SignalEnum> {+ SignalEnumDiscriminants::VARIANTS+ .iter()+ .copied()+ .map(|v| v.into())+ }++ pub fn get(field_id: usize) -> Option<SignalEnum> {+ SignalEnumDiscriminants::VARIANTS+ .get(field_id)+ .copied()+ .map(SignalEnum::from)+ }+}++impl InsertEnumMapKey for SignalEnum {+ fn into_usize(self) -> usize {+ SignalEnumDiscriminants::from(self) as usize+ }+}diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rsnew file mode 100644index 00000000..6d247486--- /dev/null+++ b/crates/core/src/ranking/signal/text.rs@@ -0,0 +1,573 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use crate::schema::{self, Field};+use itertools::Itertools;++use tantivy::DocSet;+use tantivy::{DocId, Postings};++use super::computer::TextFieldData;+use super::{Signal, SignalComputer};++fn bm25(field: &mut TextFieldData, doc: DocId) -> f64 {+ if field.postings.is_empty() {+ return 0.0;+ }++ let fieldnorm_id = field.fieldnorm_reader.fieldnorm_id(doc);++ field+ .weight+ .score(field.postings.iter_mut().map(move |posting| {+ if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {+ (fieldnorm_id, posting.term_freq())+ } else {+ (fieldnorm_id, 0)+ }+ })) as f64+}++fn idf_sum(field: &mut TextFieldData, doc: DocId) -> f64 {+ if field.postings.is_empty() {+ return 0.0;+ }+ let idf = field.weight.idf();++ field+ .postings+ .iter_mut()+ .zip_eq(idf)+ .filter_map(|(posting, idf)| {+ if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {+ Some(idf)+ } else {+ None+ }+ })+ .sum::<f32>() as f64+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25Title;+impl Signal for Bm25Title {+ fn default_coefficient(&self) -> f64 {+ 0.0063+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Title.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25TitleBigrams;+impl Signal for Bm25TitleBigrams {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::TitleBigrams.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25TitleTrigrams;+impl Signal for Bm25TitleTrigrams {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::TitleTrigrams.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25CleanBody;+impl Signal for Bm25CleanBody {+ fn default_coefficient(&self) -> f64 {+ 0.0063+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::CleanBody.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25CleanBodyBigrams;+impl Signal for Bm25CleanBodyBigrams {+ fn default_coefficient(&self) -> f64 {+ 0.005+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::CleanBodyBigrams.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25CleanBodyTrigrams;+impl Signal for Bm25CleanBodyTrigrams {+ fn default_coefficient(&self) -> f64 {+ 0.005+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::CleanBodyTrigrams.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25StemmedTitle;+impl Signal for Bm25StemmedTitle {+ fn default_coefficient(&self) -> f64 {+ 0.003+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::StemmedTitle.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25StemmedCleanBody;+impl Signal for Bm25StemmedCleanBody {+ fn default_coefficient(&self) -> f64 {+ 0.001+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::StemmedCleanBody.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25AllBody;+impl Signal for Bm25AllBody {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::AllBody.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25Keywords;+impl Signal for Bm25Keywords {+ fn default_coefficient(&self) -> f64 {+ 0.001+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Keywords.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct Bm25BacklinkText;+impl Signal for Bm25BacklinkText {+ fn default_coefficient(&self) -> f64 {+ 0.003+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::BacklinkText.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| bm25(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumUrl;+impl Signal for IdfSumUrl {+ fn default_coefficient(&self) -> f64 {+ 0.0003+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Url.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumSite;+impl Signal for IdfSumSite {+ fn default_coefficient(&self) -> f64 {+ 0.00015+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::SiteWithout.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomain;+impl Signal for IdfSumDomain {+ fn default_coefficient(&self) -> f64 {+ 0.0003+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::Domain.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumSiteNoTokenizer;+impl Signal for IdfSumSiteNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.00015+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::SiteNoTokenizer.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainNoTokenizer;+impl Signal for IdfSumDomainNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.0002+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::DomainNoTokenizer.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainNameNoTokenizer;+impl Signal for IdfSumDomainNameNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.0002+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(+ schema::text_field::DomainNameNoTokenizer.into(),+ ))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainIfHomepage;+impl Signal for IdfSumDomainIfHomepage {+ fn default_coefficient(&self) -> f64 {+ 0.0004+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::DomainIfHomepage.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainNameIfHomepageNoTokenizer;+impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.0036+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(+ schema::text_field::DomainNameIfHomepageNoTokenizer.into(),+ ))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumDomainIfHomepageNoTokenizer;+impl Signal for IdfSumDomainIfHomepageNoTokenizer {+ fn default_coefficient(&self) -> f64 {+ 0.0036+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(+ schema::text_field::DomainIfHomepageNoTokenizer.into(),+ ))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct IdfSumTitleIfHomepage;+impl Signal for IdfSumTitleIfHomepage {+ fn default_coefficient(&self) -> f64 {+ 0.00022+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Text(schema::text_field::TitleIfHomepage.into()))+ }++ fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ let mut seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();++ seg_reader+ .text_fields_mut()+ .get_mut(self.as_textfield().unwrap())+ .map(|field| idf_sum(field, doc))+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct CrossEncoderSnippet;+impl Signal for CrossEncoderSnippet {+ fn default_coefficient(&self) -> f64 {+ 0.17+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ None // computed in later ranking stage+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct CrossEncoderTitle;+impl Signal for CrossEncoderTitle {+ fn default_coefficient(&self) -> f64 {+ 0.17+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ None // computed in later ranking stage+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct TitleEmbeddingSimilarity;+impl Signal for TitleEmbeddingSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::TitleEmbeddings.into()))+ }++ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ None // computed in later ranking stage+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct KeywordEmbeddingSimilarity;+impl Signal for KeywordEmbeddingSimilarity {+ fn default_coefficient(&self) -> f64 {+ 0.01+ }++ fn as_field(&self) -> Option<Field> {+ Some(Field::Fast(schema::fast_field::KeywordEmbeddings.into()))+ }++ fn compute(&self, _: DocId, _: &SignalComputer) -> Option<f64> {+ None // computed in later ranking stage+ }+}