web search done right
number of terms in query as ranking signal
Mikkel Denkeracb65099ef1e
diff --git a/crates/core/src/ranking/core/computer/mod.rs b/crates/core/src/ranking/core/computer/mod.rsindex d7a45308..1939e50a 100644--- a/crates/core/src/ranking/core/computer/mod.rs+++ b/crates/core/src/ranking/core/computer/mod.rs@@ -146,6 +146,10 @@ impl QueryData {pub fn selected_region(&self) -> Option<crate::webpage::Region> {self.selected_region}++ pub fn simple_terms(&self) -> &[String] {+ &self.simple_terms+ }}pub struct SignalComputer {@@ -228,10 +232,9 @@ impl SignalComputer {current_timestamp: None,linear_regression: None,query_data: query,- order: SignalComputeOrder::empty(),+ order: SignalComputeOrder::new(),};- s.order = SignalComputeOrder::new(&s);s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);sdiff --git a/crates/core/src/ranking/core/computer/order.rs b/crates/core/src/ranking/core/computer/order.rsindex ba4afbbd..8d3f94a5 100644--- a/crates/core/src/ranking/core/computer/order.rs+++ b/crates/core/src/ranking/core/computer/order.rs@@ -38,15 +38,11 @@ impl SignalComputeOrder {}}- pub fn new(signal_computer: &SignalComputer) -> Self {+ pub fn new() -> Self {let mut text_signals = EnumMap::new();let mut other_signals = Vec::new();for signal in SignalEnum::all() {- if signal_computer.coefficient(&signal) == 0.0 {- continue;- }-if let Some(text_field) = signal.as_textfield() {let mono = text_field.monogram_field();@@ -76,13 +72,6 @@ impl SignalComputeOrder {.values().flat_map(move |ngram| ngram.compute(doc, signal_computer)).chain(self.other_signals.iter().map(move |signal| {- if signal_computer.coefficient(signal) == 0.0 {- return ComputedSignal {- signal: *signal,- score: 0.0,- };- }-signal.compute(doc, signal_computer).map(|score| ComputedSignal {diff --git a/crates/core/src/ranking/core/mod.rs b/crates/core/src/ranking/core/mod.rsindex fb4d4340..3ca4eb8c 100644--- a/crates/core/src/ranking/core/mod.rs+++ b/crates/core/src/ranking/core/mod.rs@@ -24,6 +24,7 @@ use utoipa::ToSchema;mod computer;mod non_text;mod prelude;+mod query;mod text;pub use computer::SignalComputer;diff --git a/crates/core/src/ranking/core/non_text.rs b/crates/core/src/ranking/core/non_text.rsindex 2a3b77db..ec52a6db 100644--- a/crates/core/src/ranking/core/non_text.rs+++ b/crates/core/src/ranking/core/non_text.rs@@ -498,7 +498,8 @@ impl Signal for QueryCentrality {}fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<f64> {- unimplemented!()+ // unimplemented!()+ None}}diff --git a/crates/core/src/ranking/core/prelude.rs b/crates/core/src/ranking/core/prelude.rsindex 825c0878..d41f0d38 100644--- a/crates/core/src/ranking/core/prelude.rs+++ b/crates/core/src/ranking/core/prelude.rs@@ -1,4 +1,3 @@-use crate::enum_dispatch_from_discriminant;// Stract is an open source web search engine.// Copyright (C) 2024 Stract ApS//@@ -14,6 +13,8 @@ use crate::enum_dispatch_from_discriminant;//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>++use crate::enum_dispatch_from_discriminant;use crate::enum_map::{GetEnumMapKey, InsertEnumMapKey};use crate::schema::Field;@@ -26,6 +27,7 @@ use enum_dispatch::enum_dispatch;use strum::{EnumDiscriminants, VariantArray};use super::non_text::*;+use super::query::*;use super::text::*;use super::SignalComputer;use tantivy::DocId;@@ -120,6 +122,7 @@ pub enum SignalEnum {TitleEmbeddingSimilarity,KeywordEmbeddingSimilarity,HasAds,+ NumQueryTerms,}enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,@@ -166,6 +169,7 @@ enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,TitleEmbeddingSimilarity,KeywordEmbeddingSimilarity,HasAds,+ NumQueryTerms]);impl SignalEnum {diff --git a/crates/core/src/ranking/core/query.rs b/crates/core/src/ranking/core/query.rsnew file mode 100644index 00000000..3af796bb--- /dev/null+++ b/crates/core/src/ranking/core/query.rs@@ -0,0 +1,50 @@+use tantivy::DocId;++use crate::schema::Field;++// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use super::{Signal, SignalComputer};++#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]+pub struct NumQueryTerms;+impl Signal for NumQueryTerms {+ fn default_coefficient(&self) -> f64 {+ 0.0+ }++ fn as_field(&self) -> Option<Field> {+ None+ }++ fn compute(&self, _: DocId, signal_computer: &SignalComputer) -> Option<f64> {+ signal_computer+ .query_data()+ .map(|d| d.simple_terms().len() as f64)+ }+}diff --git a/crates/core/src/webpage/adservers.txt b/crates/core/src/webpage/adservers.txtindex dec966b2..8ee8caf8 100644--- a/crates/core/src/webpage/adservers.txt+++ b/crates/core/src/webpage/adservers.txt@@ -1842,7 +1842,6 @@ gloyah.net,gmads.net,gml.email,go-clicks.de,-go-mpulse.net,go-rank.de,go.dhs.gov,go.eu.sparkpostmail1.com,@@ -2435,7 +2434,6 @@ nondescriptstocking.com,nothingunit.com,novem.pl,npttech.com,-nr-data.net,nr.mmcdn.com,nr.static.mmcdn.com,ns1p.net,