Skip to content

web search done right

number of terms in query as ranking signal

Mikkel Denkeracb65099ef1e
diff --git a/crates/core/src/ranking/core/computer/mod.rs b/crates/core/src/ranking/core/computer/mod.rs
index d7a45308..1939e50a 100644
--- a/crates/core/src/ranking/core/computer/mod.rs
+++ b/crates/core/src/ranking/core/computer/mod.rs
@@ -146,6 +146,10 @@ impl QueryData {
pub fn selected_region(&self) -> Option<crate::webpage::Region> {
self.selected_region
}
+
+ pub fn simple_terms(&self) -> &[String] {
+ &self.simple_terms
+ }
}
pub struct SignalComputer {
@@ -228,10 +232,9 @@ impl SignalComputer {
current_timestamp: None,
linear_regression: None,
query_data: query,
- order: SignalComputeOrder::empty(),
+ order: SignalComputeOrder::new(),
};
- s.order = SignalComputeOrder::new(&s);
s.set_current_timestamp(chrono::Utc::now().timestamp() as usize);
s
diff --git a/crates/core/src/ranking/core/computer/order.rs b/crates/core/src/ranking/core/computer/order.rs
index ba4afbbd..8d3f94a5 100644
--- a/crates/core/src/ranking/core/computer/order.rs
+++ b/crates/core/src/ranking/core/computer/order.rs
@@ -38,15 +38,11 @@ impl SignalComputeOrder {
}
}
- pub fn new(signal_computer: &SignalComputer) -> Self {
+ pub fn new() -> Self {
let mut text_signals = EnumMap::new();
let mut other_signals = Vec::new();
for signal in SignalEnum::all() {
- if signal_computer.coefficient(&signal) == 0.0 {
- continue;
- }
-
if let Some(text_field) = signal.as_textfield() {
let mono = text_field.monogram_field();
@@ -76,13 +72,6 @@ impl SignalComputeOrder {
.values()
.flat_map(move |ngram| ngram.compute(doc, signal_computer))
.chain(self.other_signals.iter().map(move |signal| {
- if signal_computer.coefficient(signal) == 0.0 {
- return ComputedSignal {
- signal: *signal,
- score: 0.0,
- };
- }
-
signal
.compute(doc, signal_computer)
.map(|score| ComputedSignal {
diff --git a/crates/core/src/ranking/core/mod.rs b/crates/core/src/ranking/core/mod.rs
index fb4d4340..3ca4eb8c 100644
--- a/crates/core/src/ranking/core/mod.rs
+++ b/crates/core/src/ranking/core/mod.rs
@@ -24,6 +24,7 @@ use utoipa::ToSchema;
mod computer;
mod non_text;
mod prelude;
+mod query;
mod text;
pub use computer::SignalComputer;
diff --git a/crates/core/src/ranking/core/non_text.rs b/crates/core/src/ranking/core/non_text.rs
index 2a3b77db..ec52a6db 100644
--- a/crates/core/src/ranking/core/non_text.rs
+++ b/crates/core/src/ranking/core/non_text.rs
@@ -498,7 +498,8 @@ impl Signal for QueryCentrality {
}
fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<f64> {
- unimplemented!()
+ // unimplemented!()
+ None
}
}
diff --git a/crates/core/src/ranking/core/prelude.rs b/crates/core/src/ranking/core/prelude.rs
index 825c0878..d41f0d38 100644
--- a/crates/core/src/ranking/core/prelude.rs
+++ b/crates/core/src/ranking/core/prelude.rs
@@ -1,4 +1,3 @@
-use crate::enum_dispatch_from_discriminant;
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
@@ -14,6 +13,8 @@ use crate::enum_dispatch_from_discriminant;
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use crate::enum_dispatch_from_discriminant;
use crate::enum_map::{GetEnumMapKey, InsertEnumMapKey};
use crate::schema::Field;
@@ -26,6 +27,7 @@ use enum_dispatch::enum_dispatch;
use strum::{EnumDiscriminants, VariantArray};
use super::non_text::*;
+use super::query::*;
use super::text::*;
use super::SignalComputer;
use tantivy::DocId;
@@ -120,6 +122,7 @@ pub enum SignalEnum {
TitleEmbeddingSimilarity,
KeywordEmbeddingSimilarity,
HasAds,
+ NumQueryTerms,
}
enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,
@@ -166,6 +169,7 @@ enum_dispatch_from_discriminant!(SignalEnumDiscriminants => SignalEnum,
TitleEmbeddingSimilarity,
KeywordEmbeddingSimilarity,
HasAds,
+ NumQueryTerms
]);
impl SignalEnum {
diff --git a/crates/core/src/ranking/core/query.rs b/crates/core/src/ranking/core/query.rs
new file mode 100644
index 00000000..3af796bb
--- /dev/null
+++ b/crates/core/src/ranking/core/query.rs
@@ -0,0 +1,50 @@
+use tantivy::DocId;
+
+use crate::schema::Field;
+
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::{Signal, SignalComputer};
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
+pub struct NumQueryTerms;
+impl Signal for NumQueryTerms {
+ fn default_coefficient(&self) -> f64 {
+ 0.0
+ }
+
+ fn as_field(&self) -> Option<Field> {
+ None
+ }
+
+ fn compute(&self, _: DocId, signal_computer: &SignalComputer) -> Option<f64> {
+ signal_computer
+ .query_data()
+ .map(|d| d.simple_terms().len() as f64)
+ }
+}
diff --git a/crates/core/src/webpage/adservers.txt b/crates/core/src/webpage/adservers.txt
index dec966b2..8ee8caf8 100644
--- a/crates/core/src/webpage/adservers.txt
+++ b/crates/core/src/webpage/adservers.txt
@@ -1842,7 +1842,6 @@ gloyah.net,
gmads.net,
gml.email,
go-clicks.de,
-go-mpulse.net,
go-rank.de,
go.dhs.gov,
go.eu.sparkpostmail1.com,
@@ -2435,7 +2434,6 @@ nondescriptstocking.com,
nothingunit.com,
novem.pl,
npttech.com,
-nr-data.net,
nr.mmcdn.com,
nr.static.mmcdn.com,
ns1p.net,