Skip to content

web search done right

Various different optimizations. This commit should have been structured much better.

Mikkel Denker4bba8feb0a83
diff --git a/core/src/collector.rs b/core/src/collector.rs
index 3bafff13..dd5bfd23 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -283,24 +283,15 @@ impl<T: Doc> BucketCollector<T> {
Self {
top_n,
+ documents: MinMaxHeap::with_capacity(config.max_docs_considered + 1),
count: BucketCount::new(config),
- documents: MinMaxHeap::with_capacity(top_n + 1),
}
}
pub fn insert(&mut self, doc: T) {
let mut scored_doc: ScoredDoc<T> = doc.into();
self.count.adjust_score(&mut scored_doc);
-
- if self.documents.len() > self.top_n {
- let mut min_scored_doc = self.documents.peek_min_mut().unwrap();
-
- if scored_doc.adjusted_score > min_scored_doc.adjusted_score {
- *min_scored_doc = scored_doc;
- }
- } else {
- self.documents.push(scored_doc);
- }
+ self.documents.push(scored_doc);
}
fn update_best_doc(&mut self) {
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index 7d36501a..e51937a0 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -18,16 +18,20 @@ pub struct Collector;
impl Collector {
pub fn site_penalty() -> f64 {
- 0.3
+ 0.1
}
pub fn title_penalty() -> f64 {
- 2.0
+ 1.0
}
pub fn url_penalty() -> f64 {
20.0
}
+
+ pub fn max_docs_considered() -> usize {
+ 250_000
+ }
}
pub struct Frontend;
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 8a3a9f84..7228acd7 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -153,6 +153,9 @@ pub struct CollectorConfig {
#[serde(default = "defaults::Collector::url_penalty")]
pub url_penalty: f64,
+
+ #[serde(default = "defaults::Collector::max_docs_considered")]
+ pub max_docs_considered: usize,
}
impl Default for CollectorConfig {
@@ -161,6 +164,7 @@ impl Default for CollectorConfig {
site_penalty: defaults::Collector::site_penalty(),
title_penalty: defaults::Collector::title_penalty(),
url_penalty: defaults::Collector::url_penalty(),
+ max_docs_considered: defaults::Collector::max_docs_considered(),
}
}
}
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index a0ca9579..d857d2cf 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -48,6 +48,9 @@ pub enum Error {
#[error("invalid politeness factor")]
InvalidPolitenessFactor,
+
+ #[error("invalid redirect")]
+ InvalidRedirect,
}
type Result<T> = std::result::Result<T, anyhow::Error>;
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 27fa39d4..913743ec 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -65,10 +65,21 @@ impl WorkerThread {
return Err(Error::InvalidPolitenessFactor.into());
}
+ let mut headers = reqwest::header::HeaderMap::default();
+ headers.insert(
+ reqwest::header::ACCEPT,
+ reqwest::header::HeaderValue::from_static("text/html"),
+ );
+ headers.insert(
+ reqwest::header::ACCEPT_LANGUAGE,
+ reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),
+ );
+
let client = reqwest::Client::builder()
.timeout(timeout)
.connect_timeout(timeout)
.http2_keep_alive_interval(None)
+ .default_headers(headers)
.user_agent(&config.user_agent.full)
.build()?;
@@ -248,71 +259,74 @@ impl WorkerThread {
match fetch {
Ok(datum) => {
if matches!(datum.status_code, 200 | 301 | 302) {
- self.save_datum(datum.clone()).await;
-
- match Html::parse(&datum.body, datum.url.as_str()) {
- Ok(html) => {
- let new_urls = html
- .all_links()
- .into_iter()
- .map(|link| link.destination)
- .filter(|url| {
- !url.path().ends_with(".pdf")
- && !url.path().ends_with(".jpg")
- && !url.path().ends_with(".zip")
- && !url.path().ends_with(".png")
- && !url.path().ends_with(".css")
- && !url.path().ends_with(".js")
- && !url.path().ends_with(".json")
- && !url.path().ends_with(".jsonp")
- && !url.path().ends_with(".woff2")
- && !url.path().ends_with(".woff")
- && !url.path().ends_with(".ttf")
- && !url.path().ends_with(".svg")
- && !url.path().ends_with(".gif")
- && !url.path().ends_with(".jpeg")
- && !url.path().ends_with(".ico")
- && !url.path().ends_with(".mp4")
- && !url.path().ends_with(".mp3")
- && !url.path().ends_with(".avi")
- && !url.path().ends_with(".mov")
- && !url.path().ends_with(".mpeg")
- && !url.path().ends_with(".webm")
- && !url.path().ends_with(".wav")
- && !url.path().ends_with(".flac")
- && !url.path().ends_with(".aac")
- && !url.path().ends_with(".ogg")
- && !url.path().ends_with(".m4a")
- && !url.path().ends_with(".m4v")
- })
- .collect();
-
- let url_res = if datum.status_code == 301
- || datum.status_code == 302
- || url != datum.url
- {
- UrlResponse::Redirected {
- url,
- new_url: datum.url,
+ if datum.status_code == 200 {
+ self.save_datum(datum.clone()).await;
+
+ match Html::parse(&datum.body, datum.url.as_str()) {
+ Ok(html) => {
+ let new_urls = html
+ .all_links()
+ .into_iter()
+ .map(|link| link.destination)
+ .filter(|url| {
+ !url.path().ends_with(".pdf")
+ && !url.path().ends_with(".jpg")
+ && !url.path().ends_with(".zip")
+ && !url.path().ends_with(".png")
+ && !url.path().ends_with(".css")
+ && !url.path().ends_with(".js")
+ && !url.path().ends_with(".json")
+ && !url.path().ends_with(".jsonp")
+ && !url.path().ends_with(".woff2")
+ && !url.path().ends_with(".woff")
+ && !url.path().ends_with(".ttf")
+ && !url.path().ends_with(".svg")
+ && !url.path().ends_with(".gif")
+ && !url.path().ends_with(".jpeg")
+ && !url.path().ends_with(".ico")
+ && !url.path().ends_with(".mp4")
+ && !url.path().ends_with(".mp3")
+ && !url.path().ends_with(".avi")
+ && !url.path().ends_with(".mov")
+ && !url.path().ends_with(".mpeg")
+ && !url.path().ends_with(".webm")
+ && !url.path().ends_with(".wav")
+ && !url.path().ends_with(".flac")
+ && !url.path().ends_with(".aac")
+ && !url.path().ends_with(".ogg")
+ && !url.path().ends_with(".m4a")
+ && !url.path().ends_with(".m4v")
+ })
+ .collect();
+
+ let url_res = UrlResponse::Success { url: datum.url };
+
+ ProcessedUrl {
+ new_urls,
+ response: url_res,
+ fetch_time: Duration::from_millis(datum.fetch_time_ms),
}
- } else {
- UrlResponse::Success { url: datum.url }
- };
-
- ProcessedUrl {
- new_urls,
- response: url_res,
- fetch_time: Duration::from_millis(datum.fetch_time_ms),
}
+ Err(_) => ProcessedUrl {
+ new_urls: Vec::new(),
+ response: UrlResponse::Failed {
+ url,
+ status_code: None,
+ },
+ fetch_time: Duration::from_millis(datum.fetch_time_ms),
+ },
}
- Err(_) => ProcessedUrl {
+ } else {
+ let url_res = UrlResponse::Redirected {
+ url,
+ new_url: datum.url,
+ };
+
+ ProcessedUrl {
new_urls: Vec::new(),
- response: UrlResponse::Failed {
- url,
- status_code: None,
- },
+ response: url_res,
fetch_time: Duration::from_millis(datum.fetch_time_ms),
- },
+ }
}
} else {
if datum.status_code == 429 {
@@ -356,6 +370,10 @@ impl WorkerThread {
}
async fn save_datum(&self, datum: CrawlDatum) {
+ if datum.status_code != 200 {
+ return;
+ }
+
self.writer.write(datum).await.ok();
}
@@ -411,6 +429,27 @@ impl WorkerThread {
let status_code = res.status().as_u16();
+ if status_code == 301 || status_code == 302 {
+ let location = res
+ .headers()
+ .get("location")
+ .ok_or(Error::InvalidRedirect)?;
+
+ let location = location.to_str().map_err(|_| Error::InvalidRedirect)?;
+
+ let url = Url::parse(location)
+ .or_else(|_| url.join(location))
+ .map_err(|_| Error::InvalidRedirect)?;
+
+ return Ok(CrawlDatum {
+ url,
+ status_code,
+ headers,
+ body: String::new(),
+ fetch_time_ms: fetch_time.as_millis() as u64,
+ });
+ }
+
let res_url = res.url().clone();
let body = res.text().await?;
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 16070e4a..66bef9d3 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -182,7 +182,7 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {
loop {
if let Err(e) = server.accept().await {
- tracing::error!("{}", e);
+ tracing::error!("{:?}", e);
}
}
}
diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rs
index eeaba9fa..0f316e02 100644
--- a/core/src/entrypoint/webgraph_server.rs
+++ b/core/src/entrypoint/webgraph_server.rs
@@ -152,7 +152,7 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
loop {
if let Err(e) = server.accept().await {
- tracing::error!("{}", e);
+ tracing::error!("{:?}", e);
}
}
}
diff --git a/core/src/enum_map.rs b/core/src/enum_map.rs
index 92fa9feb..07d06c2d 100644
--- a/core/src/enum_map.rs
+++ b/core/src/enum_map.rs
@@ -72,6 +72,10 @@ where
self.inner.iter_mut().filter_map(|value| value.as_mut())
}
+ pub fn values(&self) -> impl Iterator<Item = &V> {
+ self.inner.iter().filter_map(|value| value.as_ref())
+ }
+
pub fn get_mut(&mut self, key: K) -> Option<&mut V> {
let key = key.into();
if key >= self.inner.len() {
diff --git a/core/src/index.rs b/core/src/index.rs
index cd235326..1003d183 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -336,6 +336,6 @@ mod tests {
.unwrap()
.get(&crate::ranking::Signal::Bm25)
.unwrap())
- .all(|&v| v > 0.0));
+ .all(|&v| v.value > 0.0));
}
}
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 1a4ee3e4..75395032 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -339,6 +339,8 @@ fn parse_term(term: &str) -> Box<Term> {
#[allow(clippy::vec_box)]
pub fn parse(query: &str) -> Vec<Box<Term>> {
+ let query = query.to_lowercase();
+
let mut res = Vec::new();
let mut cur_term_begin = 0;
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 0f7b4958..9f95bae7 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -41,6 +41,7 @@ pub struct Scorer {
disliked: Vec<NodeScorer>,
vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,
cache: DashMap<NodeID, f64>,
+ normalized: bool,
}
#[derive(Debug)]
@@ -74,22 +75,29 @@ impl NodeScorer {
impl Scorer {
fn calculate_score(&self, node: &NodeID) -> f64 {
- match self.vectors.get(node) {
- Some(vec) => ((self.disliked.len() as f64)
- + (self
- .liked
- .iter()
- .map(|liked| liked.sim(node, vec))
- .sum::<f64>()
- - self
- .disliked
+ let s = match self.vectors.get(node) {
+ Some(vec) => {
+ (self.disliked.len() as f64)
+ + (self
+ .liked
.iter()
- .map(|disliked| disliked.sim(node, vec))
- .sum::<f64>())
- / self.liked.len().max(1) as f64)
- .max(0.0),
+ .map(|liked| liked.sim(node, vec))
+ .sum::<f64>()
+ - self
+ .disliked
+ .iter()
+ .map(|disliked| disliked.sim(node, vec))
+ .sum::<f64>())
+ }
None => 0.0,
+ };
+
+ if self.normalized {
+ s / self.liked.len().max(1) as f64
+ } else {
+ s
}
+ .max(0.0)
}
pub fn score(&self, node: &NodeID) -> f64 {
if let Some(cached) = self.cache.get(node) {
@@ -199,7 +207,12 @@ impl InboundSimilarity {
}
}
- pub fn scorer(&self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer {
+ pub fn scorer(
+ &self,
+ liked_sites: &[NodeID],
+ disliked_sites: &[NodeID],
+ normalized: bool,
+ ) -> Scorer {
let liked: Vec<_> = liked_sites
.iter()
.filter_map(|id| self.vectors.get(id).cloned().map(|vec| (id, vec)))
@@ -217,6 +230,7 @@ impl InboundSimilarity {
disliked,
vectors: self.vectors.clone(),
cache: DashMap::new(),
+ normalized,
}
}
@@ -297,7 +311,7 @@ mod tests {
let inbound =
InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);
- let scorer = inbound.scorer(&[Node::from("b.com").id()], &[]);
+ let scorer = inbound.scorer(&[Node::from("b.com").id()], &[], false);
let e = Node::from("e.com").id();
let d = Node::from("d.com").id();
diff --git a/core/src/ranking/initial.rs b/core/src/ranking/initial.rs
index d360b339..8b06e713 100644
--- a/core/src/ranking/initial.rs
+++ b/core/src/ranking/initial.rs
@@ -80,7 +80,7 @@ impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {
.aggregator
.compute_signals(doc)
.flatten()
- .map(|computed| computed.coefficient * computed.value)
+ .map(|computed| computed.score.coefficient * computed.score.value)
.sum();
if let Some(boost) = self.aggregator.boosts(doc) {
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index b775bb77..96a2fde6 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -647,6 +647,7 @@ mod tests {
}
#[test]
+ #[ignore = "proximity signals are currently disabled"]
fn term_proximity_ranking() {
let mut index = Index::temporary().expect("Unable to open index");
let mut w = Webpage {
diff --git a/core/src/ranking/models/lambdamart.rs b/core/src/ranking/models/lambdamart.rs
index 87555e51..94c9526e 100644
--- a/core/src/ranking/models/lambdamart.rs
+++ b/core/src/ranking/models/lambdamart.rs
@@ -49,6 +49,16 @@ enum NodeOrLeaf {
Leaf(usize),
}
+pub trait AsValue {
+ fn as_value(&self) -> f64;
+}
+
+impl AsValue for f64 {
+ fn as_value(&self) -> f64 {
+ *self
+ }
+}
+
#[derive(Debug)]
struct Node {
threshold: f64,
@@ -59,9 +69,9 @@ struct Node {
}
impl Node {
- fn next(&self, features: &EnumMap<Signal, f64>) -> Option<&NodeOrLeaf> {
+ fn next<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> Option<&NodeOrLeaf> {
self.feature.and_then(|feature| {
- let value = features.get(feature).copied().unwrap_or(0.0);
+ let value = features.get(feature).map(|v| v.as_value()).unwrap_or(0.0);
if value <= self.threshold {
self.left.as_ref()
} else {
@@ -205,7 +215,7 @@ impl Tree {
Ok(Self { nodes })
}
- fn predict(&self, features: &EnumMap<Signal, f64>) -> Result<f64> {
+ fn predict<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> Result<f64> {
let mut node = &self.nodes[0];
while let Some(next) = node.next(features) {
node = match next {
@@ -255,7 +265,7 @@ impl LambdaMART {
Ok(Self { tree })
}
- pub fn predict(&self, features: &EnumMap<Signal, f64>) -> f64 {
+ pub fn predict<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> f64 {
self.tree.predict(features).unwrap()
}
}
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 2ac97ecc..da791350 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -98,7 +98,7 @@ mod tests {
.unwrap(),
backlinks: vec![],
host_centrality: 1.0,
- fetch_time_ms: 50,
+ fetch_time_ms: 500,
pre_computed_score: 0.0,
page_centrality: 0.0,
@@ -127,7 +127,7 @@ mod tests {
.unwrap(),
backlinks: vec![],
host_centrality: 1.0,
- fetch_time_ms: 49,
+ fetch_time_ms: 499,
page_centrality: 0.0,
pre_computed_score: 0.0,
@@ -174,7 +174,7 @@ mod tests {
query: "test".to_string(),
site_rankings: Some(SiteRankings {
liked: vec!["www.first.com".to_string()],
- disliked: vec!["www.third.com".to_string()],
+ disliked: vec![],
blocked: vec![],
}),
..Default::default()
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 3bc624e9..16cfc426 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -28,8 +28,11 @@ use crate::{
};
use super::{
- models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},
- Signal, SignalAggregator, SignalCoefficient,
+ models::{
+ cross_encoder::CrossEncoder,
+ lambdamart::{self, LambdaMART},
+ },
+ Signal, SignalAggregator, SignalCoefficient, SignalScore,
};
pub trait AsRankingWebsite: Clone {
@@ -64,10 +67,16 @@ impl AsRankingWebsite for RankingWebsite {
}
}
+impl lambdamart::AsValue for SignalScore {
+ fn as_value(&self) -> f64 {
+ self.value
+ }
+}
+
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RankingWebsite {
pub pointer: WebsitePointer,
- pub signals: EnumMap<Signal, f64>,
+ pub signals: EnumMap<Signal, SignalScore>,
pub title: Option<String>,
pub clean_body: Option<String>,
pub optic_boost: Option<f64>,
@@ -85,15 +94,11 @@ impl RankingWebsite {
pointer: pointer.clone(),
};
- let mut total = 0.0;
for computed_signal in aggregator.compute_signals(pointer.address.doc_id).flatten() {
res.signals
- .insert(computed_signal.signal, computed_signal.value);
- total += computed_signal.value * computed_signal.coefficient;
+ .insert(computed_signal.signal, computed_signal.score);
}
- res.signals.insert(Signal::LinearRegression, total);
-
if let Some(boost) = aggregator.boosts(pointer.address.doc_id) {
res.optic_boost = Some(boost);
}
@@ -140,9 +145,22 @@ impl<M: CrossEncoder> ReRanker<M> {
for (website, score) in websites.iter_mut().zip(scores) {
let website = website.as_mut_ranking();
- website.signals.insert(Signal::CrossEncoder, score);
+ website.signals.insert(
+ Signal::CrossEncoder,
+ SignalScore {
+ coefficient: self.crossencoder_coeff(),
+ value: score,
+ },
+ );
}
}
+
+ fn crossencoder_coeff(&self) -> f64 {
+ self.signal_coefficients
+ .as_ref()
+ .and_then(|coeffs| coeffs.get(&Signal::CrossEncoder))
+ .unwrap_or(Signal::CrossEncoder.default_coefficient())
+ }
}
impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
@@ -151,22 +169,11 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
for website in websites.iter_mut() {
let website = website.as_mut_ranking();
- let score = calculate_score(
+ website.score = calculate_score(
&self.lambda_mart,
&self.signal_coefficients,
&website.signals,
);
-
- if let Some(coeffs) = &self.signal_coefficients {
- if let Some(coeff) = coeffs.get(&Signal::CrossEncoder) {
- website.score =
- score + coeff * website.signals.get(Signal::CrossEncoder).unwrap();
- } else {
- website.score = score;
- }
- } else {
- website.score = score;
- }
}
}
@@ -180,7 +187,7 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
fn calculate_score(
model: &Option<Arc<LambdaMART>>,
signal_coefficients: &Option<SignalCoefficient>,
- signals: &EnumMap<Signal, f64>,
+ signals: &EnumMap<Signal, SignalScore>,
) -> f64 {
let lambda_score = match model {
Some(model) => match signal_coefficients {
@@ -188,9 +195,9 @@ fn calculate_score(
Some(coeff) => {
if coeff == 0.0 {
signals
- .get(Signal::LinearRegression)
- .copied()
- .unwrap_or(0.0)
+ .values()
+ .map(|score| score.coefficient * score.value)
+ .sum()
} else {
coeff * model.predict(signals)
}
@@ -200,9 +207,9 @@ fn calculate_score(
None => Signal::LambdaMART.default_coefficient() * model.predict(signals),
},
None => signals
- .get(Signal::LinearRegression)
- .copied()
- .unwrap_or(0.0),
+ .values()
+ .map(|score| score.coefficient * score.value)
+ .sum(),
};
lambda_score
@@ -388,7 +395,13 @@ mod tests {
(0..n)
.map(|i| -> RankingWebsite {
let mut signals = EnumMap::new();
- signals.insert(Signal::LinearRegression, 1.0 / i as f64);
+ signals.insert(
+ Signal::Bm25,
+ SignalScore {
+ coefficient: 1.0,
+ value: 1.0 / i as f64,
+ },
+ );
RankingWebsite {
pointer: WebsitePointer {
score: Score { total: 0.0 },
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 1d5ea5f5..ab2a76de 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -27,6 +27,7 @@ use crate::{
};
use optics::ast::RankingTarget;
use optics::Optic;
+use serde::{Deserialize, Serialize};
use std::str::FromStr;
use std::sync::Arc;
use tantivy::fieldnorm::FieldNormReader;
@@ -47,6 +48,8 @@ use super::bm25::Bm25Weight;
use super::models::linear::LinearRegression;
use super::{inbound_similarity, query_centrality};
+const PROXIMITY_ENABLED: bool = false;
+
#[derive(Debug, Error)]
pub enum Error {
#[error("unknown signal: {0}")]
@@ -95,16 +98,31 @@ pub enum Signal {
Bm25BacklinkText,
#[serde(rename = "bm25_description")]
Bm25Description,
+
+ /// All proximity signals are disabled for now.
+ /// We need to figure out how to make them faster.
#[serde(rename = "proximity_slop_0")]
ProximitySlop0,
+
+ /// All proximity signals are disabled for now.
+ /// We need to figure out how to make them faster.
#[serde(rename = "proximity_slop_1")]
ProximitySlop1,
+
+ /// All proximity signals are disabled for now.
+ /// We need to figure out how to make them faster.
#[serde(rename = "proximity_slop_2")]
ProximitySlop2,
+ /// All proximity signals are disabled for now.
+ /// We need to figure out how to make them faster.
#[serde(rename = "proximity_slop_4")]
ProximitySlop4,
+
+ /// All proximity signals are disabled for now.
+ /// We need to figure out how to make them faster.
#[serde(rename = "proximity_slop_8")]
ProximitySlop8,
+
#[serde(rename = "cross_encoder")]
CrossEncoder,
#[serde(rename = "host_centrality")]
@@ -127,8 +145,6 @@ pub enum Signal {
InboundSimilarity,
#[serde(rename = "lambda_mart")]
LambdaMART,
- #[serde(rename = "linear_regression")]
- LinearRegression,
#[serde(rename = "url_digits")]
UrlDigits,
#[serde(rename = "url_slashes")]
@@ -249,8 +265,8 @@ impl Signal {
pub fn default_coefficient(&self) -> f64 {
match self {
- Signal::Bm25 => 0.0005,
- Signal::Bm25Title => 0.005,
+ Signal::Bm25 => 0.0002,
+ Signal::Bm25Title => 0.004,
Signal::Bm25TitleBigrams => 0.005,
Signal::Bm25TitleTrigrams => 0.005,
Signal::Bm25CleanBody => 0.005,
@@ -264,28 +280,27 @@ impl Signal {
Signal::Bm25Domain => 0.00005,
Signal::Bm25SiteNoTokenizer => 0.00005,
Signal::Bm25DomainNoTokenizer => 0.00005,
- Signal::Bm25DomainIfHomepage => 0.00005,
- Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0001,
- Signal::Bm25TitleIfHomepage => 0.00001,
+ Signal::Bm25DomainIfHomepage => 0.0002,
+ Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0002,
+ Signal::Bm25TitleIfHomepage => 0.00002,
Signal::Bm25BacklinkText => 0.001,
- Signal::Bm25Description => 5.99679347128802e-05,
+ Signal::Bm25Description => 0.00001,
Signal::ProximitySlop0 => 0.01,
Signal::ProximitySlop1 => 0.01,
Signal::ProximitySlop2 => 0.01,
Signal::ProximitySlop4 => 0.01,
Signal::ProximitySlop8 => 0.01,
- Signal::CrossEncoder => 0.16801589371906178,
+ Signal::CrossEncoder => 0.17,
Signal::HostCentrality => 1.0,
- Signal::PageCentrality => 2.0,
+ Signal::PageCentrality => 4.0,
Signal::QueryCentrality => 0.0,
- Signal::IsHomepage => 0.01,
+ Signal::IsHomepage => 0.0005,
Signal::FetchTimeMs => 0.001,
Signal::UpdateTimestamp => 0.001,
- Signal::TrackerScore => 0.019542120533715634,
- Signal::Region => 0.06437975586036043,
- Signal::InboundSimilarity => 10.0,
+ Signal::TrackerScore => 0.02,
+ Signal::Region => 0.005,
+ Signal::InboundSimilarity => 1.0,
Signal::LambdaMART => 10.0,
- Signal::LinearRegression => 1.0,
Signal::UrlSlashes => 0.01,
Signal::UrlDigits => 0.01,
}
@@ -333,6 +348,11 @@ impl Signal {
signal_aggregator: &mut SignalAggregator,
doc: DocId,
) -> Option<ComputedSignal> {
+ let coefficient = signal_aggregator.coefficient(&self);
+ if coefficient == 0.0 {
+ return None;
+ }
+
let value: Option<f64> = match self {
Signal::HostCentrality | Signal::PageCentrality => {
let field_value: Option<u64> = self
@@ -462,13 +482,11 @@ impl Signal {
}
Signal::CrossEncoder => None, // this is calculated in a later step
Signal::LambdaMART => None,
- Signal::LinearRegression => None, // this is calculated just before input to lambdamart
};
value.map(|value| ComputedSignal {
signal: self,
- coefficient: signal_aggregator.coefficient(&self),
- value,
+ score: SignalScore { coefficient, value },
})
}
@@ -566,7 +584,6 @@ impl Signal {
| Signal::CrossEncoder
| Signal::InboundSimilarity
| Signal::LambdaMART
- | Signal::LinearRegression
| Signal::QueryCentrality => {
tracing::error!("signal {self:?} cannot be precomputed");
None
@@ -575,8 +592,10 @@ impl Signal {
value.map(|value| ComputedSignal {
signal: self,
- coefficient: signal_aggregator.coefficient(&self),
- value,
+ score: SignalScore {
+ coefficient: signal_aggregator.coefficient(&self),
+ value,
+ },
})
}
@@ -903,6 +922,10 @@ impl SignalAggregator {
tv_searcher: &tantivy::Searcher,
segment_reader: &tantivy::SegmentReader,
) -> EnumMap<Signal, ProximityScorer> {
+ if !PROXIMITY_ENABLED {
+ return EnumMap::new();
+ }
+
let mut proximity_scorers = EnumMap::new();
let proximity_fields = [TextField::Title, TextField::CleanBody];
let schema = tv_searcher.schema();
@@ -1060,7 +1083,7 @@ impl SignalAggregator {
ALL_SIGNALS
.into_iter()
.filter_map(|signal| signal.precompute(self, webpage))
- .map(|computed| computed.coefficient * computed.value)
+ .map(|computed| computed.score.coefficient * computed.score.value)
.sum()
}
@@ -1080,6 +1103,11 @@ impl SignalAggregator {
#[derive(Debug, Clone, Copy)]
pub struct ComputedSignal {
pub signal: Signal,
+ pub score: SignalScore,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct SignalScore {
pub coefficient: f64,
pub value: f64,
}
@@ -1095,7 +1123,7 @@ mod tests {
use super::*;
- fn signals_for_query(searcher: &LocalSearcher, q: &str) -> HashMap<Signal, f64> {
+ fn signals_for_query(searcher: &LocalSearcher, q: &str) -> HashMap<Signal, SignalScore> {
let res = searcher
.search(&SearchQuery {
query: q.to_string(),
@@ -1106,6 +1134,7 @@ mod tests {
res[0].ranking_signals.clone().unwrap()
}
#[test]
+ #[ignore = "proximity signals are currently disabled"]
fn test_proximity() {
let mut index = Index::temporary().expect("Unable to open index");
@@ -1138,45 +1167,45 @@ mod tests {
let searcher = LocalSearcher::new(index);
let signals = signals_for_query(&searcher, "a b");
- assert!(signals[&Signal::ProximitySlop8] > 0.0);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop1]);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop0]);
+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop1].value);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop0].value);
let signals = signals_for_query(&searcher, "a c");
- assert!(signals[&Signal::ProximitySlop8] > 0.0);
- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop1]);
+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop1].value);
let signals = signals_for_query(&searcher, "a d");
- assert!(signals[&Signal::ProximitySlop8] > 0.0);
- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop1], 0.0);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);
+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);
let signals = signals_for_query(&searcher, "a e");
- assert!(signals[&Signal::ProximitySlop8] > 0.0);
- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop1], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop2], 0.0);
- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);
+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);
+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);
let signals = signals_for_query(&searcher, "a g");
- assert!(signals[&Signal::ProximitySlop8] > 0.0);
- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop1], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop2], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop4], 0.0);
+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop4].value, 0.0);
let signals = signals_for_query(&searcher, "a k");
- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop1], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop2], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop4], 0.0);
- assert_eq!(signals[&Signal::ProximitySlop8], 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop4].value, 0.0);
+ assert_eq!(signals[&Signal::ProximitySlop8].value, 0.0);
}
}
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index 60911a05..8e26d3b4 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -26,7 +26,7 @@ use url::Url;
use crate::{
inverted_index::RetrievedWebpage,
- ranking::Signal,
+ ranking::{Signal, SignalScore},
spell::{self, CorrectionTerm},
webpage::url_ext::UrlExt,
};
@@ -168,7 +168,7 @@ pub struct DisplayedWebpage {
pub pretty_url: String,
pub snippet: Snippet,
pub body: String,
- pub ranking_signals: Option<HashMap<Signal, f64>>,
+ pub ranking_signals: Option<HashMap<Signal, SignalScore>>,
}
#[derive(Debug, Serialize, Deserialize)]
diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rs
index 51c5b09f..43f49995 100644
--- a/core/src/searcher/frontend.rs
+++ b/core/src/searcher/frontend.rs
@@ -302,10 +302,9 @@ impl FrontendSearcher {
let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());
for signal in ALL_SIGNALS {
- signals.insert(
- signal,
- pointer.website.signals.get(signal).copied().unwrap_or(0.0),
- );
+ if let Some(signal_value) = pointer.website.signals.get(signal) {
+ signals.insert(signal, *signal_value);
+ }
}
website.ranking_signals = Some(signals);
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index c9156299..369dcce2 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -141,7 +141,7 @@ impl LocalSearcher {
if query_centrality_coeff > 0.0 {
if let Some(centrality_store) = self.centrality_store.as_ref() {
ranker = ranker
- .with_max_docs(10_000, self.index.num_segments())
+ .with_max_docs(1_000, self.index.num_segments())
.with_num_results(100);
let top_host_nodes =
@@ -149,9 +149,10 @@ impl LocalSearcher {
.top_nodes(query, ctx, ranker.collector(ctx.clone()))?;
if !top_host_nodes.is_empty() {
- let inbound = centrality_store
- .inbound_similarity
- .scorer(&top_host_nodes, &[]);
+ let inbound =
+ centrality_store
+ .inbound_similarity
+ .scorer(&top_host_nodes, &[], false);
let query_centrality = query_centrality::Scorer::new(inbound);
@@ -161,7 +162,10 @@ impl LocalSearcher {
}
Ok(ranker
- .with_max_docs(250_000, self.index.num_segments())
+ .with_max_docs(
+ self.collector_config.max_docs_considered,
+ self.index.num_segments(),
+ )
.with_num_results(query.num_results())
.with_offset(query.offset()))
}
@@ -199,11 +203,15 @@ impl LocalSearcher {
.map(|node| node.id())
.collect();
- aggregator.set_inbound_similarity(
- store
- .inbound_similarity
- .scorer(&liked_sites, &disliked_sites),
- )
+ let mut scorer = store
+ .inbound_similarity
+ .scorer(&liked_sites, &disliked_sites, false);
+
+ if liked_sites.len() + disliked_sites.len() > 1 {
+ scorer.set_self_score(0.0);
+ }
+
+ aggregator.set_inbound_similarity(scorer);
}
aggregator.set_region_count(self.index.region_count.clone());
@@ -357,7 +365,9 @@ impl LocalSearcher {
let mut ranking_signals = HashMap::new();
for signal in ALL_SIGNALS {
- ranking_signals.insert(signal, *ranking.signals.get(signal).unwrap_or(&0.0));
+ if let Some(score) = ranking.signals.get(signal) {
+ ranking_signals.insert(signal, *score);
+ }
}
webpage.ranking_signals = Some(ranking_signals);
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
index 2f9009c5..d49a564a 100644
--- a/core/src/similar_sites.rs
+++ b/core/src/similar_sites.rs
@@ -81,7 +81,7 @@ impl SimilarSitesFinder {
.map(|node| node.id())
.collect();
- let scorer = self.inbound_similarity.scorer(&nodes, &[]);
+ let scorer = self.inbound_similarity.scorer(&nodes, &[], true);
let mut backlinks: IntMap<NodeID, f64> = IntMap::new();
diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rs
index d31fe3d9..b6091e45 100644
--- a/core/src/spell/mod.rs
+++ b/core/src/spell/mod.rs
@@ -118,7 +118,7 @@ impl Spell {
info!("Building spell correction dictionary");
let searcher = index.inverted_index.tv_searcher();
let schema = searcher.schema();
- let mut dict = DictionaryBuilder::new(1_000_000);
+ let mut dict = DictionaryBuilder::new(10_000);
#[allow(unused_assignments, unused_mut)]
let mut limit_terms: Option<usize> = None;
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
index 2b2f31ea..df57008c 100644
--- a/core/src/tokenizer.rs
+++ b/core/src/tokenizer.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{array, ops::Range, str::CharIndices};
+use std::{array, str::CharIndices};
use logos::{Lexer, Logos};
use tantivy::tokenizer::{
@@ -170,9 +170,7 @@ impl tantivy::tokenizer::Tokenizer for Normal {
impl tantivy::tokenizer::Tokenizer for Stemmed {
fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
- let analyzer = TextAnalyzer::from(Simple)
- .filter(LowerCaser)
- .filter(StopWordFilter::remove(vec![]));
+ let analyzer = TextAnalyzer::from(Simple).filter(LowerCaser);
let lang = match self.force_language {
Some(lang) => Some(lang),
@@ -195,7 +193,7 @@ impl tantivy::tokenizer::Tokenizer for Identity {
impl tantivy::tokenizer::Tokenizer for BigramTokenizer {
fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
let inner = Normal::default().token_stream(text);
- let stream: NGramTokenStream<2> = NGramTokenStream::new(text, inner);
+ let stream: NGramTokenStream<2> = NGramTokenStream::new(inner);
BoxTokenStream::from(stream)
}
}
@@ -203,7 +201,7 @@ impl tantivy::tokenizer::Tokenizer for BigramTokenizer {
impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {
fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {
let inner = Normal::default().token_stream(text);
- let stream: NGramTokenStream<3> = NGramTokenStream::new(text, inner);
+ let stream: NGramTokenStream<3> = NGramTokenStream::new(inner);
BoxTokenStream::from(stream)
}
}
@@ -306,61 +304,49 @@ impl<'a> tantivy::tokenizer::TokenStream for SimpleTokenStream<'a> {
}
}
-#[derive(Default)]
-struct TokenRef {
- range: Range<usize>,
- position: usize,
-}
-
-impl From<&tantivy::tokenizer::Token> for TokenRef {
- fn from(token: &tantivy::tokenizer::Token) -> Self {
- Self {
- range: token.offset_from..token.offset_to,
- position: token.position,
- }
- }
-}
-
struct NGramTokenStream<'a, const N: usize> {
inner: BoxTokenStream<'a>,
- text: &'a str,
token: tantivy::tokenizer::Token,
- token_refs: [TokenRef; N],
+ token_window: [tantivy::tokenizer::Token; N],
is_first: bool,
next_pos: usize,
}
impl<'a, const N: usize> NGramTokenStream<'a, N> {
- fn new(text: &'a str, inner: BoxTokenStream<'a>) -> Self {
+ fn new(inner: BoxTokenStream<'a>) -> Self {
Self {
inner,
- text,
token: tantivy::tokenizer::Token::default(),
- token_refs: array::from_fn(|_| TokenRef::default()),
+ token_window: array::from_fn(|_| tantivy::tokenizer::Token::default()),
is_first: true,
next_pos: 0,
}
}
}
+fn reuse_token_alloc(token: &mut tantivy::tokenizer::Token, new_token: &tantivy::tokenizer::Token) {
+ token.text.clear();
+ token.text += new_token.text.as_str();
+ token.offset_from = new_token.offset_from;
+ token.offset_to = new_token.offset_to;
+ token.position = new_token.position;
+ token.position_length = new_token.position_length;
+}
+
impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 1> {
fn advance(&mut self) -> bool {
self.is_first = false;
let res = self.inner.advance();
+ self.token_window[0].text.clear();
+
if res {
- self.token_refs[0].range = self.inner.token().offset_from..self.inner.token().offset_to;
- self.token_refs[0].position = self.next_pos;
+ self.token_window[0].text += self.inner.token().text.as_str();
self.next_pos += 1;
}
- self.token = tantivy::tokenizer::Token {
- offset_from: self.token_refs[0].range.start,
- offset_to: self.token_refs[0].range.end,
- position: self.token_refs[0].position,
- text: self.text[self.token_refs[0].range.clone()].to_string(),
- position_length: 1,
- };
+ reuse_token_alloc(&mut self.token, &self.token_window[0]);
+ self.token.position_length = 1;
res
}
@@ -380,29 +366,36 @@ impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 2> {
if !self.inner.advance() {
return false;
}
-
- self.token_refs[0] = self.inner.token().into();
+ reuse_token_alloc(&mut self.token_window[0], self.inner.token());
if !self.inner.advance() {
return false;
}
- self.token_refs[1] = self.inner.token().into();
+ reuse_token_alloc(&mut self.token_window[1], self.inner.token());
} else {
if !self.inner.advance() {
return false;
}
- self.token_refs.rotate_left(1);
- self.token_refs[1] = self.inner.token().into();
+ self.token_window.rotate_left(1);
+ reuse_token_alloc(&mut self.token_window[1], self.inner.token());
}
- self.is_first = false;
+ self.is_first = false;
self.next_pos += 1;
self.token.position = self.next_pos;
- self.token.text =
- self.text[self.token_refs[0].range.start..self.token_refs[1].range.end].to_string();
+ self.token.offset_from = self.token_window[0].offset_from;
+ self.token.offset_to = self.token_window[1].offset_to;
self.token.position_length = 2;
+
+ self.token.text.clear();
+ for token in &self.token_window {
+ self.token.text += token.text.as_str();
+ self.token.text += " ";
+ }
+ self.token.text.pop();
+
true
}
@@ -422,33 +415,41 @@ impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 3> {
return false;
}
- self.token_refs[0] = self.inner.token().into();
+ reuse_token_alloc(&mut self.token_window[0], self.inner.token());
if !self.inner.advance() {
return false;
}
- self.token_refs[1] = self.inner.token().into();
+ reuse_token_alloc(&mut self.token_window[1], self.inner.token());
if !self.inner.advance() {
return false;
}
- self.token_refs[2] = self.inner.token().into();
+ reuse_token_alloc(&mut self.token_window[2], self.inner.token());
} else {
if !self.inner.advance() {
return false;
}
- self.token_refs.rotate_left(1);
- self.token_refs[2] = self.inner.token().into();
+ self.token_window.rotate_left(1);
+ reuse_token_alloc(&mut self.token_window[2], self.inner.token());
}
- self.is_first = false;
+ self.is_first = false;
self.next_pos += 1;
self.token.position = self.next_pos;
- self.token.text =
- self.text[self.token_refs[0].range.start..self.token_refs[2].range.end].to_string();
+ self.token.offset_from = self.token_window[0].offset_from;
+ self.token.offset_to = self.token_window[2].offset_to;
self.token.position_length = 3;
+
+ self.token.text.clear();
+ for token in &self.token_window {
+ self.token.text += token.text.as_str();
+ self.token.text += " ";
+ }
+ self.token.text.pop();
+
true
}
@@ -906,6 +907,11 @@ key1.key2="this\" is @ a # test""#;
"a test".to_string()
]
);
+
+ assert_eq!(
+ tokenize_bigram("this.is"),
+ vec!["this .".to_string(), ". is".to_string()]
+ );
}
#[test]
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 2dff6f90..a43613ad 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -26,6 +26,9 @@ use std::time::Duration;
use flate2::read::MultiGzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
+#[cfg(test)]
+use proptest_derive::Arbitrary;
+
use tracing::{debug, trace};
pub struct WarcFile {
@@ -205,6 +208,7 @@ struct RawWarcRecord {
}
#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]
pub struct WarcRecord {
pub request: Request,
pub response: Response,
@@ -212,6 +216,7 @@ pub struct WarcRecord {
}
#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]
pub struct Request {
// WARC-Target-URI
pub url: String,
@@ -230,6 +235,7 @@ impl Request {
}
#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]
pub struct Response {
pub body: String,
pub payload_type: Option<String>,
@@ -251,6 +257,7 @@ impl Response {
}
#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]
pub struct Metadata {
// fetchTimeMs
pub fetch_time_ms: usize,
@@ -308,14 +315,23 @@ impl<R: Read> RecordIterator<R> {
return Some(Err(io.into()));
}
- rtrim(&mut line_buf);
-
if &line_buf == "\r\n" || line_buf.is_empty() {
// end of header
break;
}
if let Some(semi) = line_buf.find(':') {
- let value = line_buf.split_off(semi + 1).trim().to_string();
+ let mut value = line_buf.split_off(semi + 1).to_string();
+
+ if let Some(stripped) = value.strip_suffix("\r\n") {
+ value = stripped.to_string();
+ } else if let Some(stripped) = value.strip_suffix('\n') {
+ value = stripped.to_string();
+ }
+
+ if let Some(stripped) = value.strip_prefix(' ') {
+ value = stripped.to_string();
+ }
+
line_buf.pop(); // remove colon
let key = line_buf;
@@ -370,15 +386,9 @@ impl<R: Read> Iterator for RecordIterator<R> {
}
self.num_reads += 1;
- let items = [self.next_raw(), self.next_raw(), self.next_raw()];
-
- let mut response = None;
let mut request = None;
- let mut metadata = None;
-
- for item in items {
- let item = item?;
+ while let Some(item) = self.next_raw() {
if item.is_err() {
return Some(Err(item.err().unwrap()));
}
@@ -386,28 +396,70 @@ impl<R: Read> Iterator for RecordIterator<R> {
let item = item.unwrap();
if let Some(warc_type) = item.header.get("WARC-TYPE") {
- match warc_type.as_str() {
- "request" => request = Some(Request::from_raw(item)),
- "response" => response = Some(Response::from_raw(item)),
- "metadata" => metadata = Some(Metadata::from_raw(item)),
- _ => {
- return Some(Err(Error::WarcParse("Unsupported WARC type").into()));
- }
+ if warc_type.as_str() == "request" {
+ request = Some(Request::from_raw(item));
+ break;
}
}
}
- if request.is_none() || response.is_none() || metadata.is_none() {
- return Some(Err(Error::WarcParse(
- "Request, response or metadata not found",
- )
- .into()));
+ let request = request?;
+
+ // next 2 should be response and metadata
+ let response = self.next_raw()?;
+
+ if response.is_err() {
+ return Some(Err(response.err().unwrap()));
}
- let request = request.unwrap();
let response = response.unwrap();
+
+ match response.header.get("WARC-TYPE") {
+ Some(warc_type) => {
+ if warc_type.as_str() != "response" {
+ return Some(Err(Error::WarcParse(
+ "Expected response, got something else",
+ )
+ .into()));
+ }
+ }
+ None => {
+ return Some(Err(Error::WarcParse(
+ "Expected response, got something else",
+ )
+ .into()));
+ }
+ }
+
+ let response = Response::from_raw(response);
+
+ let metadata = self.next_raw()?;
+
+ if metadata.is_err() {
+ return Some(Err(metadata.err().unwrap()));
+ }
+
let metadata = metadata.unwrap();
+ match metadata.header.get("WARC-TYPE") {
+ Some(warc_type) => {
+ if warc_type.as_str() != "metadata" {
+ return Some(Err(Error::WarcParse(
+ "Expected metadata, got something else",
+ )
+ .into()));
+ }
+ }
+ None => {
+ return Some(Err(Error::WarcParse(
+ "Expected metadata, got something else",
+ )
+ .into()));
+ }
+ }
+
+ let metadata = Metadata::from_raw(metadata);
+
if request.is_err() || response.is_err() || metadata.is_err() {
return Some(Err(Error::WarcParse(
"Request, response or metadata is error",
@@ -534,6 +586,7 @@ mod tests {
use super::*;
use flate2::write::GzEncoder;
use flate2::Compression;
+ use proptest::prelude::*;
use std::io::Write;
#[test]
@@ -689,4 +742,22 @@ mod tests {
assert_eq!(&records[0].response.body, body);
assert_eq!(records[0].metadata.fetch_time_ms, 0);
}
+
+ proptest! {
+ #[test]
+ fn write_read_invariant_prop(records: Vec<WarcRecord>) {
+ let mut writer = WarcWriter::new();
+ for record in records.iter() {
+ writer.write(record).unwrap();
+ }
+ let compressed = writer.finish().unwrap();
+
+ let read_records: Vec<WarcRecord> = WarcFile::new(compressed)
+ .records()
+ .map(|res| res.unwrap())
+ .collect();
+
+ prop_assert_eq!(records, read_records);
+ }
+ }
}
diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rs
index 19132092..560f20dc 100644
--- a/core/src/webpage/url_ext.rs
+++ b/core/src/webpage/url_ext.rs
@@ -36,8 +36,8 @@ impl UrlExt for url::Url {
}
fn subdomain(&self) -> Option<&str> {
- let domain = dbg!(self.root_domain()?);
- let host = dbg!(self.host_str()?);
+ let domain = self.root_domain()?;
+ let host = self.host_str()?;
let mut subdomain = host.strip_suffix(domain)?;