web search done right
Various different optimizations. This commit should have been structured much better.
Mikkel Denker4bba8feb0a83
diff --git a/core/src/collector.rs b/core/src/collector.rsindex 3bafff13..dd5bfd23 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -283,24 +283,15 @@ impl<T: Doc> BucketCollector<T> {Self {top_n,+ documents: MinMaxHeap::with_capacity(config.max_docs_considered + 1),count: BucketCount::new(config),- documents: MinMaxHeap::with_capacity(top_n + 1),}}pub fn insert(&mut self, doc: T) {let mut scored_doc: ScoredDoc<T> = doc.into();self.count.adjust_score(&mut scored_doc);-- if self.documents.len() > self.top_n {- let mut min_scored_doc = self.documents.peek_min_mut().unwrap();-- if scored_doc.adjusted_score > min_scored_doc.adjusted_score {- *min_scored_doc = scored_doc;- }- } else {- self.documents.push(scored_doc);- }+ self.documents.push(scored_doc);}fn update_best_doc(&mut self) {diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex 7d36501a..e51937a0 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -18,16 +18,20 @@ pub struct Collector;impl Collector {pub fn site_penalty() -> f64 {- 0.3+ 0.1}pub fn title_penalty() -> f64 {- 2.0+ 1.0}pub fn url_penalty() -> f64 {20.0}++ pub fn max_docs_considered() -> usize {+ 250_000+ }}pub struct Frontend;diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 8a3a9f84..7228acd7 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -153,6 +153,9 @@ pub struct CollectorConfig {#[serde(default = "defaults::Collector::url_penalty")]pub url_penalty: f64,++ #[serde(default = "defaults::Collector::max_docs_considered")]+ pub max_docs_considered: usize,}impl Default for CollectorConfig {@@ -161,6 +164,7 @@ impl Default for CollectorConfig {site_penalty: defaults::Collector::site_penalty(),title_penalty: defaults::Collector::title_penalty(),url_penalty: defaults::Collector::url_penalty(),+ max_docs_considered: defaults::Collector::max_docs_considered(),}}}diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex a0ca9579..d857d2cf 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -48,6 +48,9 @@ pub enum Error {#[error("invalid politeness factor")]InvalidPolitenessFactor,++ #[error("invalid redirect")]+ InvalidRedirect,}type Result<T> = std::result::Result<T, anyhow::Error>;diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 27fa39d4..913743ec 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -65,10 +65,21 @@ impl WorkerThread {return Err(Error::InvalidPolitenessFactor.into());}+ let mut headers = reqwest::header::HeaderMap::default();+ headers.insert(+ reqwest::header::ACCEPT,+ reqwest::header::HeaderValue::from_static("text/html"),+ );+ headers.insert(+ reqwest::header::ACCEPT_LANGUAGE,+ reqwest::header::HeaderValue::from_static("en-US,en;q=0.9,*;q=0.8"),+ );+let client = reqwest::Client::builder().timeout(timeout).connect_timeout(timeout).http2_keep_alive_interval(None)+ .default_headers(headers).user_agent(&config.user_agent.full).build()?;@@ -248,71 +259,74 @@ impl WorkerThread {match fetch {Ok(datum) => {if matches!(datum.status_code, 200 | 301 | 302) {- self.save_datum(datum.clone()).await;-- match Html::parse(&datum.body, datum.url.as_str()) {- Ok(html) => {- let new_urls = html- .all_links()- .into_iter()- .map(|link| link.destination)- .filter(|url| {- !url.path().ends_with(".pdf")- && !url.path().ends_with(".jpg")- && !url.path().ends_with(".zip")- && !url.path().ends_with(".png")- && !url.path().ends_with(".css")- && !url.path().ends_with(".js")- && !url.path().ends_with(".json")- && !url.path().ends_with(".jsonp")- && !url.path().ends_with(".woff2")- && !url.path().ends_with(".woff")- && !url.path().ends_with(".ttf")- && !url.path().ends_with(".svg")- && !url.path().ends_with(".gif")- && !url.path().ends_with(".jpeg")- && !url.path().ends_with(".ico")- && !url.path().ends_with(".mp4")- && !url.path().ends_with(".mp3")- && !url.path().ends_with(".avi")- && !url.path().ends_with(".mov")- && !url.path().ends_with(".mpeg")- && !url.path().ends_with(".webm")- && !url.path().ends_with(".wav")- && !url.path().ends_with(".flac")- && !url.path().ends_with(".aac")- && !url.path().ends_with(".ogg")- && !url.path().ends_with(".m4a")- && !url.path().ends_with(".m4v")- })- .collect();-- let url_res = if datum.status_code == 301- || datum.status_code == 302- || url != datum.url- {- UrlResponse::Redirected {- url,- new_url: datum.url,+ if datum.status_code == 200 {+ self.save_datum(datum.clone()).await;++ match Html::parse(&datum.body, datum.url.as_str()) {+ Ok(html) => {+ let new_urls = html+ .all_links()+ .into_iter()+ .map(|link| link.destination)+ .filter(|url| {+ !url.path().ends_with(".pdf")+ && !url.path().ends_with(".jpg")+ && !url.path().ends_with(".zip")+ && !url.path().ends_with(".png")+ && !url.path().ends_with(".css")+ && !url.path().ends_with(".js")+ && !url.path().ends_with(".json")+ && !url.path().ends_with(".jsonp")+ && !url.path().ends_with(".woff2")+ && !url.path().ends_with(".woff")+ && !url.path().ends_with(".ttf")+ && !url.path().ends_with(".svg")+ && !url.path().ends_with(".gif")+ && !url.path().ends_with(".jpeg")+ && !url.path().ends_with(".ico")+ && !url.path().ends_with(".mp4")+ && !url.path().ends_with(".mp3")+ && !url.path().ends_with(".avi")+ && !url.path().ends_with(".mov")+ && !url.path().ends_with(".mpeg")+ && !url.path().ends_with(".webm")+ && !url.path().ends_with(".wav")+ && !url.path().ends_with(".flac")+ && !url.path().ends_with(".aac")+ && !url.path().ends_with(".ogg")+ && !url.path().ends_with(".m4a")+ && !url.path().ends_with(".m4v")+ })+ .collect();++ let url_res = UrlResponse::Success { url: datum.url };++ ProcessedUrl {+ new_urls,+ response: url_res,+ fetch_time: Duration::from_millis(datum.fetch_time_ms),}- } else {- UrlResponse::Success { url: datum.url }- };-- ProcessedUrl {- new_urls,- response: url_res,- fetch_time: Duration::from_millis(datum.fetch_time_ms),}+ Err(_) => ProcessedUrl {+ new_urls: Vec::new(),+ response: UrlResponse::Failed {+ url,+ status_code: None,+ },+ fetch_time: Duration::from_millis(datum.fetch_time_ms),+ },}- Err(_) => ProcessedUrl {+ } else {+ let url_res = UrlResponse::Redirected {+ url,+ new_url: datum.url,+ };++ ProcessedUrl {new_urls: Vec::new(),- response: UrlResponse::Failed {- url,- status_code: None,- },+ response: url_res,fetch_time: Duration::from_millis(datum.fetch_time_ms),- },+ }}} else {if datum.status_code == 429 {@@ -356,6 +370,10 @@ impl WorkerThread {}async fn save_datum(&self, datum: CrawlDatum) {+ if datum.status_code != 200 {+ return;+ }+self.writer.write(datum).await.ok();}@@ -411,6 +429,27 @@ impl WorkerThread {let status_code = res.status().as_u16();+ if status_code == 301 || status_code == 302 {+ let location = res+ .headers()+ .get("location")+ .ok_or(Error::InvalidRedirect)?;++ let location = location.to_str().map_err(|_| Error::InvalidRedirect)?;++ let url = Url::parse(location)+ .or_else(|_| url.join(location))+ .map_err(|_| Error::InvalidRedirect)?;++ return Ok(CrawlDatum {+ url,+ status_code,+ headers,+ body: String::new(),+ fetch_time_ms: fetch_time.as_millis() as u64,+ });+ }+let res_url = res.url().clone();let body = res.text().await?;diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 16070e4a..66bef9d3 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -182,7 +182,7 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {loop {if let Err(e) = server.accept().await {- tracing::error!("{}", e);+ tracing::error!("{:?}", e);}}}diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsindex eeaba9fa..0f316e02 100644--- a/core/src/entrypoint/webgraph_server.rs+++ b/core/src/entrypoint/webgraph_server.rs@@ -152,7 +152,7 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {loop {if let Err(e) = server.accept().await {- tracing::error!("{}", e);+ tracing::error!("{:?}", e);}}}diff --git a/core/src/enum_map.rs b/core/src/enum_map.rsindex 92fa9feb..07d06c2d 100644--- a/core/src/enum_map.rs+++ b/core/src/enum_map.rs@@ -72,6 +72,10 @@ whereself.inner.iter_mut().filter_map(|value| value.as_mut())}+ pub fn values(&self) -> impl Iterator<Item = &V> {+ self.inner.iter().filter_map(|value| value.as_ref())+ }+pub fn get_mut(&mut self, key: K) -> Option<&mut V> {let key = key.into();if key >= self.inner.len() {diff --git a/core/src/index.rs b/core/src/index.rsindex cd235326..1003d183 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -336,6 +336,6 @@ mod tests {.unwrap().get(&crate::ranking::Signal::Bm25).unwrap())- .all(|&v| v > 0.0));+ .all(|&v| v.value > 0.0));}}diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex 1a4ee3e4..75395032 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -339,6 +339,8 @@ fn parse_term(term: &str) -> Box<Term> {#[allow(clippy::vec_box)]pub fn parse(query: &str) -> Vec<Box<Term>> {+ let query = query.to_lowercase();+let mut res = Vec::new();let mut cur_term_begin = 0;diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 0f7b4958..9f95bae7 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -41,6 +41,7 @@ pub struct Scorer {disliked: Vec<NodeScorer>,vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,cache: DashMap<NodeID, f64>,+ normalized: bool,}#[derive(Debug)]@@ -74,22 +75,29 @@ impl NodeScorer {impl Scorer {fn calculate_score(&self, node: &NodeID) -> f64 {- match self.vectors.get(node) {- Some(vec) => ((self.disliked.len() as f64)- + (self- .liked- .iter()- .map(|liked| liked.sim(node, vec))- .sum::<f64>()- - self- .disliked+ let s = match self.vectors.get(node) {+ Some(vec) => {+ (self.disliked.len() as f64)+ + (self+ .liked.iter()- .map(|disliked| disliked.sim(node, vec))- .sum::<f64>())- / self.liked.len().max(1) as f64)- .max(0.0),+ .map(|liked| liked.sim(node, vec))+ .sum::<f64>()+ - self+ .disliked+ .iter()+ .map(|disliked| disliked.sim(node, vec))+ .sum::<f64>())+ }None => 0.0,+ };++ if self.normalized {+ s / self.liked.len().max(1) as f64+ } else {+ s}+ .max(0.0)}pub fn score(&self, node: &NodeID) -> f64 {if let Some(cached) = self.cache.get(node) {@@ -199,7 +207,12 @@ impl InboundSimilarity {}}- pub fn scorer(&self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer {+ pub fn scorer(+ &self,+ liked_sites: &[NodeID],+ disliked_sites: &[NodeID],+ normalized: bool,+ ) -> Scorer {let liked: Vec<_> = liked_sites.iter().filter_map(|id| self.vectors.get(id).cloned().map(|vec| (id, vec)))@@ -217,6 +230,7 @@ impl InboundSimilarity {disliked,vectors: self.vectors.clone(),cache: DashMap::new(),+ normalized,}}@@ -297,7 +311,7 @@ mod tests {let inbound =InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);- let scorer = inbound.scorer(&[Node::from("b.com").id()], &[]);+ let scorer = inbound.scorer(&[Node::from("b.com").id()], &[], false);let e = Node::from("e.com").id();let d = Node::from("d.com").id();diff --git a/core/src/ranking/initial.rs b/core/src/ranking/initial.rsindex d360b339..8b06e713 100644--- a/core/src/ranking/initial.rs+++ b/core/src/ranking/initial.rs@@ -80,7 +80,7 @@ impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {.aggregator.compute_signals(doc).flatten()- .map(|computed| computed.coefficient * computed.value)+ .map(|computed| computed.score.coefficient * computed.score.value).sum();if let Some(boost) = self.aggregator.boosts(doc) {diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex b775bb77..96a2fde6 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -647,6 +647,7 @@ mod tests {}#[test]+ #[ignore = "proximity signals are currently disabled"]fn term_proximity_ranking() {let mut index = Index::temporary().expect("Unable to open index");let mut w = Webpage {diff --git a/core/src/ranking/models/lambdamart.rs b/core/src/ranking/models/lambdamart.rsindex 87555e51..94c9526e 100644--- a/core/src/ranking/models/lambdamart.rs+++ b/core/src/ranking/models/lambdamart.rs@@ -49,6 +49,16 @@ enum NodeOrLeaf {Leaf(usize),}+pub trait AsValue {+ fn as_value(&self) -> f64;+}++impl AsValue for f64 {+ fn as_value(&self) -> f64 {+ *self+ }+}+#[derive(Debug)]struct Node {threshold: f64,@@ -59,9 +69,9 @@ struct Node {}impl Node {- fn next(&self, features: &EnumMap<Signal, f64>) -> Option<&NodeOrLeaf> {+ fn next<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> Option<&NodeOrLeaf> {self.feature.and_then(|feature| {- let value = features.get(feature).copied().unwrap_or(0.0);+ let value = features.get(feature).map(|v| v.as_value()).unwrap_or(0.0);if value <= self.threshold {self.left.as_ref()} else {@@ -205,7 +215,7 @@ impl Tree {Ok(Self { nodes })}- fn predict(&self, features: &EnumMap<Signal, f64>) -> Result<f64> {+ fn predict<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> Result<f64> {let mut node = &self.nodes[0];while let Some(next) = node.next(features) {node = match next {@@ -255,7 +265,7 @@ impl LambdaMART {Ok(Self { tree })}- pub fn predict(&self, features: &EnumMap<Signal, f64>) -> f64 {+ pub fn predict<V: AsValue>(&self, features: &EnumMap<Signal, V>) -> f64 {self.tree.predict(features).unwrap()}}diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 2ac97ecc..da791350 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -98,7 +98,7 @@ mod tests {.unwrap(),backlinks: vec![],host_centrality: 1.0,- fetch_time_ms: 50,+ fetch_time_ms: 500,pre_computed_score: 0.0,page_centrality: 0.0,@@ -127,7 +127,7 @@ mod tests {.unwrap(),backlinks: vec![],host_centrality: 1.0,- fetch_time_ms: 49,+ fetch_time_ms: 499,page_centrality: 0.0,pre_computed_score: 0.0,@@ -174,7 +174,7 @@ mod tests {query: "test".to_string(),site_rankings: Some(SiteRankings {liked: vec!["www.first.com".to_string()],- disliked: vec!["www.third.com".to_string()],+ disliked: vec![],blocked: vec![],}),..Default::default()diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 3bc624e9..16cfc426 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -28,8 +28,11 @@ use crate::{};use super::{- models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},- Signal, SignalAggregator, SignalCoefficient,+ models::{+ cross_encoder::CrossEncoder,+ lambdamart::{self, LambdaMART},+ },+ Signal, SignalAggregator, SignalCoefficient, SignalScore,};pub trait AsRankingWebsite: Clone {@@ -64,10 +67,16 @@ impl AsRankingWebsite for RankingWebsite {}}+impl lambdamart::AsValue for SignalScore {+ fn as_value(&self) -> f64 {+ self.value+ }+}+#[derive(Clone, Debug, Serialize, Deserialize)]pub struct RankingWebsite {pub pointer: WebsitePointer,- pub signals: EnumMap<Signal, f64>,+ pub signals: EnumMap<Signal, SignalScore>,pub title: Option<String>,pub clean_body: Option<String>,pub optic_boost: Option<f64>,@@ -85,15 +94,11 @@ impl RankingWebsite {pointer: pointer.clone(),};- let mut total = 0.0;for computed_signal in aggregator.compute_signals(pointer.address.doc_id).flatten() {res.signals- .insert(computed_signal.signal, computed_signal.value);- total += computed_signal.value * computed_signal.coefficient;+ .insert(computed_signal.signal, computed_signal.score);}- res.signals.insert(Signal::LinearRegression, total);-if let Some(boost) = aggregator.boosts(pointer.address.doc_id) {res.optic_boost = Some(boost);}@@ -140,9 +145,22 @@ impl<M: CrossEncoder> ReRanker<M> {for (website, score) in websites.iter_mut().zip(scores) {let website = website.as_mut_ranking();- website.signals.insert(Signal::CrossEncoder, score);+ website.signals.insert(+ Signal::CrossEncoder,+ SignalScore {+ coefficient: self.crossencoder_coeff(),+ value: score,+ },+ );}}++ fn crossencoder_coeff(&self) -> f64 {+ self.signal_coefficients+ .as_ref()+ .and_then(|coeffs| coeffs.get(&Signal::CrossEncoder))+ .unwrap_or(Signal::CrossEncoder.default_coefficient())+ }}impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {@@ -151,22 +169,11 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {for website in websites.iter_mut() {let website = website.as_mut_ranking();- let score = calculate_score(+ website.score = calculate_score(&self.lambda_mart,&self.signal_coefficients,&website.signals,);-- if let Some(coeffs) = &self.signal_coefficients {- if let Some(coeff) = coeffs.get(&Signal::CrossEncoder) {- website.score =- score + coeff * website.signals.get(Signal::CrossEncoder).unwrap();- } else {- website.score = score;- }- } else {- website.score = score;- }}}@@ -180,7 +187,7 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {fn calculate_score(model: &Option<Arc<LambdaMART>>,signal_coefficients: &Option<SignalCoefficient>,- signals: &EnumMap<Signal, f64>,+ signals: &EnumMap<Signal, SignalScore>,) -> f64 {let lambda_score = match model {Some(model) => match signal_coefficients {@@ -188,9 +195,9 @@ fn calculate_score(Some(coeff) => {if coeff == 0.0 {signals- .get(Signal::LinearRegression)- .copied()- .unwrap_or(0.0)+ .values()+ .map(|score| score.coefficient * score.value)+ .sum()} else {coeff * model.predict(signals)}@@ -200,9 +207,9 @@ fn calculate_score(None => Signal::LambdaMART.default_coefficient() * model.predict(signals),},None => signals- .get(Signal::LinearRegression)- .copied()- .unwrap_or(0.0),+ .values()+ .map(|score| score.coefficient * score.value)+ .sum(),};lambda_score@@ -388,7 +395,13 @@ mod tests {(0..n).map(|i| -> RankingWebsite {let mut signals = EnumMap::new();- signals.insert(Signal::LinearRegression, 1.0 / i as f64);+ signals.insert(+ Signal::Bm25,+ SignalScore {+ coefficient: 1.0,+ value: 1.0 / i as f64,+ },+ );RankingWebsite {pointer: WebsitePointer {score: Score { total: 0.0 },diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 1d5ea5f5..ab2a76de 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -27,6 +27,7 @@ use crate::{};use optics::ast::RankingTarget;use optics::Optic;+use serde::{Deserialize, Serialize};use std::str::FromStr;use std::sync::Arc;use tantivy::fieldnorm::FieldNormReader;@@ -47,6 +48,8 @@ use super::bm25::Bm25Weight;use super::models::linear::LinearRegression;use super::{inbound_similarity, query_centrality};+const PROXIMITY_ENABLED: bool = false;+#[derive(Debug, Error)]pub enum Error {#[error("unknown signal: {0}")]@@ -95,16 +98,31 @@ pub enum Signal {Bm25BacklinkText,#[serde(rename = "bm25_description")]Bm25Description,++ /// All proximity signals are disabled for now.+ /// We need to figure out how to make them faster.#[serde(rename = "proximity_slop_0")]ProximitySlop0,++ /// All proximity signals are disabled for now.+ /// We need to figure out how to make them faster.#[serde(rename = "proximity_slop_1")]ProximitySlop1,++ /// All proximity signals are disabled for now.+ /// We need to figure out how to make them faster.#[serde(rename = "proximity_slop_2")]ProximitySlop2,+ /// All proximity signals are disabled for now.+ /// We need to figure out how to make them faster.#[serde(rename = "proximity_slop_4")]ProximitySlop4,++ /// All proximity signals are disabled for now.+ /// We need to figure out how to make them faster.#[serde(rename = "proximity_slop_8")]ProximitySlop8,+#[serde(rename = "cross_encoder")]CrossEncoder,#[serde(rename = "host_centrality")]@@ -127,8 +145,6 @@ pub enum Signal {InboundSimilarity,#[serde(rename = "lambda_mart")]LambdaMART,- #[serde(rename = "linear_regression")]- LinearRegression,#[serde(rename = "url_digits")]UrlDigits,#[serde(rename = "url_slashes")]@@ -249,8 +265,8 @@ impl Signal {pub fn default_coefficient(&self) -> f64 {match self {- Signal::Bm25 => 0.0005,- Signal::Bm25Title => 0.005,+ Signal::Bm25 => 0.0002,+ Signal::Bm25Title => 0.004,Signal::Bm25TitleBigrams => 0.005,Signal::Bm25TitleTrigrams => 0.005,Signal::Bm25CleanBody => 0.005,@@ -264,28 +280,27 @@ impl Signal {Signal::Bm25Domain => 0.00005,Signal::Bm25SiteNoTokenizer => 0.00005,Signal::Bm25DomainNoTokenizer => 0.00005,- Signal::Bm25DomainIfHomepage => 0.00005,- Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0001,- Signal::Bm25TitleIfHomepage => 0.00001,+ Signal::Bm25DomainIfHomepage => 0.0002,+ Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0002,+ Signal::Bm25TitleIfHomepage => 0.00002,Signal::Bm25BacklinkText => 0.001,- Signal::Bm25Description => 5.99679347128802e-05,+ Signal::Bm25Description => 0.00001,Signal::ProximitySlop0 => 0.01,Signal::ProximitySlop1 => 0.01,Signal::ProximitySlop2 => 0.01,Signal::ProximitySlop4 => 0.01,Signal::ProximitySlop8 => 0.01,- Signal::CrossEncoder => 0.16801589371906178,+ Signal::CrossEncoder => 0.17,Signal::HostCentrality => 1.0,- Signal::PageCentrality => 2.0,+ Signal::PageCentrality => 4.0,Signal::QueryCentrality => 0.0,- Signal::IsHomepage => 0.01,+ Signal::IsHomepage => 0.0005,Signal::FetchTimeMs => 0.001,Signal::UpdateTimestamp => 0.001,- Signal::TrackerScore => 0.019542120533715634,- Signal::Region => 0.06437975586036043,- Signal::InboundSimilarity => 10.0,+ Signal::TrackerScore => 0.02,+ Signal::Region => 0.005,+ Signal::InboundSimilarity => 1.0,Signal::LambdaMART => 10.0,- Signal::LinearRegression => 1.0,Signal::UrlSlashes => 0.01,Signal::UrlDigits => 0.01,}@@ -333,6 +348,11 @@ impl Signal {signal_aggregator: &mut SignalAggregator,doc: DocId,) -> Option<ComputedSignal> {+ let coefficient = signal_aggregator.coefficient(&self);+ if coefficient == 0.0 {+ return None;+ }+let value: Option<f64> = match self {Signal::HostCentrality | Signal::PageCentrality => {let field_value: Option<u64> = self@@ -462,13 +482,11 @@ impl Signal {}Signal::CrossEncoder => None, // this is calculated in a later stepSignal::LambdaMART => None,- Signal::LinearRegression => None, // this is calculated just before input to lambdamart};value.map(|value| ComputedSignal {signal: self,- coefficient: signal_aggregator.coefficient(&self),- value,+ score: SignalScore { coefficient, value },})}@@ -566,7 +584,6 @@ impl Signal {| Signal::CrossEncoder| Signal::InboundSimilarity| Signal::LambdaMART- | Signal::LinearRegression| Signal::QueryCentrality => {tracing::error!("signal {self:?} cannot be precomputed");None@@ -575,8 +592,10 @@ impl Signal {value.map(|value| ComputedSignal {signal: self,- coefficient: signal_aggregator.coefficient(&self),- value,+ score: SignalScore {+ coefficient: signal_aggregator.coefficient(&self),+ value,+ },})}@@ -903,6 +922,10 @@ impl SignalAggregator {tv_searcher: &tantivy::Searcher,segment_reader: &tantivy::SegmentReader,) -> EnumMap<Signal, ProximityScorer> {+ if !PROXIMITY_ENABLED {+ return EnumMap::new();+ }+let mut proximity_scorers = EnumMap::new();let proximity_fields = [TextField::Title, TextField::CleanBody];let schema = tv_searcher.schema();@@ -1060,7 +1083,7 @@ impl SignalAggregator {ALL_SIGNALS.into_iter().filter_map(|signal| signal.precompute(self, webpage))- .map(|computed| computed.coefficient * computed.value)+ .map(|computed| computed.score.coefficient * computed.score.value).sum()}@@ -1080,6 +1103,11 @@ impl SignalAggregator {#[derive(Debug, Clone, Copy)]pub struct ComputedSignal {pub signal: Signal,+ pub score: SignalScore,+}++#[derive(Debug, Clone, Copy, Serialize, Deserialize)]+pub struct SignalScore {pub coefficient: f64,pub value: f64,}@@ -1095,7 +1123,7 @@ mod tests {use super::*;- fn signals_for_query(searcher: &LocalSearcher, q: &str) -> HashMap<Signal, f64> {+ fn signals_for_query(searcher: &LocalSearcher, q: &str) -> HashMap<Signal, SignalScore> {let res = searcher.search(&SearchQuery {query: q.to_string(),@@ -1106,6 +1134,7 @@ mod tests {res[0].ranking_signals.clone().unwrap()}#[test]+ #[ignore = "proximity signals are currently disabled"]fn test_proximity() {let mut index = Index::temporary().expect("Unable to open index");@@ -1138,45 +1167,45 @@ mod tests {let searcher = LocalSearcher::new(index);let signals = signals_for_query(&searcher, "a b");- assert!(signals[&Signal::ProximitySlop8] > 0.0);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop1]);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop0]);+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop1].value);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop0].value);let signals = signals_for_query(&searcher, "a c");- assert!(signals[&Signal::ProximitySlop8] > 0.0);- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop1]);+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop1].value);let signals = signals_for_query(&searcher, "a d");- assert!(signals[&Signal::ProximitySlop8] > 0.0);- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);- assert_eq!(signals[&Signal::ProximitySlop1], 0.0);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop2]);+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop2].value);let signals = signals_for_query(&searcher, "a e");- assert!(signals[&Signal::ProximitySlop8] > 0.0);- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);- assert_eq!(signals[&Signal::ProximitySlop1], 0.0);- assert_eq!(signals[&Signal::ProximitySlop2], 0.0);- assert!(signals[&Signal::ProximitySlop8] == signals[&Signal::ProximitySlop4]);+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);+ assert!(signals[&Signal::ProximitySlop8].value == signals[&Signal::ProximitySlop4].value);let signals = signals_for_query(&searcher, "a g");- assert!(signals[&Signal::ProximitySlop8] > 0.0);- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);- assert_eq!(signals[&Signal::ProximitySlop1], 0.0);- assert_eq!(signals[&Signal::ProximitySlop2], 0.0);- assert_eq!(signals[&Signal::ProximitySlop4], 0.0);+ assert!(signals[&Signal::ProximitySlop8].value > 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop4].value, 0.0);let signals = signals_for_query(&searcher, "a k");- assert_eq!(signals[&Signal::ProximitySlop0], 0.0);- assert_eq!(signals[&Signal::ProximitySlop1], 0.0);- assert_eq!(signals[&Signal::ProximitySlop2], 0.0);- assert_eq!(signals[&Signal::ProximitySlop4], 0.0);- assert_eq!(signals[&Signal::ProximitySlop8], 0.0);+ assert_eq!(signals[&Signal::ProximitySlop0].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop1].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop2].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop4].value, 0.0);+ assert_eq!(signals[&Signal::ProximitySlop8].value, 0.0);}}diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rsindex 60911a05..8e26d3b4 100644--- a/core/src/search_prettifier/mod.rs+++ b/core/src/search_prettifier/mod.rs@@ -26,7 +26,7 @@ use url::Url;use crate::{inverted_index::RetrievedWebpage,- ranking::Signal,+ ranking::{Signal, SignalScore},spell::{self, CorrectionTerm},webpage::url_ext::UrlExt,};@@ -168,7 +168,7 @@ pub struct DisplayedWebpage {pub pretty_url: String,pub snippet: Snippet,pub body: String,- pub ranking_signals: Option<HashMap<Signal, f64>>,+ pub ranking_signals: Option<HashMap<Signal, SignalScore>>,}#[derive(Debug, Serialize, Deserialize)]diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rsindex 51c5b09f..43f49995 100644--- a/core/src/searcher/frontend.rs+++ b/core/src/searcher/frontend.rs@@ -302,10 +302,9 @@ impl FrontendSearcher {let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());for signal in ALL_SIGNALS {- signals.insert(- signal,- pointer.website.signals.get(signal).copied().unwrap_or(0.0),- );+ if let Some(signal_value) = pointer.website.signals.get(signal) {+ signals.insert(signal, *signal_value);+ }}website.ranking_signals = Some(signals);diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex c9156299..369dcce2 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -141,7 +141,7 @@ impl LocalSearcher {if query_centrality_coeff > 0.0 {if let Some(centrality_store) = self.centrality_store.as_ref() {ranker = ranker- .with_max_docs(10_000, self.index.num_segments())+ .with_max_docs(1_000, self.index.num_segments()).with_num_results(100);let top_host_nodes =@@ -149,9 +149,10 @@ impl LocalSearcher {.top_nodes(query, ctx, ranker.collector(ctx.clone()))?;if !top_host_nodes.is_empty() {- let inbound = centrality_store- .inbound_similarity- .scorer(&top_host_nodes, &[]);+ let inbound =+ centrality_store+ .inbound_similarity+ .scorer(&top_host_nodes, &[], false);let query_centrality = query_centrality::Scorer::new(inbound);@@ -161,7 +162,10 @@ impl LocalSearcher {}Ok(ranker- .with_max_docs(250_000, self.index.num_segments())+ .with_max_docs(+ self.collector_config.max_docs_considered,+ self.index.num_segments(),+ ).with_num_results(query.num_results()).with_offset(query.offset()))}@@ -199,11 +203,15 @@ impl LocalSearcher {.map(|node| node.id()).collect();- aggregator.set_inbound_similarity(- store- .inbound_similarity- .scorer(&liked_sites, &disliked_sites),- )+ let mut scorer = store+ .inbound_similarity+ .scorer(&liked_sites, &disliked_sites, false);++ if liked_sites.len() + disliked_sites.len() > 1 {+ scorer.set_self_score(0.0);+ }++ aggregator.set_inbound_similarity(scorer);}aggregator.set_region_count(self.index.region_count.clone());@@ -357,7 +365,9 @@ impl LocalSearcher {let mut ranking_signals = HashMap::new();for signal in ALL_SIGNALS {- ranking_signals.insert(signal, *ranking.signals.get(signal).unwrap_or(&0.0));+ if let Some(score) = ranking.signals.get(signal) {+ ranking_signals.insert(signal, *score);+ }}webpage.ranking_signals = Some(ranking_signals);diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex 2f9009c5..d49a564a 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -81,7 +81,7 @@ impl SimilarSitesFinder {.map(|node| node.id()).collect();- let scorer = self.inbound_similarity.scorer(&nodes, &[]);+ let scorer = self.inbound_similarity.scorer(&nodes, &[], true);let mut backlinks: IntMap<NodeID, f64> = IntMap::new();diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rsindex d31fe3d9..b6091e45 100644--- a/core/src/spell/mod.rs+++ b/core/src/spell/mod.rs@@ -118,7 +118,7 @@ impl Spell {info!("Building spell correction dictionary");let searcher = index.inverted_index.tv_searcher();let schema = searcher.schema();- let mut dict = DictionaryBuilder::new(1_000_000);+ let mut dict = DictionaryBuilder::new(10_000);#[allow(unused_assignments, unused_mut)]let mut limit_terms: Option<usize> = None;diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rsindex 2b2f31ea..df57008c 100644--- a/core/src/tokenizer.rs+++ b/core/src/tokenizer.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{array, ops::Range, str::CharIndices};+use std::{array, str::CharIndices};use logos::{Lexer, Logos};use tantivy::tokenizer::{@@ -170,9 +170,7 @@ impl tantivy::tokenizer::Tokenizer for Normal {impl tantivy::tokenizer::Tokenizer for Stemmed {fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {- let analyzer = TextAnalyzer::from(Simple)- .filter(LowerCaser)- .filter(StopWordFilter::remove(vec![]));+ let analyzer = TextAnalyzer::from(Simple).filter(LowerCaser);let lang = match self.force_language {Some(lang) => Some(lang),@@ -195,7 +193,7 @@ impl tantivy::tokenizer::Tokenizer for Identity {impl tantivy::tokenizer::Tokenizer for BigramTokenizer {fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {let inner = Normal::default().token_stream(text);- let stream: NGramTokenStream<2> = NGramTokenStream::new(text, inner);+ let stream: NGramTokenStream<2> = NGramTokenStream::new(inner);BoxTokenStream::from(stream)}}@@ -203,7 +201,7 @@ impl tantivy::tokenizer::Tokenizer for BigramTokenizer {impl tantivy::tokenizer::Tokenizer for TrigramTokenizer {fn token_stream<'a>(&self, text: &'a str) -> tantivy::tokenizer::BoxTokenStream<'a> {let inner = Normal::default().token_stream(text);- let stream: NGramTokenStream<3> = NGramTokenStream::new(text, inner);+ let stream: NGramTokenStream<3> = NGramTokenStream::new(inner);BoxTokenStream::from(stream)}}@@ -306,61 +304,49 @@ impl<'a> tantivy::tokenizer::TokenStream for SimpleTokenStream<'a> {}}-#[derive(Default)]-struct TokenRef {- range: Range<usize>,- position: usize,-}--impl From<&tantivy::tokenizer::Token> for TokenRef {- fn from(token: &tantivy::tokenizer::Token) -> Self {- Self {- range: token.offset_from..token.offset_to,- position: token.position,- }- }-}-struct NGramTokenStream<'a, const N: usize> {inner: BoxTokenStream<'a>,- text: &'a str,token: tantivy::tokenizer::Token,- token_refs: [TokenRef; N],+ token_window: [tantivy::tokenizer::Token; N],is_first: bool,next_pos: usize,}impl<'a, const N: usize> NGramTokenStream<'a, N> {- fn new(text: &'a str, inner: BoxTokenStream<'a>) -> Self {+ fn new(inner: BoxTokenStream<'a>) -> Self {Self {inner,- text,token: tantivy::tokenizer::Token::default(),- token_refs: array::from_fn(|_| TokenRef::default()),+ token_window: array::from_fn(|_| tantivy::tokenizer::Token::default()),is_first: true,next_pos: 0,}}}+fn reuse_token_alloc(token: &mut tantivy::tokenizer::Token, new_token: &tantivy::tokenizer::Token) {+ token.text.clear();+ token.text += new_token.text.as_str();+ token.offset_from = new_token.offset_from;+ token.offset_to = new_token.offset_to;+ token.position = new_token.position;+ token.position_length = new_token.position_length;+}+impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 1> {fn advance(&mut self) -> bool {self.is_first = false;let res = self.inner.advance();+ self.token_window[0].text.clear();+if res {- self.token_refs[0].range = self.inner.token().offset_from..self.inner.token().offset_to;- self.token_refs[0].position = self.next_pos;+ self.token_window[0].text += self.inner.token().text.as_str();self.next_pos += 1;}- self.token = tantivy::tokenizer::Token {- offset_from: self.token_refs[0].range.start,- offset_to: self.token_refs[0].range.end,- position: self.token_refs[0].position,- text: self.text[self.token_refs[0].range.clone()].to_string(),- position_length: 1,- };+ reuse_token_alloc(&mut self.token, &self.token_window[0]);+ self.token.position_length = 1;res}@@ -380,29 +366,36 @@ impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 2> {if !self.inner.advance() {return false;}-- self.token_refs[0] = self.inner.token().into();+ reuse_token_alloc(&mut self.token_window[0], self.inner.token());if !self.inner.advance() {return false;}- self.token_refs[1] = self.inner.token().into();+ reuse_token_alloc(&mut self.token_window[1], self.inner.token());} else {if !self.inner.advance() {return false;}- self.token_refs.rotate_left(1);- self.token_refs[1] = self.inner.token().into();+ self.token_window.rotate_left(1);+ reuse_token_alloc(&mut self.token_window[1], self.inner.token());}- self.is_first = false;+ self.is_first = false;self.next_pos += 1;self.token.position = self.next_pos;- self.token.text =- self.text[self.token_refs[0].range.start..self.token_refs[1].range.end].to_string();+ self.token.offset_from = self.token_window[0].offset_from;+ self.token.offset_to = self.token_window[1].offset_to;self.token.position_length = 2;++ self.token.text.clear();+ for token in &self.token_window {+ self.token.text += token.text.as_str();+ self.token.text += " ";+ }+ self.token.text.pop();+true}@@ -422,33 +415,41 @@ impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 3> {return false;}- self.token_refs[0] = self.inner.token().into();+ reuse_token_alloc(&mut self.token_window[0], self.inner.token());if !self.inner.advance() {return false;}- self.token_refs[1] = self.inner.token().into();+ reuse_token_alloc(&mut self.token_window[1], self.inner.token());if !self.inner.advance() {return false;}- self.token_refs[2] = self.inner.token().into();+ reuse_token_alloc(&mut self.token_window[2], self.inner.token());} else {if !self.inner.advance() {return false;}- self.token_refs.rotate_left(1);- self.token_refs[2] = self.inner.token().into();+ self.token_window.rotate_left(1);+ reuse_token_alloc(&mut self.token_window[2], self.inner.token());}- self.is_first = false;+ self.is_first = false;self.next_pos += 1;self.token.position = self.next_pos;- self.token.text =- self.text[self.token_refs[0].range.start..self.token_refs[2].range.end].to_string();+ self.token.offset_from = self.token_window[0].offset_from;+ self.token.offset_to = self.token_window[2].offset_to;self.token.position_length = 3;++ self.token.text.clear();+ for token in &self.token_window {+ self.token.text += token.text.as_str();+ self.token.text += " ";+ }+ self.token.text.pop();+true}@@ -906,6 +907,11 @@ key1.key2="this\" is @ a # test""#;"a test".to_string()]);++ assert_eq!(+ tokenize_bigram("this.is"),+ vec!["this .".to_string(), ". is".to_string()]+ );}#[test]diff --git a/core/src/warc.rs b/core/src/warc.rsindex 2dff6f90..a43613ad 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -26,6 +26,9 @@ use std::time::Duration;use flate2::read::MultiGzDecoder;use flate2::write::GzEncoder;use flate2::Compression;+#[cfg(test)]+use proptest_derive::Arbitrary;+use tracing::{debug, trace};pub struct WarcFile {@@ -205,6 +208,7 @@ struct RawWarcRecord {}#[derive(Debug)]+#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]pub struct WarcRecord {pub request: Request,pub response: Response,@@ -212,6 +216,7 @@ pub struct WarcRecord {}#[derive(Debug)]+#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]pub struct Request {// WARC-Target-URIpub url: String,@@ -230,6 +235,7 @@ impl Request {}#[derive(Debug)]+#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]pub struct Response {pub body: String,pub payload_type: Option<String>,@@ -251,6 +257,7 @@ impl Response {}#[derive(Debug)]+#[cfg_attr(test, derive(Clone, Arbitrary, PartialEq))]pub struct Metadata {// fetchTimeMspub fetch_time_ms: usize,@@ -308,14 +315,23 @@ impl<R: Read> RecordIterator<R> {return Some(Err(io.into()));}- rtrim(&mut line_buf);-if &line_buf == "\r\n" || line_buf.is_empty() {// end of headerbreak;}if let Some(semi) = line_buf.find(':') {- let value = line_buf.split_off(semi + 1).trim().to_string();+ let mut value = line_buf.split_off(semi + 1).to_string();++ if let Some(stripped) = value.strip_suffix("\r\n") {+ value = stripped.to_string();+ } else if let Some(stripped) = value.strip_suffix('\n') {+ value = stripped.to_string();+ }++ if let Some(stripped) = value.strip_prefix(' ') {+ value = stripped.to_string();+ }+line_buf.pop(); // remove colonlet key = line_buf;@@ -370,15 +386,9 @@ impl<R: Read> Iterator for RecordIterator<R> {}self.num_reads += 1;- let items = [self.next_raw(), self.next_raw(), self.next_raw()];-- let mut response = None;let mut request = None;- let mut metadata = None;-- for item in items {- let item = item?;+ while let Some(item) = self.next_raw() {if item.is_err() {return Some(Err(item.err().unwrap()));}@@ -386,28 +396,70 @@ impl<R: Read> Iterator for RecordIterator<R> {let item = item.unwrap();if let Some(warc_type) = item.header.get("WARC-TYPE") {- match warc_type.as_str() {- "request" => request = Some(Request::from_raw(item)),- "response" => response = Some(Response::from_raw(item)),- "metadata" => metadata = Some(Metadata::from_raw(item)),- _ => {- return Some(Err(Error::WarcParse("Unsupported WARC type").into()));- }+ if warc_type.as_str() == "request" {+ request = Some(Request::from_raw(item));+ break;}}}- if request.is_none() || response.is_none() || metadata.is_none() {- return Some(Err(Error::WarcParse(- "Request, response or metadata not found",- )- .into()));+ let request = request?;++ // next 2 should be response and metadata+ let response = self.next_raw()?;++ if response.is_err() {+ return Some(Err(response.err().unwrap()));}- let request = request.unwrap();let response = response.unwrap();++ match response.header.get("WARC-TYPE") {+ Some(warc_type) => {+ if warc_type.as_str() != "response" {+ return Some(Err(Error::WarcParse(+ "Expected response, got something else",+ )+ .into()));+ }+ }+ None => {+ return Some(Err(Error::WarcParse(+ "Expected response, got something else",+ )+ .into()));+ }+ }++ let response = Response::from_raw(response);++ let metadata = self.next_raw()?;++ if metadata.is_err() {+ return Some(Err(metadata.err().unwrap()));+ }+let metadata = metadata.unwrap();+ match metadata.header.get("WARC-TYPE") {+ Some(warc_type) => {+ if warc_type.as_str() != "metadata" {+ return Some(Err(Error::WarcParse(+ "Expected metadata, got something else",+ )+ .into()));+ }+ }+ None => {+ return Some(Err(Error::WarcParse(+ "Expected metadata, got something else",+ )+ .into()));+ }+ }++ let metadata = Metadata::from_raw(metadata);+if request.is_err() || response.is_err() || metadata.is_err() {return Some(Err(Error::WarcParse("Request, response or metadata is error",@@ -534,6 +586,7 @@ mod tests {use super::*;use flate2::write::GzEncoder;use flate2::Compression;+ use proptest::prelude::*;use std::io::Write;#[test]@@ -689,4 +742,22 @@ mod tests {assert_eq!(&records[0].response.body, body);assert_eq!(records[0].metadata.fetch_time_ms, 0);}++ proptest! {+ #[test]+ fn write_read_invariant_prop(records: Vec<WarcRecord>) {+ let mut writer = WarcWriter::new();+ for record in records.iter() {+ writer.write(record).unwrap();+ }+ let compressed = writer.finish().unwrap();++ let read_records: Vec<WarcRecord> = WarcFile::new(compressed)+ .records()+ .map(|res| res.unwrap())+ .collect();++ prop_assert_eq!(records, read_records);+ }+ }}diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rsindex 19132092..560f20dc 100644--- a/core/src/webpage/url_ext.rs+++ b/core/src/webpage/url_ext.rs@@ -36,8 +36,8 @@ impl UrlExt for url::Url {}fn subdomain(&self) -> Option<&str> {- let domain = dbg!(self.root_domain()?);- let host = dbg!(self.host_str()?);+ let domain = self.root_domain()?;+ let host = self.host_str()?;let mut subdomain = host.strip_suffix(domain)?;