web search done right
Crawl stability field
Mikkel Denkerce7a3f35993b
diff --git a/configs/indexer/local.toml b/configs/indexer/local.tomlindex 81f75d4d..811b225a 100644--- a/configs/indexer/local.toml+++ b/configs/indexer/local.toml@@ -1,11 +1,12 @@centrality_store_path = "./data/centrality"limit_warc_files = 1download_images = false+# crawl_stability_path = "data/crawl-stability"[warc_source]-# type = "Local"-# folder = "./data/warc_files"-# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]-type = "HTTP"-base_url = "https://data.commoncrawl.org/"-warc_paths_file = "warc.paths"+type = "Local"+folder = "./data"+names = ["sample.warc.gz"]+# type = "HTTP"+# base_url = "https://data.commoncrawl.org/"+# warc_paths_file = "warc.paths"diff --git a/src/entrypoint/crawl_stability.rs b/src/entrypoint/crawl_stability.rsnew file mode 100644index 00000000..86760192--- /dev/null+++ b/src/entrypoint/crawl_stability.rs@@ -0,0 +1,68 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use flate2::bufread::MultiGzDecoder;++use crate::{+ kv::{rocksdb_store::RocksDbStore, Kv},+ Result,+};+use std::{+ fs::File,+ io::{BufRead, BufReader},+ path::Path,+};++pub struct CrawlStability {+ scores: Box<dyn Kv<String, f64>>,+}++impl CrawlStability {+ pub fn build<P: AsRef<Path>>(host_ranks_paths: Vec<String>, path: P) -> Result<Self> {+ let scores = RocksDbStore::open(path.as_ref());++ let norm: f64 = (0..host_ranks_paths.len())+ .map(|i| 1.0 / (i + 1) as f64)+ .sum();++ for (i, path) in host_ranks_paths.into_iter().enumerate() {+ let file = File::open(path)?;+ let reader = BufReader::new(file);+ let reader = BufReader::new(MultiGzDecoder::new(reader));++ for line in reader.lines().skip(1) {+ let line = line?;+ let rev_host = line.split('\t').last().unwrap();+ let host: String = itertools::intersperse(rev_host.split('.').rev(), ".").collect();+ let current_score: f64 = scores.get(&host).unwrap_or_default();+ scores.insert(host, current_score + ((1.0 / (i + 1) as f64) / norm));+ }+ }++ scores.flush();++ Ok(Self { scores })+ }++ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ let hosts = RocksDbStore::open(path);+ Self { scores: hosts }+ }++ pub fn get(&self, host: &String) -> Option<f64> {+ self.scores.get(host)+ }+}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 73ec6f04..f65ec2fb 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -35,6 +35,8 @@ use crate::{HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig, Result, WarcSource,};+use super::crawl_stability::CrawlStability;+pub struct Indexer {}#[derive(Debug, Serialize, Deserialize, Clone)]@@ -56,10 +58,15 @@ pub struct Job {pub struct IndexingWorker {centrality_store: CentralityStore,webgraph: Option<Webgraph>,+ crawl_stabilty: Option<CrawlStability>,}impl IndexingWorker {- pub fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {+ pub fn new(+ centrality_store_path: String,+ webgraph_path: Option<String>,+ crawl_stability_path: Option<String>,+ ) -> Self {Self {centrality_store: CentralityStore::open(centrality_store_path),webgraph: webgraph_path.map(|path| {@@ -69,6 +76,7 @@ impl IndexingWorker {.read_only(true).open()}),+ crawl_stabilty: crawl_stability_path.map(CrawlStability::open),}}}@@ -124,6 +132,12 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {}}+ let crawl_stability = worker+ .crawl_stabilty+ .as_ref()+ .and_then(|stability| stability.get(&html.url().site().to_string()))+ .unwrap_or_default();+html.parse_text();let backlinks: Vec<Link> = worker@@ -172,6 +186,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {primary_image: None,pre_computed_score: 0.0,node_id,+ crawl_stability,};webpage.pre_computed_score =@@ -327,13 +342,14 @@ impl Indexer {worker_addr: String,centrality_store_path: String,webgraph_path: Option<String>,+ crawl_stability_path: Option<String>,) -> Result<()> {tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap().block_on(async {- IndexingWorker::new(centrality_store_path, webgraph_path)+ IndexingWorker::new(centrality_store_path, webgraph_path, crawl_stability_path).run::<Job, FrozenIndex>(worker_addr.parse::<SocketAddr>()@@ -356,6 +372,7 @@ impl Indexer {let worker = IndexingWorker::new(config.centrality_store_path.clone(),config.webgraph_path.clone(),+ config.crawl_stability_path.clone(),);let indexes: Vec<_> = warc_pathsdiff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex 549241f3..eb8d8982 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -17,6 +17,7 @@ pub mod autosuggest_scrape;mod centrality;#[cfg(feature = "dev")]pub mod configure;+pub mod crawl_stability;mod entity;pub mod frontend;pub mod indexer;diff --git a/src/inverted_index.rs b/src/inverted_index.rsindex 1f7c5ede..688fefd0 100644--- a/src/inverted_index.rs+++ b/src/inverted_index.rs@@ -723,6 +723,7 @@ mod tests {pre_computed_score: 0.0,primary_image: None,node_id: None,+ crawl_stability: 0.0,}).expect("failed to insert webpage");diff --git a/src/lib.rs b/src/lib.rsindex 4cb0486c..1a48fd9f 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -77,6 +77,7 @@ pub struct IndexingMasterConfig {#[derive(Debug, Deserialize, Clone)]pub struct IndexingLocalConfig {+ crawl_stability_path: Option<String>,limit_warc_files: Option<usize>,final_num_segments: Option<u32>,warc_source: WarcSource,diff --git a/src/main.rs b/src/main.rsindex f780f45d..14557cde 100644--- a/src/main.rs+++ b/src/main.rs@@ -45,6 +45,10 @@ enum Commands {webgraph_path: String,output_path: String,},+ CrawlStability {+ output_path: String,+ host_rank_paths: Vec<String>,+ },Webgraph {#[clap(subcommand)]options: WebgraphOptions,@@ -81,6 +85,7 @@ enum IndexingOptions {address: String,centrality_store_path: String,webgraph_path: Option<String>,+ crawl_stability_path: Option<String>,},Local {config_path: String,@@ -118,8 +123,14 @@ fn main() -> Result<()> {address,centrality_store_path,webgraph_path,+ crawl_stability_path,} => {- entrypoint::Indexer::run_worker(address, centrality_store_path, webgraph_path)?;+ entrypoint::Indexer::run_worker(+ address,+ centrality_store_path,+ webgraph_path,+ crawl_stability_path,+ )?;}IndexingOptions::Local { config_path } => {let config = load_toml_config(&config_path);@@ -184,6 +195,13 @@ fn main() -> Result<()> {}#[cfg(feature = "dev")]Commands::Configure => configure::run()?,+ Commands::CrawlStability {+ output_path,+ host_rank_paths,+ } => {+ entrypoint::crawl_stability::CrawlStability::build(host_rank_paths, output_path)+ .unwrap();+ }}Ok(())diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex 4b5023d4..62adc9fb 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -441,6 +441,7 @@ mod tests {page_centrality: 0.0,fetch_time_ms: 500,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -468,6 +469,7 @@ mod tests {page_centrality: 0.0,primary_image: None,pre_computed_score: 0.0,+ crawl_stability: 0.0,fetch_time_ms: 500,node_id: None,})@@ -574,6 +576,7 @@ mod tests {page_centrality: 0.0,fetch_time_ms: 500,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -600,6 +603,7 @@ mod tests {page_centrality: 0.0,primary_image: None,pre_computed_score: 0.0,+ crawl_stability: 0.0,fetch_time_ms: 500,node_id: None,})@@ -685,6 +689,7 @@ mod tests {page_centrality: 0.0,fetch_time_ms: 500,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -712,6 +717,7 @@ mod tests {page_centrality: 0.0,primary_image: None,pre_computed_score: 0.0,+ crawl_stability: 0.0,fetch_time_ms: 500,node_id: None,})@@ -739,6 +745,7 @@ mod tests {page_centrality: 0.0,primary_image: None,pre_computed_score: 0.0,+ crawl_stability: 0.0,fetch_time_ms: 500,node_id: None,})@@ -821,6 +828,7 @@ mod tests {page_centrality: 0.0,fetch_time_ms: 500,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: Some(*centrality_store@@ -854,6 +862,7 @@ mod tests {page_centrality: 0.0,primary_image: None,pre_computed_score: 0.0,+ crawl_stability: 0.0,fetch_time_ms: 500,node_id: Some(*centrality_store@@ -887,6 +896,7 @@ mod tests {page_centrality: 0.0,primary_image: None,pre_computed_score: 0.0,+ crawl_stability: 0.0,fetch_time_ms: 500,node_id: Some(*centrality_storediff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex df15de30..1b289256 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -150,6 +150,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -177,6 +178,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -228,6 +230,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -255,6 +258,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 5.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -305,6 +309,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -332,6 +337,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -357,6 +363,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -407,6 +414,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -433,6 +441,7 @@ mod tests {host_centrality: 0.09,fetch_time_ms: 500,pre_computed_score: 0.0,+ crawl_stability: 0.0,page_centrality: 0.0,primary_image: None,node_id: None,@@ -480,6 +489,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -516,6 +526,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -567,6 +578,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -590,6 +602,7 @@ mod tests {host_centrality: 0.00003,fetch_time_ms: 500,pre_computed_score: 0.0,+ crawl_stability: 0.0,page_centrality: 0.0,primary_image: None,node_id: None,@@ -637,6 +650,7 @@ mod tests {host_centrality: 1.0,fetch_time_ms: 20,pre_computed_score: 0.0,+ crawl_stability: 0.0,page_centrality: 0.0,primary_image: None,node_id: None,@@ -663,6 +677,7 @@ mod tests {fetch_time_ms: 20,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -689,6 +704,7 @@ mod tests {fetch_time_ms: 500,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})@@ -785,6 +801,7 @@ mod tests {backlinks: vec![],host_centrality: 1.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,fetch_time_ms: 500,page_centrality: 0.0,primary_image: None,@@ -813,6 +830,7 @@ mod tests {host_centrality: 1.0,fetch_time_ms: 500,pre_computed_score: 0.0,+ crawl_stability: 0.0,page_centrality: 0.0,primary_image: None,node_id: None,@@ -840,6 +858,7 @@ mod tests {host_centrality: 1.0,fetch_time_ms: 500,pre_computed_score: 0.0,+ crawl_stability: 0.0,page_centrality: 0.0,primary_image: None,node_id: None,@@ -894,6 +913,7 @@ mod tests {host_centrality: 1.0,fetch_time_ms: 0,pre_computed_score: 0.0,+ crawl_stability: 0.0,page_centrality: 0.0,primary_image: None,node_id: None,@@ -922,6 +942,89 @@ mod tests {fetch_time_ms: 5000,page_centrality: 0.0,pre_computed_score: 0.0,+ crawl_stability: 0.0,+ primary_image: None,+ node_id: None,+ })+ .expect("failed to insert webpage");+ index.commit().expect("failed to commit index");+ let searcher = LocalSearcher::new(index);++ let result = searcher+ .search(&SearchQuery {+ original: "test".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ site_rankings: None,+ })+ .expect("Search failed")+ .into_websites()+ .unwrap()+ .webpages;++ assert_eq!(result.num_docs, 2);+ assert_eq!(result.documents.len(), 2);+ assert_eq!(result.documents[0].url, "https://www.first.com");+ assert_eq!(result.documents[1].url, "https://www.second.com");+ }++ #[test]+ fn crawl_stability() {+ let mut index = Index::temporary().expect("Unable to open index");++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.first.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 1000,+ pre_computed_score: 0.0,+ crawl_stability: 1.0,+ page_centrality: 0.0,+ primary_image: None,+ node_id: None,+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.second.com",+ ),+ backlinks: vec![],+ host_centrality: 1.0,+ fetch_time_ms: 0,+ page_centrality: 0.0,+ pre_computed_score: 0.0,+ crawl_stability: 0.0,primary_image: None,node_id: None,})diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex 2326bb33..f3a5a87d 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -27,7 +27,7 @@ use chrono::Utc;use tantivy::{DocId, Score};use crate::{- schema::{Field, CENTRALITY_SCALING},+ schema::{Field, FLOAT_SCALING},webpage::region::{Region, RegionCount},};@@ -44,9 +44,10 @@ pub enum Signal {NumTrackers,Region,PersonalCentrality,+ CrawlStability,}-pub const ALL_SIGNALS: [Signal; 9] = [+pub const ALL_SIGNALS: [Signal; 10] = [Signal::Bm25,Signal::HostCentrality,Signal::PageCentrality,@@ -56,6 +57,7 @@ pub const ALL_SIGNALS: [Signal; 9] = [Signal::NumTrackers,Signal::Region,Signal::PersonalCentrality,+ Signal::CrawlStability,];struct SignalValue {@@ -80,7 +82,7 @@ impl Signal {match self {Signal::Bm25 => value.bm25 as f64,Signal::HostCentrality | Signal::PageCentrality => {- value.fastfield_value.unwrap() as f64 / CENTRALITY_SCALING as f64+ value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64}Signal::PersonalCentrality => value.personal_centrality,Signal::IsHomepage => value.fastfield_value.unwrap() as f64,@@ -126,6 +128,7 @@ impl Signal {boost + region_count.score(&webpage_region)}+ Signal::CrawlStability => value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64,}}@@ -135,11 +138,12 @@ impl Signal {Signal::HostCentrality => 2048.0,Signal::PageCentrality => 4096.0,Signal::IsHomepage => 0.1,- Signal::FetchTimeMs => 0.1,+ Signal::FetchTimeMs => 0.01,Signal::UpdateTimestamp => 80.0,Signal::NumTrackers => 20.0,Signal::Region => 60.0,Signal::PersonalCentrality => 2048.0,+ Signal::CrawlStability => 20.0,}}@@ -168,6 +172,7 @@ impl Signal {Signal::UpdateTimestamp => Some(FastField::LastUpdated),Signal::NumTrackers => Some(FastField::NumTrackers),Signal::Region => Some(FastField::Region),+ Signal::CrawlStability => Some(FastField::CrawlStability),Signal::PersonalCentrality => None,}}@@ -331,10 +336,10 @@ impl SignalAggregator {.map(|signal| {let fastfield_value = match &signal {Signal::HostCentrality => {- (webpage.host_centrality * (CENTRALITY_SCALING as f64)) as u64+ (webpage.host_centrality * (FLOAT_SCALING as f64)) as u64}Signal::PageCentrality => {- (webpage.page_centrality * (CENTRALITY_SCALING as f64)) as u64+ (webpage.page_centrality * (FLOAT_SCALING as f64)) as u64}Signal::IsHomepage => webpage.html.url().is_homepage().into(),Signal::FetchTimeMs => webpage.fetch_time_ms,diff --git a/src/ranking/site_rankings.rs b/src/ranking/site_rankings.rsindex 5a624ab7..52da2aa2 100644--- a/src/ranking/site_rankings.rs+++ b/src/ranking/site_rankings.rs@@ -151,6 +151,7 @@ mod tests {pre_computed_score: 0.0,page_centrality: 0.0,primary_image: None,+ crawl_stability: 0.0,node_id: Some(graph.host@@ -185,6 +186,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ crawl_stability: 0.0,node_id: Some(graph.host@@ -219,6 +221,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,primary_image: None,+ crawl_stability: 0.0,node_id: Some(graph.hostdiff --git a/src/schema.rs b/src/schema.rsindex 3ea4606a..fe040981 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -20,7 +20,7 @@ use tantivy::schema::{use crate::tokenizer::{Identity, Normal, Stemmed};-pub const CENTRALITY_SCALING: u64 = 1_000_000_000;+pub const FLOAT_SCALING: u64 = 1_000_000_000;#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum TextField {@@ -66,6 +66,7 @@ pub enum FastField {PreComputedScore,HostNodeID,SimHash,+ CrawlStability,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]@@ -74,7 +75,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 35] = [+pub static ALL_FIELDS: [Field; 36] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -111,6 +112,7 @@ pub static ALL_FIELDS: [Field; 35] = [Field::Fast(FastField::PreComputedScore),Field::Fast(FastField::HostNodeID),Field::Fast(FastField::SimHash),+ Field::Fast(FastField::CrawlStability),];impl Field {@@ -264,6 +266,9 @@ impl Field {.set_indexed().set_stored(),),+ Field::Fast(FastField::CrawlStability) => IndexingOption::Integer(+ NumericOptions::default().set_fast(Cardinality::SingleValue),+ ),}}@@ -306,6 +311,7 @@ impl Field {Field::Fast(FastField::DomainHash) => "domain_hash",Field::Fast(FastField::HostNodeID) => "host_node_id",Field::Fast(FastField::SimHash) => "sim_hash",+ Field::Fast(FastField::CrawlStability) => "crawl_stability",}}@@ -376,6 +382,7 @@ impl Field {"title_hash" => Some(Field::Fast(FastField::TitleHash)),"host_node_id" => Some(Field::Fast(FastField::HostNodeID)),"sim_hash" => Some(Field::Fast(FastField::SimHash)),+ "crawl_stability" => Some(Field::Fast(FastField::CrawlStability)),_ => None,}}@@ -445,6 +452,7 @@ impl FastField {FastField::PreComputedScore => DataType::F64,FastField::HostNodeID => DataType::U64,FastField::SimHash => DataType::U64,+ FastField::CrawlStability => DataType::U64,}}}diff --git a/src/searcher/local.rs b/src/searcher/local.rsindex 736d6b09..6380b354 100644--- a/src/searcher/local.rs+++ b/src/searcher/local.rs@@ -287,6 +287,7 @@ mod tests {pre_computed_score: 0.0,primary_image: None,node_id: None,+ crawl_stability: 0.0,}).expect("failed to insert webpage");}diff --git a/src/ttl_cache.rs b/src/ttl_cache.rsindex 350e237d..c775e4c3 100644--- a/src/ttl_cache.rs+++ b/src/ttl_cache.rs@@ -100,21 +100,21 @@ mod tests {#[test]fn simple() {- let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(5), 5);+ let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(50), 5);cache.insert(0, 0);- std::thread::sleep(Duration::from_millis(3));+ std::thread::sleep(Duration::from_millis(30));cache.insert(1, 1);assert_eq!(cache.get(&0), Some(&0));assert_eq!(cache.get(&1), Some(&1));- std::thread::sleep(Duration::from_millis(3));+ std::thread::sleep(Duration::from_millis(30));assert_eq!(cache.get(&0), None);assert_eq!(cache.get(&1), Some(&1));- std::thread::sleep(Duration::from_millis(3));+ std::thread::sleep(Duration::from_millis(30));assert_eq!(cache.get(&0), None);assert_eq!(cache.get(&1), None);@@ -128,7 +128,7 @@ mod tests {#[test]fn max_size() {- let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(5), 1);+ let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(50), 1);cache.insert(0, 0);cache.insert(1, 1);diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex fe19a422..953c5bfa 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -37,7 +37,7 @@ mod just_text;pub mod region;mod url;-use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};+use crate::schema::{Field, ALL_FIELDS, FLOAT_SCALING};pub use self::url::Url;use self::{@@ -129,6 +129,7 @@ pub struct Webpage {pub pre_computed_score: f64,pub primary_image: Option<StoredPrimaryImage>,pub node_id: Option<u64>,+ pub crawl_stability: f64,}impl Webpage {@@ -145,6 +146,7 @@ impl Webpage {pre_computed_score: 0.0,primary_image: None,node_id: None,+ crawl_stability: 0.0,}}@@ -186,14 +188,14 @@ impl Webpage {schema.get_field(Field::Fast(FastField::HostCentrality).name()).expect("Failed to get host_centrality field"),- (self.host_centrality * CENTRALITY_SCALING as f64) as u64,+ (self.host_centrality * FLOAT_SCALING as f64) as u64,);doc.add_u64(schema.get_field(Field::Fast(FastField::PageCentrality).name()).expect("Failed to get page_centrality field"),- (self.page_centrality * CENTRALITY_SCALING as f64) as u64,+ (self.page_centrality * FLOAT_SCALING as f64) as u64,);doc.add_u64(@@ -225,6 +227,13 @@ impl Webpage {self.node_id.unwrap_or(u64::MAX),);+ doc.add_u64(+ schema+ .get_field(Field::Fast(FastField::CrawlStability).name())+ .expect("failed to get crawl_stability field"),+ (self.crawl_stability * FLOAT_SCALING as f64) as u64,+ );+Ok(doc)}@@ -715,6 +724,7 @@ impl Html {| Field::Fast(FastField::PreComputedScore)| Field::Fast(FastField::Region)| Field::Fast(FastField::HostNodeID)+ | Field::Fast(FastField::CrawlStability)| Field::Text(TextField::PrimaryImage) => {}}}