Skip to content

web search done right

Crawl stability field

Mikkel Denkerce7a3f35993b
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index 81f75d4d..811b225a 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,11 +1,12 @@
centrality_store_path = "./data/centrality"
limit_warc_files = 1
download_images = false
+# crawl_stability_path = "data/crawl-stability"
[warc_source]
-# type = "Local"
-# folder = "./data/warc_files"
-# names = ["CC-MAIN-20220116093137-20220116123137-00049.warc.gz"]
-type = "HTTP"
-base_url = "https://data.commoncrawl.org/"
-warc_paths_file = "warc.paths"
+type = "Local"
+folder = "./data"
+names = ["sample.warc.gz"]
+# type = "HTTP"
+# base_url = "https://data.commoncrawl.org/"
+# warc_paths_file = "warc.paths"
diff --git a/src/entrypoint/crawl_stability.rs b/src/entrypoint/crawl_stability.rs
new file mode 100644
index 00000000..86760192
--- /dev/null
+++ b/src/entrypoint/crawl_stability.rs
@@ -0,0 +1,68 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use flate2::bufread::MultiGzDecoder;
+
+use crate::{
+ kv::{rocksdb_store::RocksDbStore, Kv},
+ Result,
+};
+use std::{
+ fs::File,
+ io::{BufRead, BufReader},
+ path::Path,
+};
+
+pub struct CrawlStability {
+ scores: Box<dyn Kv<String, f64>>,
+}
+
+impl CrawlStability {
+ pub fn build<P: AsRef<Path>>(host_ranks_paths: Vec<String>, path: P) -> Result<Self> {
+ let scores = RocksDbStore::open(path.as_ref());
+
+ let norm: f64 = (0..host_ranks_paths.len())
+ .map(|i| 1.0 / (i + 1) as f64)
+ .sum();
+
+ for (i, path) in host_ranks_paths.into_iter().enumerate() {
+ let file = File::open(path)?;
+ let reader = BufReader::new(file);
+ let reader = BufReader::new(MultiGzDecoder::new(reader));
+
+ for line in reader.lines().skip(1) {
+ let line = line?;
+ let rev_host = line.split('\t').last().unwrap();
+ let host: String = itertools::intersperse(rev_host.split('.').rev(), ".").collect();
+ let current_score: f64 = scores.get(&host).unwrap_or_default();
+ scores.insert(host, current_score + ((1.0 / (i + 1) as f64) / norm));
+ }
+ }
+
+ scores.flush();
+
+ Ok(Self { scores })
+ }
+
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ let hosts = RocksDbStore::open(path);
+ Self { scores: hosts }
+ }
+
+ pub fn get(&self, host: &String) -> Option<f64> {
+ self.scores.get(host)
+ }
+}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 73ec6f04..f65ec2fb 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -35,6 +35,8 @@ use crate::{
HttpConfig, IndexingLocalConfig, IndexingMasterConfig, LocalConfig, Result, WarcSource,
};
+use super::crawl_stability::CrawlStability;
+
pub struct Indexer {}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -56,10 +58,15 @@ pub struct Job {
pub struct IndexingWorker {
centrality_store: CentralityStore,
webgraph: Option<Webgraph>,
+ crawl_stabilty: Option<CrawlStability>,
}
impl IndexingWorker {
- pub fn new(centrality_store_path: String, webgraph_path: Option<String>) -> Self {
+ pub fn new(
+ centrality_store_path: String,
+ webgraph_path: Option<String>,
+ crawl_stability_path: Option<String>,
+ ) -> Self {
Self {
centrality_store: CentralityStore::open(centrality_store_path),
webgraph: webgraph_path.map(|path| {
@@ -69,6 +76,7 @@ impl IndexingWorker {
.read_only(true)
.open()
}),
+ crawl_stabilty: crawl_stability_path.map(CrawlStability::open),
}
}
}
@@ -124,6 +132,12 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
}
}
+ let crawl_stability = worker
+ .crawl_stabilty
+ .as_ref()
+ .and_then(|stability| stability.get(&html.url().site().to_string()))
+ .unwrap_or_default();
+
html.parse_text();
let backlinks: Vec<Link> = worker
@@ -172,6 +186,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
primary_image: None,
pre_computed_score: 0.0,
node_id,
+ crawl_stability,
};
webpage.pre_computed_score =
@@ -327,13 +342,14 @@ impl Indexer {
worker_addr: String,
centrality_store_path: String,
webgraph_path: Option<String>,
+ crawl_stability_path: Option<String>,
) -> Result<()> {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
- IndexingWorker::new(centrality_store_path, webgraph_path)
+ IndexingWorker::new(centrality_store_path, webgraph_path, crawl_stability_path)
.run::<Job, FrozenIndex>(
worker_addr
.parse::<SocketAddr>()
@@ -356,6 +372,7 @@ impl Indexer {
let worker = IndexingWorker::new(
config.centrality_store_path.clone(),
config.webgraph_path.clone(),
+ config.crawl_stability_path.clone(),
);
let indexes: Vec<_> = warc_paths
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index 549241f3..eb8d8982 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -17,6 +17,7 @@ pub mod autosuggest_scrape;
mod centrality;
#[cfg(feature = "dev")]
pub mod configure;
+pub mod crawl_stability;
mod entity;
pub mod frontend;
pub mod indexer;
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 1f7c5ede..688fefd0 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -723,6 +723,7 @@ mod tests {
pre_computed_score: 0.0,
primary_image: None,
node_id: None,
+ crawl_stability: 0.0,
})
.expect("failed to insert webpage");
diff --git a/src/lib.rs b/src/lib.rs
index 4cb0486c..1a48fd9f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -77,6 +77,7 @@ pub struct IndexingMasterConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingLocalConfig {
+ crawl_stability_path: Option<String>,
limit_warc_files: Option<usize>,
final_num_segments: Option<u32>,
warc_source: WarcSource,
diff --git a/src/main.rs b/src/main.rs
index f780f45d..14557cde 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -45,6 +45,10 @@ enum Commands {
webgraph_path: String,
output_path: String,
},
+ CrawlStability {
+ output_path: String,
+ host_rank_paths: Vec<String>,
+ },
Webgraph {
#[clap(subcommand)]
options: WebgraphOptions,
@@ -81,6 +85,7 @@ enum IndexingOptions {
address: String,
centrality_store_path: String,
webgraph_path: Option<String>,
+ crawl_stability_path: Option<String>,
},
Local {
config_path: String,
@@ -118,8 +123,14 @@ fn main() -> Result<()> {
address,
centrality_store_path,
webgraph_path,
+ crawl_stability_path,
} => {
- entrypoint::Indexer::run_worker(address, centrality_store_path, webgraph_path)?;
+ entrypoint::Indexer::run_worker(
+ address,
+ centrality_store_path,
+ webgraph_path,
+ crawl_stability_path,
+ )?;
}
IndexingOptions::Local { config_path } => {
let config = load_toml_config(&config_path);
@@ -184,6 +195,13 @@ fn main() -> Result<()> {
}
#[cfg(feature = "dev")]
Commands::Configure => configure::run()?,
+ Commands::CrawlStability {
+ output_path,
+ host_rank_paths,
+ } => {
+ entrypoint::crawl_stability::CrawlStability::build(host_rank_paths, output_path)
+ .unwrap();
+ }
}
Ok(())
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 4b5023d4..62adc9fb 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -441,6 +441,7 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -468,6 +469,7 @@ mod tests {
page_centrality: 0.0,
primary_image: None,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
})
@@ -574,6 +576,7 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -600,6 +603,7 @@ mod tests {
page_centrality: 0.0,
primary_image: None,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
})
@@ -685,6 +689,7 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -712,6 +717,7 @@ mod tests {
page_centrality: 0.0,
primary_image: None,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
})
@@ -739,6 +745,7 @@ mod tests {
page_centrality: 0.0,
primary_image: None,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
})
@@ -821,6 +828,7 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: Some(
*centrality_store
@@ -854,6 +862,7 @@ mod tests {
page_centrality: 0.0,
primary_image: None,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: Some(
*centrality_store
@@ -887,6 +896,7 @@ mod tests {
page_centrality: 0.0,
primary_image: None,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: Some(
*centrality_store
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index df15de30..1b289256 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -150,6 +150,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -177,6 +178,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -228,6 +230,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -255,6 +258,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 5.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -305,6 +309,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -332,6 +337,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -357,6 +363,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -407,6 +414,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -433,6 +441,7 @@ mod tests {
host_centrality: 0.09,
fetch_time_ms: 500,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
node_id: None,
@@ -480,6 +489,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -516,6 +526,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -567,6 +578,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -590,6 +602,7 @@ mod tests {
host_centrality: 0.00003,
fetch_time_ms: 500,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
node_id: None,
@@ -637,6 +650,7 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 20,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
node_id: None,
@@ -663,6 +677,7 @@ mod tests {
fetch_time_ms: 20,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -689,6 +704,7 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
@@ -785,6 +801,7 @@ mod tests {
backlinks: vec![],
host_centrality: 1.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
primary_image: None,
@@ -813,6 +830,7 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
node_id: None,
@@ -840,6 +858,7 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
node_id: None,
@@ -894,6 +913,7 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
page_centrality: 0.0,
primary_image: None,
node_id: None,
@@ -922,6 +942,89 @@ mod tests {
fetch_time_ms: 5000,
page_centrality: 0.0,
pre_computed_score: 0.0,
+ crawl_stability: 0.0,
+ primary_image: None,
+ node_id: None,
+ })
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::new(index);
+
+ let result = searcher
+ .search(&SearchQuery {
+ original: "test".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ site_rankings: None,
+ })
+ .expect("Search failed")
+ .into_websites()
+ .unwrap()
+ .webpages;
+
+ assert_eq!(result.num_docs, 2);
+ assert_eq!(result.documents.len(), 2);
+ assert_eq!(result.documents[0].url, "https://www.first.com");
+ assert_eq!(result.documents[1].url, "https://www.second.com");
+ }
+
+ #[test]
+ fn crawl_stability() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.first.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 1000,
+ pre_computed_score: 0.0,
+ crawl_stability: 1.0,
+ page_centrality: 0.0,
+ primary_image: None,
+ node_id: None,
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ {CONTENT} {}
+ </body>
+ </html>
+ "#,
+ crate::rand_words(100)
+ ),
+ "https://www.second.com",
+ ),
+ backlinks: vec![],
+ host_centrality: 1.0,
+ fetch_time_ms: 0,
+ page_centrality: 0.0,
+ pre_computed_score: 0.0,
+ crawl_stability: 0.0,
primary_image: None,
node_id: None,
})
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index 2326bb33..f3a5a87d 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -27,7 +27,7 @@ use chrono::Utc;
use tantivy::{DocId, Score};
use crate::{
- schema::{Field, CENTRALITY_SCALING},
+ schema::{Field, FLOAT_SCALING},
webpage::region::{Region, RegionCount},
};
@@ -44,9 +44,10 @@ pub enum Signal {
NumTrackers,
Region,
PersonalCentrality,
+ CrawlStability,
}
-pub const ALL_SIGNALS: [Signal; 9] = [
+pub const ALL_SIGNALS: [Signal; 10] = [
Signal::Bm25,
Signal::HostCentrality,
Signal::PageCentrality,
@@ -56,6 +57,7 @@ pub const ALL_SIGNALS: [Signal; 9] = [
Signal::NumTrackers,
Signal::Region,
Signal::PersonalCentrality,
+ Signal::CrawlStability,
];
struct SignalValue {
@@ -80,7 +82,7 @@ impl Signal {
match self {
Signal::Bm25 => value.bm25 as f64,
Signal::HostCentrality | Signal::PageCentrality => {
- value.fastfield_value.unwrap() as f64 / CENTRALITY_SCALING as f64
+ value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64
}
Signal::PersonalCentrality => value.personal_centrality,
Signal::IsHomepage => value.fastfield_value.unwrap() as f64,
@@ -126,6 +128,7 @@ impl Signal {
boost + region_count.score(&webpage_region)
}
+ Signal::CrawlStability => value.fastfield_value.unwrap() as f64 / FLOAT_SCALING as f64,
}
}
@@ -135,11 +138,12 @@ impl Signal {
Signal::HostCentrality => 2048.0,
Signal::PageCentrality => 4096.0,
Signal::IsHomepage => 0.1,
- Signal::FetchTimeMs => 0.1,
+ Signal::FetchTimeMs => 0.01,
Signal::UpdateTimestamp => 80.0,
Signal::NumTrackers => 20.0,
Signal::Region => 60.0,
Signal::PersonalCentrality => 2048.0,
+ Signal::CrawlStability => 20.0,
}
}
@@ -168,6 +172,7 @@ impl Signal {
Signal::UpdateTimestamp => Some(FastField::LastUpdated),
Signal::NumTrackers => Some(FastField::NumTrackers),
Signal::Region => Some(FastField::Region),
+ Signal::CrawlStability => Some(FastField::CrawlStability),
Signal::PersonalCentrality => None,
}
}
@@ -331,10 +336,10 @@ impl SignalAggregator {
.map(|signal| {
let fastfield_value = match &signal {
Signal::HostCentrality => {
- (webpage.host_centrality * (CENTRALITY_SCALING as f64)) as u64
+ (webpage.host_centrality * (FLOAT_SCALING as f64)) as u64
}
Signal::PageCentrality => {
- (webpage.page_centrality * (CENTRALITY_SCALING as f64)) as u64
+ (webpage.page_centrality * (FLOAT_SCALING as f64)) as u64
}
Signal::IsHomepage => webpage.html.url().is_homepage().into(),
Signal::FetchTimeMs => webpage.fetch_time_ms,
diff --git a/src/ranking/site_rankings.rs b/src/ranking/site_rankings.rs
index 5a624ab7..52da2aa2 100644
--- a/src/ranking/site_rankings.rs
+++ b/src/ranking/site_rankings.rs
@@ -151,6 +151,7 @@ mod tests {
pre_computed_score: 0.0,
page_centrality: 0.0,
primary_image: None,
+ crawl_stability: 0.0,
node_id: Some(
graph
.host
@@ -185,6 +186,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
primary_image: None,
+ crawl_stability: 0.0,
node_id: Some(
graph
.host
@@ -219,6 +221,7 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
primary_image: None,
+ crawl_stability: 0.0,
node_id: Some(
graph
.host
diff --git a/src/schema.rs b/src/schema.rs
index 3ea4606a..fe040981 100644
--- a/src/schema.rs
+++ b/src/schema.rs
@@ -20,7 +20,7 @@ use tantivy::schema::{
use crate::tokenizer::{Identity, Normal, Stemmed};
-pub const CENTRALITY_SCALING: u64 = 1_000_000_000;
+pub const FLOAT_SCALING: u64 = 1_000_000_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TextField {
@@ -66,6 +66,7 @@ pub enum FastField {
PreComputedScore,
HostNodeID,
SimHash,
+ CrawlStability,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -74,7 +75,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 35] = [
+pub static ALL_FIELDS: [Field; 36] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -111,6 +112,7 @@ pub static ALL_FIELDS: [Field; 35] = [
Field::Fast(FastField::PreComputedScore),
Field::Fast(FastField::HostNodeID),
Field::Fast(FastField::SimHash),
+ Field::Fast(FastField::CrawlStability),
];
impl Field {
@@ -264,6 +266,9 @@ impl Field {
.set_indexed()
.set_stored(),
),
+ Field::Fast(FastField::CrawlStability) => IndexingOption::Integer(
+ NumericOptions::default().set_fast(Cardinality::SingleValue),
+ ),
}
}
@@ -306,6 +311,7 @@ impl Field {
Field::Fast(FastField::DomainHash) => "domain_hash",
Field::Fast(FastField::HostNodeID) => "host_node_id",
Field::Fast(FastField::SimHash) => "sim_hash",
+ Field::Fast(FastField::CrawlStability) => "crawl_stability",
}
}
@@ -376,6 +382,7 @@ impl Field {
"title_hash" => Some(Field::Fast(FastField::TitleHash)),
"host_node_id" => Some(Field::Fast(FastField::HostNodeID)),
"sim_hash" => Some(Field::Fast(FastField::SimHash)),
+ "crawl_stability" => Some(Field::Fast(FastField::CrawlStability)),
_ => None,
}
}
@@ -445,6 +452,7 @@ impl FastField {
FastField::PreComputedScore => DataType::F64,
FastField::HostNodeID => DataType::U64,
FastField::SimHash => DataType::U64,
+ FastField::CrawlStability => DataType::U64,
}
}
}
diff --git a/src/searcher/local.rs b/src/searcher/local.rs
index 736d6b09..6380b354 100644
--- a/src/searcher/local.rs
+++ b/src/searcher/local.rs
@@ -287,6 +287,7 @@ mod tests {
pre_computed_score: 0.0,
primary_image: None,
node_id: None,
+ crawl_stability: 0.0,
})
.expect("failed to insert webpage");
}
diff --git a/src/ttl_cache.rs b/src/ttl_cache.rs
index 350e237d..c775e4c3 100644
--- a/src/ttl_cache.rs
+++ b/src/ttl_cache.rs
@@ -100,21 +100,21 @@ mod tests {
#[test]
fn simple() {
- let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(5), 5);
+ let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(50), 5);
cache.insert(0, 0);
- std::thread::sleep(Duration::from_millis(3));
+ std::thread::sleep(Duration::from_millis(30));
cache.insert(1, 1);
assert_eq!(cache.get(&0), Some(&0));
assert_eq!(cache.get(&1), Some(&1));
- std::thread::sleep(Duration::from_millis(3));
+ std::thread::sleep(Duration::from_millis(30));
assert_eq!(cache.get(&0), None);
assert_eq!(cache.get(&1), Some(&1));
- std::thread::sleep(Duration::from_millis(3));
+ std::thread::sleep(Duration::from_millis(30));
assert_eq!(cache.get(&0), None);
assert_eq!(cache.get(&1), None);
@@ -128,7 +128,7 @@ mod tests {
#[test]
fn max_size() {
- let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(5), 1);
+ let mut cache = TTLCache::with_ttl_and_max_size(Duration::from_millis(50), 1);
cache.insert(0, 0);
cache.insert(1, 1);
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index fe19a422..953c5bfa 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -37,7 +37,7 @@ mod just_text;
pub mod region;
mod url;
-use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};
+use crate::schema::{Field, ALL_FIELDS, FLOAT_SCALING};
pub use self::url::Url;
use self::{
@@ -129,6 +129,7 @@ pub struct Webpage {
pub pre_computed_score: f64,
pub primary_image: Option<StoredPrimaryImage>,
pub node_id: Option<u64>,
+ pub crawl_stability: f64,
}
impl Webpage {
@@ -145,6 +146,7 @@ impl Webpage {
pre_computed_score: 0.0,
primary_image: None,
node_id: None,
+ crawl_stability: 0.0,
}
}
@@ -186,14 +188,14 @@ impl Webpage {
schema
.get_field(Field::Fast(FastField::HostCentrality).name())
.expect("Failed to get host_centrality field"),
- (self.host_centrality * CENTRALITY_SCALING as f64) as u64,
+ (self.host_centrality * FLOAT_SCALING as f64) as u64,
);
doc.add_u64(
schema
.get_field(Field::Fast(FastField::PageCentrality).name())
.expect("Failed to get page_centrality field"),
- (self.page_centrality * CENTRALITY_SCALING as f64) as u64,
+ (self.page_centrality * FLOAT_SCALING as f64) as u64,
);
doc.add_u64(
@@ -225,6 +227,13 @@ impl Webpage {
self.node_id.unwrap_or(u64::MAX),
);
+ doc.add_u64(
+ schema
+ .get_field(Field::Fast(FastField::CrawlStability).name())
+ .expect("failed to get crawl_stability field"),
+ (self.crawl_stability * FLOAT_SCALING as f64) as u64,
+ );
+
Ok(doc)
}
@@ -715,6 +724,7 @@ impl Html {
| Field::Fast(FastField::PreComputedScore)
| Field::Fast(FastField::Region)
| Field::Fast(FastField::HostNodeID)
+ | Field::Fast(FastField::CrawlStability)
| Field::Text(TextField::PrimaryImage) => {}
}
}