Skip to content

web search done right

removed crawl stability for simplicity

Mikkel Denker9268cdd7e8df
diff --git a/configs/indexer/local.toml b/configs/indexer/local.toml
index 9eb2afee..4636c551 100644
--- a/configs/indexer/local.toml
+++ b/configs/indexer/local.toml
@@ -1,7 +1,6 @@
centrality_store_path = "./data/centrality"
limit_warc_files = 10
minimum_clean_words = 40
-# crawl_stability_path = "data/crawl-stability"
[warc_source]
# type = "Local"
diff --git a/core/src/entrypoint/crawl_stability.rs b/core/src/entrypoint/crawl_stability.rs
deleted file mode 100644
index 07d9aa2f..00000000
--- a/core/src/entrypoint/crawl_stability.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use flate2::bufread::MultiGzDecoder;
-
-use crate::{
- kv::{rocksdb_store::RocksDbStore, Kv},
- Result,
-};
-use std::{
- fs::File,
- io::{BufRead, BufReader},
- path::Path,
-};
-
-pub struct CrawlStability {
- scores: Box<dyn Kv<String, f64>>,
-}
-
-impl CrawlStability {
- pub fn build<P: AsRef<Path>>(host_ranks_paths: Vec<String>, path: P) -> Result<Self> {
- let scores = RocksDbStore::open(path.as_ref());
-
- let norm: f64 = (0..host_ranks_paths.len())
- .map(|i| 1.0 / (i + 1) as f64)
- .sum();
-
- for (i, path) in host_ranks_paths.into_iter().enumerate() {
- let file = File::open(path)?;
- let reader = BufReader::new(file);
- let reader = BufReader::new(MultiGzDecoder::new(reader));
-
- for line in reader.lines().skip(1) {
- let line = line?;
- let rev_host = line.split('\t').last().unwrap();
- let host: String = itertools::intersperse(rev_host.split('.').rev(), ".").collect();
- let current_score: f64 = scores.get(&host).unwrap_or_default();
- scores.insert(host, current_score + ((1.0 / (i + 1) as f64) / norm));
- }
- }
-
- scores.flush();
-
- Ok(Self { scores })
- }
-
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- let hosts = RocksDbStore::open(path);
- Self { scores: hosts }
- }
-
- pub fn get(&self, host: &String) -> Option<f64> {
- self.scores.get(host)
- }
-}
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 43a91ff6..91e24cca 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -37,8 +37,6 @@ use crate::{
Result, WarcSource,
};
-use super::crawl_stability::CrawlStability;
-
pub struct Indexer {}
#[derive(Debug, Serialize, Deserialize, Clone)]
@@ -59,7 +57,6 @@ pub struct Job {
pub struct IndexingWorker {
centrality_store: IndexerCentralityStore,
webgraph: Option<Webgraph>,
- crawl_stabilty: Option<CrawlStability>,
topics: Option<human_website_annotations::Mapper>,
}
@@ -67,13 +64,11 @@ impl IndexingWorker {
pub fn new(
centrality_store_path: String,
webgraph_path: Option<String>,
- crawl_stability_path: Option<String>,
topics_path: Option<String>,
) -> Self {
Self {
centrality_store: IndexerCentralityStore::open(centrality_store_path),
webgraph: webgraph_path.map(|path| WebgraphBuilder::new(path).open()),
- crawl_stabilty: crawl_stability_path.map(CrawlStability::open),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
}
}
@@ -141,12 +136,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
}
}
- let crawl_stability = worker
- .crawl_stabilty
- .as_ref()
- .and_then(|stability| stability.get(&html.url().site().to_string()))
- .unwrap_or_default();
-
let backlinks: Vec<Link> = worker
.webgraph
.as_ref()
@@ -203,7 +192,6 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
fetch_time_ms,
pre_computed_score: 0.0,
node_id,
- crawl_stability,
host_topic,
dmoz_description,
};
@@ -345,7 +333,6 @@ impl Indexer {
worker_addr: String,
centrality_store_path: String,
webgraph_path: Option<String>,
- crawl_stability_path: Option<String>,
topics_path: Option<String>,
) -> Result<()> {
tokio::runtime::Builder::new_current_thread()
@@ -353,19 +340,14 @@ impl Indexer {
.build()
.unwrap()
.block_on(async {
- IndexingWorker::new(
- centrality_store_path,
- webgraph_path,
- crawl_stability_path,
- topics_path,
- )
- .run::<Job, FrozenIndex>(
- worker_addr
- .parse::<SocketAddr>()
- .expect("Could not parse worker address"),
- )
- .await
- .unwrap();
+ IndexingWorker::new(centrality_store_path, webgraph_path, topics_path)
+ .run::<Job, FrozenIndex>(
+ worker_addr
+ .parse::<SocketAddr>()
+ .expect("Could not parse worker address"),
+ )
+ .await
+ .unwrap();
});
Ok(())
}
@@ -381,7 +363,6 @@ impl Indexer {
let worker = IndexingWorker::new(
config.centrality_store_path.clone(),
config.webgraph_path.clone(),
- config.crawl_stability_path.clone(),
config.topics_path.clone(),
);
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index 12af3807..7c3ca0ad 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -18,7 +18,6 @@ pub mod autosuggest_scrape;
mod centrality;
#[cfg(feature = "dev")]
pub mod configure;
-pub mod crawl_stability;
pub mod crawler;
pub mod dmoz_parser;
mod entity;
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 8bf6a6bb..99b76cd6 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -852,7 +852,6 @@ mod tests {
pre_computed_score: 0.0,
node_id: None,
host_topic: None,
- crawl_stability: 0.0,
dmoz_description: None,
})
.expect("failed to insert webpage");
diff --git a/core/src/lib.rs b/core/src/lib.rs
index ec03b89b..b87bcf85 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -95,7 +95,6 @@ pub struct IndexingMasterConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingLocalConfig {
- crawl_stability_path: Option<String>,
limit_warc_files: Option<usize>,
skip_num_warc_files: Option<usize>,
warc_source: WarcSource,
diff --git a/core/src/main.rs b/core/src/main.rs
index 1ef81fa3..3a74c947 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -55,10 +55,6 @@ enum Commands {
webgraph_path: String,
output_path: String,
},
- CrawlStability {
- output_path: String,
- host_rank_paths: Vec<String>,
- },
DmozParser {
dmoz_file: String,
output_path: String,
@@ -147,7 +143,6 @@ enum IndexingOptions {
address: String,
centrality_store_path: String,
webgraph_path: Option<String>,
- crawl_stability_path: Option<String>,
topics_path: Option<String>,
},
Local {
@@ -185,14 +180,12 @@ fn main() -> Result<()> {
address,
centrality_store_path,
webgraph_path,
- crawl_stability_path,
topics_path,
} => {
entrypoint::Indexer::run_worker(
address,
centrality_store_path,
webgraph_path,
- crawl_stability_path,
topics_path,
)?;
}
@@ -300,13 +293,6 @@ fn main() -> Result<()> {
configure::run(skip_download)?;
}
}
- Commands::CrawlStability {
- output_path,
- host_rank_paths,
- } => {
- entrypoint::crawl_stability::CrawlStability::build(host_rank_paths, output_path)
- .unwrap();
- }
Commands::DmozParser {
dmoz_file,
output_path,
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index bf3bb015..eab037b4 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -265,7 +265,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -295,7 +294,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
host_topic: None,
@@ -395,7 +393,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -424,7 +421,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
host_topic: None,
@@ -502,7 +498,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -532,7 +527,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
dmoz_description: None,
@@ -562,7 +556,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
host_topic: None,
@@ -667,7 +660,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
dmoz_description: None,
@@ -697,7 +689,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
fetch_time_ms: 500,
dmoz_description: None,
host_topic: None,
@@ -727,7 +718,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
fetch_time_ms: 500,
host_topic: None,
dmoz_description: None,
@@ -804,7 +794,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -849,7 +838,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
fetch_time_ms: 500,
node_id: None,
dmoz_description: None,
@@ -985,7 +973,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1061,7 +1048,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1092,7 +1078,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1148,7 +1133,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1208,7 +1192,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1371,7 +1354,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1446,7 +1428,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1508,7 +1489,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1539,7 +1519,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
@@ -1639,7 +1618,6 @@ mod tests {
page_centrality: 0.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
node_id: None,
host_topic: None,
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 51b00ede..f58c9e59 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -332,7 +332,6 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: Some(*graph.node2id(&Node::from("e.com")).unwrap()),
@@ -362,7 +361,6 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: Some(*graph.node2id(&Node::from("d.com")).unwrap()),
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 9147cd38..ebb125cb 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -205,7 +205,6 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: None,
@@ -235,7 +234,6 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: None,
@@ -284,7 +282,6 @@ mod tests {
// fetch_time_ms: 500,
// page_centrality: 0.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: None,
// node_id: None,
@@ -314,7 +311,6 @@ mod tests {
// fetch_time_ms: 500,
// page_centrality: 5.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: None,
// node_id: None,
@@ -362,7 +358,6 @@ mod tests {
// fetch_time_ms: 500,
// page_centrality: 0.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: None,
// node_id: None,
@@ -392,7 +387,6 @@ mod tests {
// fetch_time_ms: 500,
// page_centrality: 0.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: None,
// node_id: None,
@@ -420,7 +414,6 @@ mod tests {
// fetch_time_ms: 500,
// page_centrality: 0.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: None,
// node_id: None,
@@ -468,7 +461,6 @@ mod tests {
// fetch_time_ms: 500,
// page_centrality: 0.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: None,
// node_id: None,
@@ -497,7 +489,6 @@ mod tests {
// host_centrality: 0.009,
// fetch_time_ms: 500,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
// page_centrality: 0.0,
//
// host_topic: None,
@@ -542,7 +533,6 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: None,
@@ -581,7 +571,6 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: None,
@@ -630,7 +619,6 @@ mod tests {
// fetch_time_ms: 500,
// page_centrality: 0.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: None,
// node_id: None,
@@ -656,7 +644,6 @@ mod tests {
// host_centrality: 0.00003,
// fetch_time_ms: 500,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
// page_centrality: 0.0,
//
// host_topic: None,
@@ -701,7 +688,6 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 20,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
page_centrality: 0.0,
host_topic: None,
@@ -730,7 +716,6 @@ mod tests {
fetch_time_ms: 20,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: None,
@@ -759,7 +744,6 @@ mod tests {
fetch_time_ms: 500,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: None,
@@ -832,7 +816,6 @@ mod tests {
backlinks: vec![],
host_centrality: 1.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
fetch_time_ms: 500,
page_centrality: 0.0,
@@ -867,7 +850,6 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
page_centrality: 0.0,
host_topic: None,
@@ -901,7 +883,6 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 500,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
page_centrality: 0.0,
host_topic: None,
@@ -957,7 +938,6 @@ mod tests {
host_centrality: 1.0,
fetch_time_ms: 0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
page_centrality: 0.0,
host_topic: None,
@@ -988,87 +968,6 @@ mod tests {
fetch_time_ms: 5000,
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
-
- host_topic: None,
- node_id: None,
- dmoz_description: None,
- })
- .expect("failed to insert webpage");
- index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::new(index);
-
- let result = searcher
- .search(&SearchQuery {
- query: "test".to_string(),
- ..Default::default()
- })
- .expect("Search failed");
-
- assert_eq!(result.num_hits, 2);
- assert_eq!(result.webpages.len(), 2);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
- assert_eq!(result.webpages[1].url, "https://www.second.com");
- }
-
- #[test]
- fn crawl_stability() {
- let mut index = Index::temporary().expect("Unable to open index");
-
- index
- .insert(Webpage {
- html: Html::parse(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT} {}
- </body>
- </html>
- "#,
- crate::rand_words(100)
- ),
- "https://www.first.com",
- ),
- backlinks: vec![],
- host_centrality: 1.0,
- fetch_time_ms: 1000,
- pre_computed_score: 0.0,
- crawl_stability: 1.0,
- page_centrality: 0.0,
-
- host_topic: None,
- node_id: None,
- dmoz_description: None,
- })
- .expect("failed to insert webpage");
- index
- .insert(Webpage {
- html: Html::parse(
- &format!(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- {CONTENT} {}
- </body>
- </html>
- "#,
- crate::rand_words(100)
- ),
- "https://www.second.com",
- ),
- backlinks: vec![],
- host_centrality: 1.0,
- fetch_time_ms: 0,
- page_centrality: 0.0,
- pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
node_id: None,
@@ -1124,7 +1023,6 @@ mod tests {
// host_centrality: 1.0,
// fetch_time_ms: 1000,
// pre_computed_score: 0.0,
- // crawl_stability: 1.0,
// page_centrality: 0.0,
//
// host_topic: Some(topic_a.clone()),
@@ -1160,7 +1058,6 @@ mod tests {
// fetch_time_ms: 1000,
// page_centrality: 0.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: Some(topic_b.clone()),
// node_id: Some(2.into()),
@@ -1195,7 +1092,6 @@ mod tests {
// fetch_time_ms: 0,
// page_centrality: 0.0,
// pre_computed_score: 0.0,
- // crawl_stability: 0.0,
//
// host_topic: None,
// node_id: Some(0.into()),
diff --git a/core/src/ranking/models/lambdamart.rs b/core/src/ranking/models/lambdamart.rs
index aad8d409..87555e51 100644
--- a/core/src/ranking/models/lambdamart.rs
+++ b/core/src/ranking/models/lambdamart.rs
@@ -290,6 +290,6 @@ mod tests {
features.insert(Signal::HostCentrality, 0.031143193);
features.insert(Signal::Bm25StemmedCleanBody, 52.1262779236);
- assert_eq!((tree.predict(&features).unwrap() * 1000.0) as u64, 1017);
+ assert_eq!((tree.predict(&features).unwrap() * 1000.0) as u64, 1016);
}
}
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 0188945a..30fd3451 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -86,7 +86,6 @@ mod tests {
pre_computed_score: 0.0,
page_centrality: 0.0,
- crawl_stability: 0.0,
dmoz_description: None,
host_topic: None,
node_id: Some(
@@ -121,7 +120,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
host_topic: None,
dmoz_description: None,
node_id: Some(
@@ -156,7 +154,6 @@ mod tests {
page_centrality: 0.0,
pre_computed_score: 0.0,
- crawl_stability: 0.0,
dmoz_description: None,
host_topic: None,
node_id: Some(
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 8ef73088..b78fa584 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -126,8 +126,6 @@ pub enum Signal {
Region,
#[serde(rename = "personal_centrality")]
PersonalCentrality,
- #[serde(rename = "crawl_stability")]
- CrawlStability,
#[serde(rename = "topic_centrality")]
TopicCentrality,
#[serde(rename = "query_centrality")]
@@ -146,7 +144,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 39] = [
+pub const ALL_SIGNALS: [Signal; 38] = [
Signal::Bm25,
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
@@ -181,7 +179,6 @@ pub const ALL_SIGNALS: [Signal; 39] = [
Signal::TrackerScore,
Signal::Region,
Signal::PersonalCentrality,
- Signal::CrawlStability,
Signal::TopicCentrality,
Signal::QueryCentrality,
Signal::InboundSimilarity,
@@ -265,7 +262,6 @@ impl Signal {
Signal::UpdateTimestamp => 0.0,
Signal::TrackerScore => 0.019542120533715634,
Signal::Region => 0.06437975586036043,
- Signal::CrawlStability => 0.0,
Signal::PersonalCentrality => 10.0,
Signal::InboundSimilarity => 10.0,
Signal::LambdaMART => 10.0,
@@ -307,7 +303,7 @@ impl Signal {
doc: DocId,
) -> Option<ComputedSignal> {
let value: Option<f64> = match self {
- Signal::HostCentrality | Signal::PageCentrality | Signal::CrawlStability => {
+ Signal::HostCentrality | Signal::PageCentrality => {
let field_value: Option<u64> = self
.fastfield_value(signal_aggregator, doc)
.and_then(|val| val.into());
@@ -454,7 +450,6 @@ impl Signal {
Signal::HostCentrality => Some(webpage.host_centrality),
Signal::PageCentrality => Some(webpage.page_centrality),
Signal::IsHomepage => Some(webpage.html.url().is_homepage().into()),
- Signal::CrawlStability => Some(webpage.crawl_stability),
Signal::FetchTimeMs => {
let fetch_time_ms = webpage.fetch_time_ms as usize;
if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {
@@ -544,7 +539,6 @@ impl Signal {
Signal::UpdateTimestamp => Some(FastField::LastUpdated),
Signal::TrackerScore => Some(FastField::TrackerScore),
Signal::Region => Some(FastField::Region),
- Signal::CrawlStability => Some(FastField::CrawlStability),
_ => None,
}
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index f768280e..27edc423 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -188,7 +188,6 @@ pub enum FastField {
PreComputedScore,
HostNodeID,
SimHash,
- CrawlStability,
NumFlattenedSchemaTokens,
}
@@ -222,7 +221,6 @@ impl FastField {
FastField::PreComputedScore => "pre_computed_score",
FastField::HostNodeID => "host_node_id",
FastField::SimHash => "sim_hash",
- FastField::CrawlStability => "crawl_stability",
}
}
}
@@ -239,7 +237,7 @@ pub enum Field {
Text(TextField),
}
-pub static ALL_FIELDS: [Field; 53] = [
+pub static ALL_FIELDS: [Field; 52] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -293,7 +291,6 @@ pub static ALL_FIELDS: [Field; 53] = [
Field::Fast(FastField::PreComputedScore),
Field::Fast(FastField::HostNodeID),
Field::Fast(FastField::SimHash),
- Field::Fast(FastField::CrawlStability),
];
impl Field {
@@ -504,9 +501,6 @@ impl Field {
.set_indexed()
.set_stored(),
),
- Field::Fast(FastField::CrawlStability) => IndexingOption::Integer(
- NumericOptions::default().set_fast(Cardinality::SingleValue),
- ),
}
}
@@ -606,7 +600,6 @@ impl FastField {
FastField::PreComputedScore => DataType::F64,
FastField::HostNodeID => DataType::U64,
FastField::SimHash => DataType::U64,
- FastField::CrawlStability => DataType::U64,
}
}
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 5d2393d1..e96c2918 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -436,7 +436,6 @@ mod tests {
pre_computed_score: 0.0,
node_id: None,
host_topic: None,
- crawl_stability: 0.0,
dmoz_description: None,
})
.expect("failed to insert webpage");
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 22aa3601..3d6f5d31 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -118,7 +118,6 @@ pub struct Webpage {
pub fetch_time_ms: u64,
pub pre_computed_score: f64,
pub node_id: Option<NodeID>,
- pub crawl_stability: f64,
pub host_topic: Option<Topic>,
pub dmoz_description: Option<String>,
}
@@ -137,7 +136,6 @@ impl Webpage {
pre_computed_score: 0.0,
node_id: None,
host_topic: None,
- crawl_stability: 0.0,
dmoz_description: None,
}
}
@@ -229,13 +227,6 @@ impl Webpage {
self.node_id.map(|n| n.0).unwrap_or(u64::MAX),
);
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::CrawlStability).name())
- .expect("failed to get crawl_stability field"),
- (self.crawl_stability * FLOAT_SCALING as f64) as u64,
- );
-
let facet = self
.host_topic
.map(|topic| topic.as_facet())
@@ -945,7 +936,6 @@ impl Html {
| Field::Fast(FastField::PreComputedScore)
| Field::Fast(FastField::Region)
| Field::Fast(FastField::HostNodeID)
- | Field::Fast(FastField::CrawlStability)
| Field::Text(TextField::DmozDescription) => {}
}
}
@@ -1832,7 +1822,6 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
node_id: None,
- crawl_stability: 0.0,
host_topic: None,
dmoz_description: Some("dmoz description".to_string()),
};
@@ -1867,7 +1856,6 @@ mod tests {
fetch_time_ms: 500,
pre_computed_score: 0.0,
node_id: None,
- crawl_stability: 0.0,
host_topic: None,
dmoz_description: Some("dmoz description".to_string()),
};
diff --git a/core/testcases/lambdamart.txt b/core/testcases/lambdamart.txt
index ac8ddbe3..4c214ed7 100644
--- a/core/testcases/lambdamart.txt
+++ b/core/testcases/lambdamart.txt
@@ -3,27 +3,84 @@ version=v3
num_class=1
num_tree_per_iteration=1
label_index=0
-max_feature_idx=38
+max_feature_idx=37
objective=lambdarank
-feature_names=tracker_score personal_centrality query_centrality bm25_domain_no_tokenizer update_timestamp inbound_similarity bm25_stemmed_clean_body proximity_slop_8 crawl_stability bm25_title_if_homepage bm25_title bm25_clean_body_trigrams bm25_all_body bm25_clean_body lambda_mart bm25_domain_if_homepage proximity_slop_0 bm25_title_trigrams proximity_slop_4 cross_encoder bm25_description fetch_time_ms bm25_domain bm25_stemmed_title bm25_site_no_tokenizer region bm25 topic_centrality bm25_title_bigrams bm25_clean_body_bigrams bm25_site bm25_domain_name_if_homepage_no_tokenizer bm25_backlink_text bm25_url proximity_slop_2 proximity_slop_1 host_centrality page_centrality is_homepage
-feature_infos=[0.012500000000000001:1] none none none none none [0:70.5282745361] [0:68.376205444299998] none none [0:73.018150329600005] [0:100.9539489746] [0:55.664962768599999] [0:60.530643463099999] none none [0:68.376205444299998] [0:70.553833007799994] [0:68.376205444299998] [9.6108999999999997e-06:0.99996737879999997] [0:78.642982482899995] [0:0.20000000000000001] [0:53.968997955299997] [0:71.416839599599996] none [0:0.72049163900000002] [0:723.68379974369998] none [0:90.945877075200002] [0:111.3220901489] [0:52.891826629599997] none none [0:92.092498779300001] [0:68.376205444299998] [0:68.376205444299998] [0:0.034201912000000001] none none
-tree_sizes=1410
+feature_names=tracker_score personal_centrality query_centrality bm25_domain_no_tokenizer update_timestamp inbound_similarity bm25_stemmed_clean_body proximity_slop_8 bm25_title_if_homepage bm25_title bm25_clean_body_trigrams bm25_all_body bm25_clean_body lambda_mart bm25_domain_if_homepage proximity_slop_0 bm25_title_trigrams proximity_slop_4 cross_encoder bm25_description fetch_time_ms bm25_domain bm25_stemmed_title bm25_site_no_tokenizer region bm25 topic_centrality bm25_title_bigrams bm25_clean_body_bigrams bm25_site bm25_domain_name_if_homepage_no_tokenizer bm25_backlink_text bm25_url proximity_slop_2 proximity_slop_1 host_centrality page_centrality is_homepage
+feature_infos=[0.012500000000000001:1] none none none none none [0:70.5282745361] [0:68.376205444299998] none [0:73.018150329600005] [0:100.9539489746] [0:55.664962768599999] [0:60.530643463099999] none none [0:68.376205444299998] [0:70.553833007799994] [0:68.376205444299998] [9.6108999999999997e-06:0.99996737879999997] [0:78.642982482899995] [0:0.20000000000000001] [0:53.968997955299997] [0:71.416839599599996] none [0:0.72049163900000002] [0:723.68379974369998] none [0:90.945877075200002] [0:111.3220901489] [0:52.891826629599997] none none [0:92.092498779300001] [0:68.376205444299998] [0:68.376205444299998] [0:0.034201912000000001] none none
+tree_sizes=5469 5458 5457 5464
Tree=0
-num_leaves=12
+num_leaves=50
num_cat=0
-split_feature=36 29 36 29 19 13 26 23 0 22 20
-split_gain=44.405 15.4883 10.4047 14.0192 8.21314 6.10102 6.05149 4.90787 3.70335 3.47685 3.24401
-threshold=0.019958450000000006 28.131596565250003 0.010784120500000003 5.0413153171500005 0.0066878085500000012 26.930888175950006 116.52628231050001 32.370494842550009 0.025320512800000004 25.618347167950002 21.921377182000004
-decision_type=2 2 2 2 2 2 2 2 2 2 2
-left_child=2 6 4 5 8 7 -2 10 -1 -8 -4
-right_child=1 -3 3 -5 -6 -7 9 -9 -10 -11 -12
-leaf_value=0.0018972091523196965 -0.0051266432567512142 0.0090964994344080417 -0.0039786528246929347 0.0049751017106262193 0.00265658494174438 -0.0069229892972652593 0.0047559028805715586 0.0057326588031252866 -0.0083055804289712074 -0.002607716137233586 0.0058671515241064606
-leaf_weight=0.95217576809227722 2.8653797358274486 12.764272179803809 6.881101677194243 6.7898706812411538 2.2401856034994116 7.4371381889795876 7.4259399548172933 2.1590289101004601 13.490264488966202 2.0443517789244652 0.95239565707743157
-leaf_count=30 68 140 156 152 30 176 142 28 380 38 20
-internal_value=0 0.00523533 -0.0032127 -0.00111974 -0.00625122 -0.00349404 0.00124 -0.000941973 -0.00763292 0.00316632 -0.0027816
-internal_weight=0 25.0999 40.9022 24.2195 16.6826 17.4297 12.3357 9.99253 14.4424 9.47029 7.8335
-internal_count=1360 388 972 532 440 380 248 204 410 180 176
+split_feature=35 28 35 28 18 12 25 20 22 18 20 0 6 21 0 19 11 27 9 11 11 18 10 28 35 35 0 32 32 19 35 35 35 9 24 11 20 0 28 25 19 12 35 6 20 20 0 0 18
+split_gain=44.405 15.4883 10.4047 14.0192 8.21314 6.10102 6.05149 5.02747 4.90787 4.25664 3.71093 3.70335 3.67161 3.47685 7.62133 3.24401 3.54211 3.01026 4.34126 2.68312 2.68164 3.86668 2.62272 5.87262 2.48591 2.84619 4.35285 2.45874 2.39354 1.95176 1.63381 2.53615 1.45497 4.12841 1.34471 1.3104 0.960074 0.782643 2.10217 1.83263 0.708825 0.552426 0.770031 0.592349 1.17153 0.375368 0.365438 0.617484 0.277314
+threshold=0.019958450000000006 28.131596565250003 0.010784120500000003 5.0413153171500005 0.0066878085500000012 26.930888175950006 116.52628231050001 0.0021141743500000007 32.370494842550009 0.039981298100000007 0.0056026365500000007 0.025320512800000004 42.425361633300007 25.618347167950002 0.18333333335000004 21.921377182000004 24.011768341100005 27.740101814250004 42.862270355200003 23.333234787000006 13.504986286150002 1.4367750000000001e-05 5.912924051300001 60.386631011950008 0.022674138000000003 0.021341419500000004 0.095454545450000025 25.740841865550003 1.0000000180025095e-35 23.481465339700005 0.023614218000000003 0.021798707500000004 0.016466209000000006 32.878534317050004 0.41275599895000009 23.831994056700001 0.012270400500000002 0.11805555555000001 60.787229537950004 579.16949462890011 24.733672142050004 30.347360610950002 0.013626538000000002 36.47881507875001 0.0029282639500000005 0.0011933599000000003 0.057189542500000003 0.040833333350000008 5.2856600000000005e-05
+decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
+left_child=2 6 4 5 11 8 30 -6 15 41 22 12 -1 24 -15 16 19 36 -19 20 21 -4 32 -24 25 26 -8 28 29 -26 31 -2 -5 -34 -23 -10 46 48 -39 -40 -17 -7 -43 -44 -45 -22 47 -13 -3
+right_child=1 37 3 10 7 9 13 -9 35 -11 -12 17 -14 14 -16 40 -18 18 -20 -21 45 34 23 -25 27 -27 -28 -29 -30 -31 -32 -33 33 -35 -36 -37 -38 38 39 -41 -42 42 43 44 -46 -47 -48 -49 -50
+leaf_value=0.0099999999999999378 -0.0091199608595756536 0.0062311489026760933 -0.0099999999999999083 0.01 0.009999999999999995 -0.0097878860439067922 -0.0085942871773466191 -0.0049836662301414826 0.0021769301778822117 0.0060698272247080541 -0.0081684232610071076 -0.010000000000000033 -0.0099999999999999742 -0.0099999999999999898 0.0099999999999999881 0.0099999999999999829 -0.0095811419893281225 0.010000000000000002 -0.0079474343913487171 0.00741525494488575 -0.0042878470540396816 -0.0014655660018141363 -0.0089738918116935434 0.0099999999999999881 0.0099999999999999898 -0.0099999999999999829 0.0099999999999999898 0.0099999999999999967 -0.0034467222464834037 -0.0003486167293733067 -0.0092395397883890465 0.0038055097313788372 -0.0065041014766525898 0.0099999999999999933 0.0070743822752000595 0.0099999999999999915 -0.0029580159551886899 0.0099999999999999985 -0.0062884611046340137 0.006733942969711396 0.0013650842744086682 -0.00063389996858715812 -0.0092480401123134128 0.0008833208489539662 -0.00843147895942843 -0.0098649380719365888 -0.0097211758890289497 -0.001785359774494242 0.01
+leaf_weight=0.56641171313822525 0.65762367844581893 0.51561201317235728 0.71878887712956041 3.0031022378243497 1.1422825641930097 2.6949030765099442 0.47946248389780322 1.097903039306402 1.1777106970548632 0.58112015575170506 0.49766855500638474 0.53010737895965299 0.38576405495405197 1.2887303456664088 0.75562143325805653 0.49655948393046845 2.000880990177393 0.52365588769316562 0.94496165215969086 0.69511906802654255 0.40952638909220684 0.77647283300757419 0.81176451593637444 0.81951756402850151 1.2948529701679947 0.5298236459493636 0.91614221036434174 2.5986865982413292 0.90399648249149311 0.70297556370496739 1.310375604778528 0.89738045260310173 0.58617093041539181 1.0716468780301509 1.1343857888132329 0.98131821304559697 0.5984951891005047 1.9877550266683095 0.45562868891283848 0.66372755728662014 0.45583617314696312 0.44086324051022518 2.2359819705598061 0.51910728774964787 0.96516245789825916 1.1459277309477327 10.490613478934391 0.40243090211879462 9.141548893763682
+leaf_count=24 22 24 18 42 14 60 16 16 16 10 16 12 6 32 6 8 40 16 14 8 20 8 16 30 20 8 32 40 14 12 32 14 26 22 24 12 8 18 24 12 12 14 56 16 20 38 316 14 62
+internal_value=0 0.00523533 -0.0032127 -0.00111974 -0.00625122 -0.00349404 0.00124 0.00265658 -0.000941973 -0.00692299 0.0049751 -0.00763292 0.00189721 0.00316632 -0.00260772 -0.0027816 -0.00397865 -0.00830558 -0.00154803 -0.00168164 -0.00319258 -0.000114381 0.00601466 0.000558143 0.0047559 -0.000133712 0.0036119 0.00646749 0.003304 0.00635863 -0.00512664 -0.00166078 0.0079244 0.00416448 0.00360419 0.00573266 -0.00913111 0.0090965 0.00691377 0.00143323 0.00586715 -0.00802427 -0.00688208 -0.00762251 -0.00517373 -0.00839658 -0.00945454 -0.00645502 0.00979877
+internal_weight=0 25.0999 40.9022 24.2195 16.6826 17.4297 12.3357 2.24019 9.99253 7.43714 6.78987 14.4424 0.952176 9.47029 2.04435 7.8335 6.8811 13.4903 1.46862 4.88022 4.1851 2.62965 6.2922 1.63128 7.42594 1.92543 1.3956 5.50051 2.90183 1.99783 2.86538 1.555 4.66092 1.65782 1.91086 2.15903 12.0216 12.7643 3.10711 1.11936 0.952396 6.85602 4.16111 3.72025 1.48427 1.55545 11.4232 0.932538 9.65716
+internal_count=1360 388 972 532 440 380 248 30 204 176 152 410 30 180 38 176 156 380 30 116 108 50 136 46 142 56 48 86 46 32 68 36 90 48 32 28 350 140 54 36 20 166 106 92 36 58 342 26 86
+is_linear=0
+shrinkage=0.005
+
+
+Tree=1
+num_leaves=50
+num_cat=0
+split_feature=35 9 35 28 6 22 19 25 18 20 28 18 20 35 0 6 12 27 9 10 28 11 12 25 18 11 25 0 6 35 9 18 9 0 12 20 6 9 20 12 24 35 35 18 35 0 0 9 19
+split_gain=152.25 59.9795 44.9635 69.9036 34.8326 25.4545 25.4404 23.1446 16.9731 17.7931 16.0731 24.166 15.7759 15.4379 13.7787 14.686 13.1813 13.1641 14.2814 9.50549 22.6834 8.11811 14.5284 9.68489 7.75809 11.1195 7.47554 6.31758 10.3168 6.20151 16.6614 5.36193 4.99983 12.7044 4.33421 3.4442 3.9843 2.99276 2.83137 2.73824 4.71467 2.31028 2.77357 2.23162 1.89009 1.08313 2.676 1.07044 0.941669
+threshold=0.019958450000000006 28.918019294700006 0.010153455000000002 1.5334161222000002 24.617109298700004 28.223515510550005 24.378225326500004 199.75513839720003 0.0066878085500000012 0.0021141743500000007 9.4430942535500026 0.0062548280500000003 0.0056026365500000007 0.015223103000000002 0.025320512800000004 42.425361633300007 26.930888175950006 27.740101814250004 42.862270355200003 5.912924051300001 60.386631011950008 24.193528175350004 20.459006309500001 78.93672296405002 0.00091614705000000009 24.557957649250003 124.34751701355002 0.22500000000000003 40.209836959850008 0.016466209000000006 32.878534317050004 5.7137000000000005e-05 24.108223915100002 0.087121212100000026 31.839186668400004 0.0021276981000000004 35.284496307350004 34.230171203600008 0.0014903262000000002 9.4340581894000017 0.41275599895000009 0.018645591500000006 0.014970756000000002 1.8400600000000005e-05 0.019852502000000005 0.057189542500000003 0.040833333350000008 25.625818252600002 44.153718948350004
+decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
+left_child=2 4 8 5 39 21 13 26 14 -10 11 16 19 37 15 -1 -3 45 -19 29 -21 22 41 -24 25 -15 31 -12 -29 -5 -31 -6 -28 -34 -27 -23 -37 -7 -25 40 -2 42 43 -4 -43 46 -16 -9 -18
+right_child=1 10 3 12 7 6 -8 47 9 -11 27 -13 -14 24 17 -17 48 18 -20 20 -22 35 23 38 -26 34 32 28 -30 30 -32 -33 33 -35 -36 36 -38 -39 -40 -41 -42 44 -44 -45 -46 -47 -48 -49 -50
+leaf_value=0.0099130311339063592 -0.009011865398763445 0.0056064980282179043 -0.0084946482773895788 0.0099062754703599393 -0.0066850502010743938 -0.0099332161041331638 -0.0088389029942627182 -0.004842649047076205 0.0099154155627031969 -0.007075919065934487 0.009742235542027541 0.0099057698944577197 -0.0085715932898408724 0.0078538848264257909 -0.0099139135924102163 -0.0099056211160224433 -0.0097398627305224309 0.0099056360027764811 -0.0068339042579592026 -0.0080001880272964603 0.0099046798605374863 -0.0093351332870926299 0.006779023677050881 0.00081884697706272839 0.0086611579307278135 -0.0099030809585950522 0.0099083742931500782 0.009935220996645483 -0.0064260958179860866 -0.0063336306371194785 0.0099052286125730759 0.005698006919842933 0.0099193622947462458 -0.004526655369067276 0.0019265029011451076 -0.0081735609371648074 -0.00067735680110681223 -0.00036224348856018384 -0.0095501624423914994 -0.0098304313006275755 0.0015933919902353011 0.0016324074209368353 -0.0099291147856710223 1.719381089374356e-05 -0.0076890664940039788 -0.0093019953939002843 -0.0014980253491691522 -0.0099576014002062795 -0.0045846494714091917
+leaf_weight=2.2552217170596118 2.5878867954015732 2.6657359898090345 1.5182959735393544 14.155592277646074 1.8700705468654697 2.1442497372627325 5.7631907761096945 1.9339653477072705 3.0606446154415599 3.1027094051241875 44.616727475076914 6.7855033203959465 1.8968457430601109 3.2769667953252783 2.3642754554748531 1.5964577347040174 3.3677338659763336 1.9774547666311253 3.5825450420379639 3.3989775478839865 3.6884894529357553 16.415852703154091 4.2500208020210257 1.1155177652835848 8.2526729777455312 1.9529078155755999 7.8693827874958497 2.3271645009517665 1.6442525088787077 2.4532973393797866 4.4352577570825815 1.6415552049875257 3.666589193046093 2.6019746363162994 1.2829959169030187 4.5192292947322121 2.9165666252374649 1.3193091787397859 1.6064561903476713 6.5796297639608383 1.7611531913280467 1.2492604553699487 6.5586289912462234 1.5624984875321386 0.96303997933864582 39.427225884050131 1.5729702226817606 2.1712007597088814 1.2019770853221414
+leaf_count=22 10 10 10 50 18 16 44 16 14 16 112 28 16 14 12 6 36 16 14 16 30 88 8 6 22 16 28 8 16 26 22 4 14 12 8 28 20 10 18 44 14 20 44 10 4 312 14 8 10
+internal_value=0 0.00497737 -0.00304754 -0.000949354 -0.000512942 -0.0038025 0.000317978 0.00314559 -0.00648982 0.00136177 0.00784344 0.0031274 0.00538506 0.00321296 -0.00740676 0.00169853 -0.00322944 -0.00812359 -0.000880364 0.00632612 0.00131794 -0.00611905 -0.00366426 0.0020629 0.00544149 0.00136174 0.00563282 0.00920433 0.00316128 0.00801283 0.0041219 -0.000896429 0.00725457 0.00392307 -0.0052128 -0.00805638 -0.0052333 -0.00628752 -0.00530074 -0.00779565 -0.00471724 -0.00703336 -0.00809093 -0.00417766 -0.00242534 -0.00905228 -0.00655168 -0.00754792 -0.00838388
+internal_weight=0 95.2925 155.636 96.6961 32.6834 66.6677 23.9923 21.7547 58.9395 6.16335 62.6091 14.021 30.0285 18.2291 52.7762 3.85168 7.23545 48.9245 5.56 28.1316 7.08747 42.6754 18.8237 6.97199 14.7655 6.51287 17.6496 48.5881 3.97142 21.0441 6.88856 3.51163 14.1379 6.26856 3.2359 23.8516 7.4358 3.46356 2.72197 10.9287 4.34904 11.8517 9.63942 3.08079 2.2123 43.3645 3.93725 4.10517 4.56971
+internal_count=1360 388 972 546 168 386 130 100 426 30 220 84 160 86 396 28 56 368 30 144 46 256 120 32 60 38 76 136 24 98 48 22 54 26 24 136 48 26 24 68 24 88 64 20 24 338 26 24 46
+is_linear=0
+shrinkage=0.005
+
+
+Tree=2
+num_leaves=50
+num_cat=0
+split_feature=35 9 18 12 0 28 28 18 22 18 11 20 22 35 32 6 25 20 12 6 28 16 20 35 6 9 32 22 25 11 0 28 0 11 28 11 18 9 18 11 9 18 12 15 19 9 28 11 19
+split_gain=123.56 48.7975 30.9068 24.6786 23.9449 34.8431 19.553 21.9259 27.3465 15.7557 14.4067 13.9334 15.1573 13.9901 26.016 13.4799 14.4816 12.2247 15.5713 12.1234 10.1412 9.37119 8.88257 8.59522 7.47114 7.23078 6.15149 9.43505 6.60996 5.10747 4.91394 4.89723 4.89465 4.70571 4.69164 4.48818 3.50872 2.9719 2.76934 5.14391 2.34614 2.08664 2.45537 1.89102 1.88417 1.59661 5.2212 1.90012 1.57823
+threshold=0.019958450000000006 27.158587455750006 0.00033038530000000004 27.547188758850002 0.057189542500000003 66.052501678450014 7.1899642944500011 0.0062548280500000003 26.600150108350004 3.4804450000000005e-05 24.557957649250003 0.0056026365500000007 24.183298110950002 0.015295206000000002 34.031578064000009 30.177063941950006 180.53593158720003 0.0029542162000000005 20.176387786900005 19.430234909050004 5.5020630360000009 1.0000000180025095e-35 0.0010005005000000003 0.012058684500000002 20.663570404050002 39.69972801210001 28.999861717200002 44.565795898400005 95.25605201720002 1.0000000180025095e-35 0.15476190480000004 60.787229537950004 0.10555555555000003 35.239673614500006 8.0346117019500021 35.187662124650011 1.0899200000000001e-05 24.846534729000002 0.0076535505000000009 28.534720420800003 35.033393859850008 1.7003300000000004e-05 45.418594360350006 61.749555587750002 45.236038208000004 17.976120948750005 1.0000000180025095e-35 36.921825408950006 21.802299499550006
+decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
+left_child=2 3 4 37 9 10 7 8 -3 19 17 12 20 14 34 -5 32 18 29 -1 -4 36 24 -7 -20 -11 28 -28 -21 -6 -8 -32 -17 -33 -14 -13 40 -2 -15 -40 -12 44 45 -35 -38 46 -43 -48 -47
+right_child=1 6 11 15 5 23 30 -9 -10 25 21 35 13 38 -16 16 -18 -19 22 26 -22 -23 -24 -25 -26 -27 27 -29 -30 -31 31 33 -34 43 -36 -37 41 -39 39 -41 -42 42 -44 -45 -46 48 47 -49 -50
+leaf_value=-0.0098162312194386963 -0.007987589736032287 0.0098682937561520873 -0.0090556226748188685 0.0088065372491295344 0.0013655468532508131 -0.0041892132805463854 0.0098295506853026491 0.0098159723809600873 -0.0078885976861671862 -0.0098236432304169251 -0.0075042214938492524 -0.00037557177776654594 0.0017275916648638301 0.0098526870661669857 -0.0098430945075329863 -0.00013864876746703706 -0.0086106537390147984 -0.0083432055572098297 0.0057623928505988022 0.0010112247913873827 0.0033296840743482288 0.0038699212576462157 0.0083760976732490233 0.0098136184012235905 -0.0073820771174936467 0.0016618625640628904 -0.006492739460603472 0.009854076780269767 0.0098359088343056809 -0.0084536593322206786 0.0098170595323266042 -0.0066374597648179469 0.0098150053284256321 0.00083393074826519924 0.0098360701093015007 -0.0098542990949112771 -0.0096628621972514265 -0.0009219703639646987 0.00034700986553109218 0.0098370951485808999 0.00080657906025458385 -0.0079606000891040592 -0.0021429397028506795 0.0098647420344281662 -0.0040736334652674757 -0.0093657853358157881 0.0098905470837741712 -0.0026733236165661944 -0.0047255320161123538
+leaf_weight=1.575511679053309 8.8357827514410037 3.2305430471897143 4.1529105007648495 6.2050460763275614 1.74238812923432 1.4567152857780445 35.826935090124607 6.4490259215235701 6.5938016995787621 4.2532508820295325 1.6178893595933908 1.9894462227821339 3.4240810871124285 7.2800140082836133 4.1033293902873993 2.5757095664739591 3.5400923937559128 7.455853819847106 1.965584009885786 3.1460566669702512 2.745403990149498 1.7009713947772969 4.823063800111413 4.4242249801754951 2.4022134616971016 2.0216841697692871 2.4229104295372959 1.3886024653911588 6.518871171399951 5.5193735957145691 3.0224805120378724 1.2530913427472117 2.3729385174810886 1.1494650691747665 3.7243828773498535 3.3547490686178207 25.345614746212963 1.7896835915744294 2.7908682078123093 2.9237627014517784 1.7873198762536047 4.809075675904757 2.1141694486141196 1.169391542673111 1.6032203286886204 12.545615587383507 0.60904508084058739 0.59486456215381622 2.1458504982292643
+leaf_count=20 64 10 22 18 12 12 98 28 60 24 12 12 16 28 24 14 26 76 6 18 12 4 36 28 26 6 22 6 38 62 12 8 14 16 10 18 194 14 12 10 8 36 18 6 8 76 12 12 36
+internal_value=0 0.0048093 -0.00283585 -0.000992561 -0.00420222 -0.005395 0.00731205 0.00265265 -0.00204959 0.000532859 -0.00627138 0.00113308 0.00241295 0.00427348 0.000191902 0.00320515 -0.000889322 -0.00303213 -0.00062531 0.00330767 -0.0041265 -0.00768272 0.00369841 0.0063451 -0.00146684 -0.0061232 0.00484197 -0.000537301 0.00696336 -0.00609764 0.00909946 0.00513295 0.00463425 0.00116938 0.00595215 -0.00632572 -0.00805229 -0.0067975 0.00780764 0.0052024 -0.00314206 -0.00838825 -0.00727564 0.00538814 -0.00933035 -0.00779974 -0.00562942 0.0036826 -0.00868803
+internal_weight=0 84.014 142.479 25.3193 105.99 84.6631 58.6947 16.2734 9.82434 21.3269 78.7821 36.4889 31.1448 24.2464 11.2518 14.6938 8.48874 23.9085 16.4526 15.052 6.89831 54.8736 9.19086 5.88094 4.3678 6.27494 13.4764 3.81151 9.66493 7.26176 42.4214 6.59443 4.94865 3.57195 7.14846 5.3442 53.1727 10.6255 12.9946 5.71463 3.40521 49.7675 22.8186 2.31886 26.9488 20.7045 6.01299 1.20391 14.6915
+internal_count=1360 388 972 150 808 674 238 98 70 134 634 164 134 100 50 72 54 218 142 104 34 416 68 40 32 30 84 28 56 74 140 42 28 30 26 30 412 78 50 22 20 392 190 22 202 172 60 24 112
+is_linear=0
+shrinkage=0.005
+
+
+Tree=3
+num_leaves=50
+num_cat=0
+split_feature=35 9 18 0 28 18 12 28 18 22 12 12 20 9 35 32 28 11 0 6 20 6 11 32 22 25 11 0 28 6 21 12 24 25 19 6 18 18 28 35 35 18 18 20 32 32 11 35 6
+split_gain=122.03 47.1967 30.9872 19.548 30.0652 20.2562 18.4669 17.3936 19.2893 24.2543 15.4895 15.1185 19.8703 13.5899 22.0719 26.5849 15.3893 13.1669 14.5908 11.5188 9.76489 9.07555 6.71338 5.67539 7.46293 6.0919 4.57457 4.42589 4.52685 3.66486 3.38861 3.30437 4.90931 2.79385 2.05371 1.83061 1.58896 1.35776 1.25748 1.23288 7.54702 1.84498 5.5099 1.42987 1.17893 4.5426 1.0637 2.91643 0.999161
+threshold=0.019958450000000006 27.158587455750006 0.00033038530000000004 0.057189542500000003 66.052501678450014 3.4804450000000005e-05 27.547188758850002 7.1899642944500011 0.0062548280500000003 26.600150108350004 27.467996597300004 20.176387786900005 0.0029542162000000005 27.462724685700007 0.015295206000000002 32.923410415650004 5.5020630360000009 27.969018936150004 0.10555555555000003 19.430234909050004 0.0010005005000000003 54.445077896150003 24.193528175350004 28.999861717200002 44.565795898400005 95.25605201720002 1.0000000180025095e-35 0.15476190480000004 60.787229537950004 40.209836959850008 43.487449645950008 9.4340581894000017 0.41275599895000009 182.36229705810004 20.416842460650006 25.280503273000004 1.0837500000000001e-05 2.3751900000000002e-05 27.136941909800004 0.010422712000000002 0.012167945000000001 1.9047900000000001e-05 2.5816900000000003e-05 0.012270400500000002 34.031578064000009 28.621761322050002 31.730455398550003 0.023952123000000002 28.991178512600005
+decision_type=2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2
+left_child=2 6 3 5 10 19 31 8 9 -3 11 26 20 16 15 38 -4 -8 37 -1 22 -6 -13 25 -25 -21 -5 -9 -29 -30 44 32 -2 -20 -28 -33 -12 -19 -15 43 -41 -42 -43 -38 45 -16 -11 -48 -44
+right_child=1 7 13 4 21 -7 17 27 -10 46 36 12 -14 14 30 -17 -18 18 33 23 -22 -23 -24 24 -26 -27 34 28 29 -31 -32 35 -34 -35 -36 -37 39 -39 -40 40 41 42 48 -45 -46 -47 47 -49 -50
+leaf_value=-0.009735484290709152 -0.0077997160604779993 0.0097892928315321519 -0.0090402090501182069 -6.7551674772469675e-06 0.0097686797150063863 -0.0084421212177286219 0.0097927814967422658 0.0097493300365762772 0.0097319240613288425 -0.009753310160297372 -0.0041787010613604869 0.0040024288708004367 -0.0094732796066520891 0.0042210158497068585 0.0097963418389761386 -0.0097600848441952732 0.0035022281440736311 -0.0090559226084526278 0.0097308144815816625 0.001034029365984943 0.0097890683830102482 -0.0056743049826746836 -0.0097669574812041318 -0.0058205312931484676 0.0097589541384304878 0.0097525410075975632 -0.0085788848013285644 0.0097369436335925184 0.0054683204088723505 -0.0052864222235926277 0.0013642457935270772 -0.0095848035561727704 0.0038651530812490106 0.0017788844547361432 -0.002146799162852557 -0.0027439546410395104 -0.0094365916208346053 -0.0030757633619894783 0.0097818753086814855 0.004305457652136766 -0.0094896984485105512 0.0028567343672873937 -0.0048401838041101774 -0.0048635013387279338 0.0097452906425091849 0.001352937347753492 0.0013759221098089832 -0.0097961901173875838 -0.0093999651687309334
+leaf_weight=1.528825238347056 1.9284897595643997 2.8999190777540189 5.8166402131319028 2.180921062827113 3.9779065251350394 5.0623577311635009 4.4770334362983695 35.064194981008768 5.8378064930438986 3.3479171991348267 2.4687739424407473 2.454122960567473 3.3166270405054084 2.0150853358209151 4.2708440870046598 5.6151065528392792 4.2199865188449621 3.0195221751928312 2.170052669942379 2.987590476870535 4.6908980458974838 1.2504300549626348 1.3846900165081022 2.3127136006951328 1.1513491310179231 6.0832049399614334 11.054187595844267 2.8841942548751822 1.9780068797990682 1.3212530016899107 2.2421094905585042 5.2508369833230972 1.694563493132593 2.249711811542511 1.39795622229576 1.2017739973962305 21.761921420693401 1.3842904716730116 2.0517177432775497 1.2901765368878866 11.20223566144705 1.3514821175485838 1.6040376499295232 1.8549927920103062 5.5904756858944893 2.5405823923647404 0.96053120121359858 1.4907369762659071 4.786165863275528
+leaf_count=20 10 10 30 20 26 30 10 98 28 30 18 22 34 14 20 34 16 24 14 18 38 14 16 22 6 38 112 12 14 16 12 44 14 14 14 10 182 10 6 18 86 18 16 10 22 10 12 18 30
+internal_value=0 0.00490488 -0.00295229 -0.00439569 -0.00550609 0.000134362 -0.00105916 0.00740408 0.00270113 -0.00201711 -0.00633785 -0.00328746 0.000911542 0.00112863 0.00314837 -0.00270903 -0.00376663 0.00280879 -0.000734844 0.00322155 0.0049495 0.00607527 -0.000964297 0.00480187 -0.000642384 0.00688098 -0.00668681 0.00906154 0.00516134 0.00116137 0.00702099 -0.00616513 -0.00234386 0.00568319 -0.00785678 -0.00831072 -0.00808165 -0.00717612 0.00702649 -0.00830138 -0.00739563 -0.00819253 -0.00631552 -0.0090774 0.00804366 0.00664705 -0.00792097 -0.00541839 -0.00825539
+internal_weight=0 79.1608 131.516 97.1536 78.0275 19.126 23.3763 55.7846 14.5369 8.6991 72.7992 26.4794 11.8463 34.3625 24.3259 9.68191 10.0366 13.3006 8.82358 14.0637 8.52971 5.22834 3.83881 12.5349 3.46406 9.0708 14.6331 41.2476 6.18345 3.29926 14.644 10.0757 3.62305 4.41976 12.4521 6.45261 46.3198 4.40381 4.0668 43.851 20.2341 18.9439 7.74169 23.6169 12.4019 6.81143 5.79919 2.45127 6.3902
+internal_count=1360 388 972 808 674 134 150 238 98 70 634 256 110 164 118 54 46 72 62 104 76 40 38 84 28 56 146 140 42 30 64 78 24 28 126 54 378 34 20 360 168 150 64 192 52 30 60 30 46
is_linear=0
shrinkage=0.005
@@ -31,15 +88,25 @@ shrinkage=0.005
end of trees
feature_importances:
-bm25_clean_body_bigrams=2
-host_centrality=2
-tracker_score=1
-bm25_clean_body=1
-cross_encoder=1
-bm25_description=1
-bm25_domain=1
-bm25_stemmed_title=1
-bm25=1
+host_centrality=23
+cross_encoder=22
+bm25_clean_body_bigrams=18
+tracker_score=17
+bm25_all_body=16
+bm25_title=15
+fetch_time_ms=15
+bm25_stemmed_clean_body=14
+bm25_clean_body=13
+bm25=9
+bm25_description=8
+bm25_url=8
+bm25_stemmed_title=7
+region=3
+bm25_clean_body_trigrams=2
+bm25_domain=2
+bm25_title_bigrams=2
+proximity_slop_0=1
+bm25_title_trigrams=1
parameters:
[boosting: gbdt]
@@ -51,14 +118,14 @@ parameters:
[valid: ]
[num_iterations: 100]
[learning_rate: 0.005]
-[num_leaves: 12]
+[num_leaves: 50]
[num_threads: -1]
[deterministic: 0]
[force_col_wise: 0]
[force_row_wise: 0]
[histogram_pool_size: -1]
[max_depth: -1]
-[min_data_in_leaf: 20]
+[min_data_in_leaf: 10]
[min_sum_hessian_in_leaf: 0.001]
[bagging_fraction: 1]
[pos_bagging_fraction: 1]