web search done right
properly interpret 'Disallow: ' robotstxt files
Mikkel Denker1010402cfc89
diff --git a/crates/robotstxt/src/lib.rs b/crates/robotstxt/src/lib.rsindex 58454a4d..627e389c 100644--- a/crates/robotstxt/src/lib.rs+++ b/crates/robotstxt/src/lib.rs@@ -145,17 +145,23 @@ impl Robots {match &lines[idx] {Line::Allow(path) => {has_captured_directive = true;- rules.push(Rule {- pattern: Pattern::new(path),- directive: Directive::Allow,- });++ if !path.is_empty() {+ rules.push(Rule {+ pattern: Pattern::new(path),+ directive: Directive::Allow,+ });+ }}Line::Disallow(path) => {has_captured_directive = true;- rules.push(Rule {- pattern: Pattern::new(path),- directive: Directive::Disallow,- });++ if !path.is_empty() {+ rules.push(Rule {+ pattern: Pattern::new(path),+ directive: Directive::Disallow,+ });+ }}Line::UserAgent(_) if has_captured_directive => break,Line::CrawlDelay(Some(delay)) => {@@ -1618,4 +1624,13 @@ Allow: /let robots = Robots::parse("BarBot", &robotstxt).unwrap();assert_eq!(robots.crawl_delay(), None);}++ #[test]+ fn test_empty_disallow() {+ let robotstxt = "User-Agent: FooBot+ Disallow:+ ";++ assert!(is_user_agent_allowed(robotstxt, "FooBot", "/"));+ }}diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex cca2022f..4febd988 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -12,14 +12,13 @@ param_grid = {"objective": ["lambdarank"],"verbosity": [-1],"metric": ["ndcg"],- "ndcg_at": [[1, 2, 3, 5, 10]],+ "ndcg_at": [[1, 2, 3, 5]],# "learning_rate": [0.003],# "num_iterations": [100],# "max_depth": [-1, 2, 4, 8],"max_depth": [-1],- # "num_leaves": [7, 15, 31],- "num_leaves": [31, 63],- "lambda_l2": [2.5],+ "num_leaves": [15],+ "lambda_l2": [2.0, 2.5],"linear_tree": [False],}