Skip to content

web search done right

properly interpret 'Disallow: ' robotstxt files

Mikkel Denker1010402cfc89
diff --git a/crates/robotstxt/src/lib.rs b/crates/robotstxt/src/lib.rs
index 58454a4d..627e389c 100644
--- a/crates/robotstxt/src/lib.rs
+++ b/crates/robotstxt/src/lib.rs
@@ -145,17 +145,23 @@ impl Robots {
match &lines[idx] {
Line::Allow(path) => {
has_captured_directive = true;
- rules.push(Rule {
- pattern: Pattern::new(path),
- directive: Directive::Allow,
- });
+
+ if !path.is_empty() {
+ rules.push(Rule {
+ pattern: Pattern::new(path),
+ directive: Directive::Allow,
+ });
+ }
}
Line::Disallow(path) => {
has_captured_directive = true;
- rules.push(Rule {
- pattern: Pattern::new(path),
- directive: Directive::Disallow,
- });
+
+ if !path.is_empty() {
+ rules.push(Rule {
+ pattern: Pattern::new(path),
+ directive: Directive::Disallow,
+ });
+ }
}
Line::UserAgent(_) if has_captured_directive => break,
Line::CrawlDelay(Some(delay)) => {
@@ -1618,4 +1624,13 @@ Allow: /
let robots = Robots::parse("BarBot", &robotstxt).unwrap();
assert_eq!(robots.crawl_delay(), None);
}
+
+ #[test]
+ fn test_empty_disallow() {
+ let robotstxt = "User-Agent: FooBot
+ Disallow:
+ ";
+
+ assert!(is_user_agent_allowed(robotstxt, "FooBot", "/"));
+ }
}
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index cca2022f..4febd988 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -12,14 +12,13 @@ param_grid = {
"objective": ["lambdarank"],
"verbosity": [-1],
"metric": ["ndcg"],
- "ndcg_at": [[1, 2, 3, 5, 10]],
+ "ndcg_at": [[1, 2, 3, 5]],
# "learning_rate": [0.003],
# "num_iterations": [100],
# "max_depth": [-1, 2, 4, 8],
"max_depth": [-1],
- # "num_leaves": [7, 15, 31],
- "num_leaves": [31, 63],
- "lambda_l2": [2.5],
+ "num_leaves": [15],
+ "lambda_l2": [2.0, 2.5],
"linear_tree": [False],
}