Skip to content

web search done right

prevent infinite loop during robotstxt parsing...

Mikkel Denker6b8d921acab6
diff --git a/crates/robotstxt/src/lib.rs b/crates/robotstxt/src/lib.rs
index 2ecb3b18..01fda3b4 100644
--- a/crates/robotstxt/src/lib.rs
+++ b/crates/robotstxt/src/lib.rs
@@ -85,6 +85,7 @@ impl Default for Params {
}
}
+#[derive(Debug)]
pub struct Robots {
rules: Vec<Rule>,
crawl_delay: Option<f32>,
@@ -140,6 +141,7 @@ impl Robots {
while idx < lines.len() {
let line = &lines[idx];
+ idx += 1;
if let Line::UserAgent(agents) = &line {
useragent_lines += 1;
@@ -150,9 +152,11 @@ impl Robots {
.all(|(c1, c2)| c1.to_ascii_lowercase() == c2)
}) {
let mut has_captured_directive = false;
- while idx + 1 < lines.len() {
+ while idx < lines.len() {
+ let line = &lines[idx];
idx += 1;
- match &lines[idx] {
+
+ match line {
Line::Allow(path) => {
has_captured_directive = true;
@@ -173,7 +177,13 @@ impl Robots {
});
}
}
- Line::UserAgent(_) if has_captured_directive => break,
+ Line::UserAgent(_) if has_captured_directive => {
+ idx -= 1;
+ break;
+ }
+ Line::Sitemap(sitemap) => {
+ sitemaps.push(sitemap.to_string());
+ }
Line::CrawlDelay(Some(delay)) => {
has_captured_directive = true;
crawl_delay = Some(*delay);
@@ -181,7 +191,6 @@ impl Robots {
_ => {}
}
}
- continue;
}
} else if useragent_lines == 0 {
// add preceding rules as global rules
@@ -208,8 +217,6 @@ impl Robots {
if let Line::Sitemap(url) = line {
sitemaps.push(url.to_string());
}
-
- idx += 1;
}
Ok(Self {
diff --git a/crates/robotstxt/src/pattern.rs b/crates/robotstxt/src/pattern.rs
index 8faa45e1..6ab5a197 100644
--- a/crates/robotstxt/src/pattern.rs
+++ b/crates/robotstxt/src/pattern.rs
@@ -291,11 +291,5 @@ mod tests {
prop_assert_eq!(rule.matches(&path), re.is_match(&path));
}
-
- #[test]
- fn percent_encode_idempotent(s: String) {
- let encoded = percent_encode(&s);
- prop_assert_eq!(percent_encode(&encoded), encoded);
- }
}
}