Skip to content

web search done right

valid useragents are always ascii, so we should be able to do the lowercase string cmp without alloc

Mikkel Denkerfb5a8cc6ab40
diff --git a/crates/robotstxt/src/lib.rs b/crates/robotstxt/src/lib.rs
index 1f785cb6..2ecb3b18 100644
--- a/crates/robotstxt/src/lib.rs
+++ b/crates/robotstxt/src/lib.rs
@@ -119,7 +119,12 @@ impl Robots {
if !lines.iter().any(|line| {
if let Line::UserAgent(agents) = line {
- agents.iter().any(|agent| agent.to_lowercase() == useragent)
+ agents.iter().any(|agent| {
+ agent
+ .chars()
+ .zip(useragent.chars())
+ .all(|(c1, c2)| c1.to_ascii_lowercase() == c2)
+ })
} else {
false
}
@@ -138,7 +143,12 @@ impl Robots {
if let Line::UserAgent(agents) = &line {
useragent_lines += 1;
- if agents.iter().any(|agent| agent.to_lowercase() == useragent) {
+ if agents.iter().any(|agent| {
+ agent
+ .chars()
+ .zip(useragent.chars())
+ .all(|(c1, c2)| c1.to_ascii_lowercase() == c2)
+ }) {
let mut has_captured_directive = false;
while idx + 1 < lines.len() {
idx += 1;