Skip to content

web search done right

when https version of robots.txt is 404 but http is unreachable, it should overall be treated as a 404 as the website has most likely just redirected all http traffic to https

Mikkel Denker3e732e0f2970
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rs
index 5eb43415..d112d8be 100644
--- a/crates/core/src/crawler/robots_txt.rs
+++ b/crates/core/src/crawler/robots_txt.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -155,23 +155,17 @@ impl InnerRobotsTxtManager {
.fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0))
.await
{
- Lookup::Unavailable | Lookup::Unreachable => {
+ Lookup::Unreachable => {
+ self.fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))
+ .await
+ }
+ Lookup::Unavailable => {
match self
.fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))
.await
{
Lookup::Found(robots_txt) => Lookup::Found(robots_txt),
- Lookup::Unreachable => Lookup::Unreachable,
- Lookup::Unavailable
- if !site.0.starts_with("www.")
- && site.0.chars().filter(|&c| c == '.').count() == 1 =>
- {
- self.fetch_robots_txt_from_url(&format!(
- "https://www.{}/robots.txt",
- &site.0
- ))
- .await
- }
+ Lookup::Unreachable => Lookup::Unavailable,
Lookup::Unavailable => Lookup::Unavailable,
}
}
diff --git a/crates/core/src/live_index/crawler/checker/feeds.rs b/crates/core/src/live_index/crawler/checker/feeds.rs
index 41ed7071..6f99b62d 100644
--- a/crates/core/src/live_index/crawler/checker/feeds.rs
+++ b/crates/core/src/live_index/crawler/checker/feeds.rs
@@ -48,11 +48,15 @@ impl Checker for Feeds {
let Ok(req) = self.client.get(feed.url.clone()).await else {
continue;
};
+
let Ok(resp) = req.send().await else {
continue;
};
- let text = resp.text().await?;
+ let Ok(text) = resp.text().await else {
+ continue;
+ };
+
let Ok(parsed_feed) = parse(&text, feed.kind) else {
continue;
};