web search done right
when https version of robots.txt is 404 but http is unreachable, it should overall be treated as a 404 as the website has most likely just redirected all http traffic to https
Mikkel Denker3e732e0f2970
diff --git a/crates/core/src/crawler/robots_txt.rs b/crates/core/src/crawler/robots_txt.rsindex 5eb43415..d112d8be 100644--- a/crates/core/src/crawler/robots_txt.rs+++ b/crates/core/src/crawler/robots_txt.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -155,23 +155,17 @@ impl InnerRobotsTxtManager {.fetch_robots_txt_from_url(&format!("https://{}/robots.txt", site.0)).await{- Lookup::Unavailable | Lookup::Unreachable => {+ Lookup::Unreachable => {+ self.fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0))+ .await+ }+ Lookup::Unavailable => {match self.fetch_robots_txt_from_url(&format!("http://{}/robots.txt", site.0)).await{Lookup::Found(robots_txt) => Lookup::Found(robots_txt),- Lookup::Unreachable => Lookup::Unreachable,- Lookup::Unavailable- if !site.0.starts_with("www.")- && site.0.chars().filter(|&c| c == '.').count() == 1 =>- {- self.fetch_robots_txt_from_url(&format!(- "https://www.{}/robots.txt",- &site.0- ))- .await- }+ Lookup::Unreachable => Lookup::Unavailable,Lookup::Unavailable => Lookup::Unavailable,}}diff --git a/crates/core/src/live_index/crawler/checker/feeds.rs b/crates/core/src/live_index/crawler/checker/feeds.rsindex 41ed7071..6f99b62d 100644--- a/crates/core/src/live_index/crawler/checker/feeds.rs+++ b/crates/core/src/live_index/crawler/checker/feeds.rs@@ -48,11 +48,15 @@ impl Checker for Feeds {let Ok(req) = self.client.get(feed.url.clone()).await else {continue;};+let Ok(resp) = req.send().await else {continue;};- let text = resp.text().await?;+ let Ok(text) = resp.text().await else {+ continue;+ };+let Ok(parsed_feed) = parse(&text, feed.kind) else {continue;};