Skip to content

web search done right

Remove ttl from robots.txt Makes sure that we don't retry download in any case

Mikkel Denker8331a7786b95
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index b5715b35..981cfbdb 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::HashMap, panic, time::Instant};
+use std::{collections::HashMap, panic};
use robotstxt_with_cache::matcher::{
CachingRobotsMatcher, LongestMatchRobotsMatchStrategy, RobotsMatcher,
@@ -62,13 +62,7 @@ impl RobotsTxtManager {
async fn get_mut(&mut self, url: &Url) -> Result<Option<&mut RobotsTxt>> {
let site = Site(url.site().to_string());
- let should_fetch = match self.cache.get(&site) {
- Some(Some(robots_txt)) => Instant::now() > robots_txt.valid_until,
- Some(None) => false, // robots.txt does not exist
- None => true,
- };
-
- if should_fetch {
+ if self.cache.get(&site).is_none() {
match self.fetch_robots_txt(&site).await {
Ok(robots_txt) => {
self.cache.insert(site.clone(), Some(robots_txt));
@@ -98,7 +92,6 @@ impl RobotsTxtManager {
}
struct RobotsTxt {
- valid_until: Instant,
matcher: CachingRobotsMatcher<LongestMatchRobotsMatchStrategy>,
sitemap: Option<Url>,
}
@@ -116,11 +109,7 @@ impl RobotsTxt {
.map(|line| line.split(':').nth(1).unwrap().trim().to_string())
.map(Url::from);
- Ok(Self {
- valid_until: Instant::now() + chrono::Duration::hours(24).to_std()?,
- matcher,
- sitemap,
- })
+ Ok(Self { matcher, sitemap })
}
}