Skip to content

web search done right

robots.txt files might contain weird characters which causes parsing to crash

Mikkel Denker77b95d3fbec3
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 8786c566..de800725 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::HashMap, time::Instant};
+use std::{collections::HashMap, panic, time::Instant};
use robotstxt_with_cache::matcher::{
CachingRobotsMatcher, LongestMatchRobotsMatchStrategy, RobotsMatcher,
@@ -53,7 +53,10 @@ impl RobotsTxtManager {
let body = res.text().await?;
- RobotsTxt::new(body)
+ match panic::catch_unwind(|| RobotsTxt::new(body)) {
+ Ok(r) => r,
+ Err(_) => Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT)),
+ }
}
async fn get_mut(&mut self, url: &Url) -> Result<Option<&mut RobotsTxt>> {