Skip to content

web search done right

[crawler] another check to avoid re-crawling urls

Mikkel Denkerad617a8151a8
diff --git a/crates/core/src/crawler/worker.rs b/crates/core/src/crawler/worker.rs
index 5b0bd55c..5ae578a2 100644
--- a/crates/core/src/crawler/worker.rs
+++ b/crates/core/src/crawler/worker.rs
@@ -13,7 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use anyhow::anyhow;
+use anyhow::{anyhow, bail};
use encoding_rs::{Encoding, UTF_8};
use hashbrown::HashSet;
use mime::Mime;
@@ -535,6 +535,10 @@ impl<S: DatumStream> JobExecutor<S> {
}
async fn crawl_url(&mut self, url: Url) -> Result<CrawlDatum> {
+ if self.crawled_urls.contains(&url) {
+ bail!("url already crawled: {}", url);
+ }
+
let start = Instant::now();
let res = self.fetch_with_https_priority(url.clone()).await;
let fetch_time = start.elapsed();