Skip to content

web search done right

log number of urls crawled per second

Mikkel Denker994a858e862f
diff --git a/core/src/call_counter.rs b/core/src/call_counter.rs
new file mode 100644
index 00000000..cff217ea
--- /dev/null
+++ b/core/src/call_counter.rs
@@ -0,0 +1,56 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::collections::VecDeque;
+use std::time::{Duration, Instant};
+
+pub struct CallCounter {
+ call_times: VecDeque<Instant>,
+ window: Duration,
+}
+
+impl CallCounter {
+ // Constructs a new `CallCounter` with a specified window duration for the rolling average.
+ pub fn new(window: Duration) -> CallCounter {
+ CallCounter {
+ call_times: VecDeque::new(),
+ window,
+ }
+ }
+
+ // Count the method call.
+ pub fn count(&mut self) {
+ let now = Instant::now();
+ self.call_times.push_back(now);
+ }
+
+ fn prune(&mut self) {
+ let now = Instant::now();
+
+ while let Some(&old) = self.call_times.front() {
+ if now.duration_since(old) < self.window {
+ break;
+ }
+
+ self.call_times.pop_front();
+ }
+ }
+
+ // Calculate the average calls per second.
+ pub fn avg_per_second(&mut self) -> f64 {
+ self.prune();
+ self.call_times.len() as f64 / self.window.as_secs_f64()
+ }
+}
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 23ed943c..5a6cba07 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::webpage::Url;
+use crate::{call_counter::CallCounter, webpage::Url};
use super::{
crawl_db::{CrawlDb, DomainStatus},
@@ -23,8 +23,11 @@ use super::{
use std::{
collections::HashSet,
path::Path,
- sync::atomic::{AtomicU64, Ordering},
- time::Instant,
+ sync::{
+ atomic::{AtomicU64, Ordering},
+ Mutex,
+ },
+ time::{Duration, Instant},
};
const DEFAULT_JOB_URLS: usize = 1000;
@@ -33,6 +36,7 @@ pub struct CrawlCoordinator {
db: CrawlDb,
num_crawled_urls: AtomicU64,
num_urls_to_crawl: u64,
+ call_counter: Mutex<CallCounter>,
}
impl CrawlCoordinator {
@@ -58,10 +62,22 @@ impl CrawlCoordinator {
db,
num_urls_to_crawl,
num_crawled_urls: AtomicU64::new(0),
+ call_counter: Mutex::new(CallCounter::new(Duration::from_secs(60))),
})
}
+ fn log_crawls_per_second(&self, num_urls: usize) {
+ let mut call_counter = self.call_counter.lock().unwrap();
+
+ for _ in 0..num_urls {
+ call_counter.count();
+ }
+
+ tracing::info!("avg crawls per second: {}", call_counter.avg_per_second());
+ }
+
pub fn add_response(&self, response: &JobResponse) -> Result<()> {
+ self.log_crawls_per_second(response.url_responses.len());
self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);
let tx = self.db.transaction()?;
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 65ab5e20..1c3ce7e1 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -37,6 +37,7 @@ pub mod alice;
mod api;
mod autosuggest;
mod bangs;
+mod call_counter;
mod collector;
mod crawler;
mod directory;