web search done right
log number of urls crawled per second
Mikkel Denker994a858e862f
diff --git a/core/src/call_counter.rs b/core/src/call_counter.rsnew file mode 100644index 00000000..cff217ea--- /dev/null+++ b/core/src/call_counter.rs@@ -0,0 +1,56 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::collections::VecDeque;+use std::time::{Duration, Instant};++pub struct CallCounter {+ call_times: VecDeque<Instant>,+ window: Duration,+}++impl CallCounter {+ // Constructs a new `CallCounter` with a specified window duration for the rolling average.+ pub fn new(window: Duration) -> CallCounter {+ CallCounter {+ call_times: VecDeque::new(),+ window,+ }+ }++ // Count the method call.+ pub fn count(&mut self) {+ let now = Instant::now();+ self.call_times.push_back(now);+ }++ fn prune(&mut self) {+ let now = Instant::now();++ while let Some(&old) = self.call_times.front() {+ if now.duration_since(old) < self.window {+ break;+ }++ self.call_times.pop_front();+ }+ }++ // Calculate the average calls per second.+ pub fn avg_per_second(&mut self) -> f64 {+ self.prune();+ self.call_times.len() as f64 / self.window.as_secs_f64()+ }+}diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 23ed943c..5a6cba07 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::webpage::Url;+use crate::{call_counter::CallCounter, webpage::Url};use super::{crawl_db::{CrawlDb, DomainStatus},@@ -23,8 +23,11 @@ use super::{use std::{collections::HashSet,path::Path,- sync::atomic::{AtomicU64, Ordering},- time::Instant,+ sync::{+ atomic::{AtomicU64, Ordering},+ Mutex,+ },+ time::{Duration, Instant},};const DEFAULT_JOB_URLS: usize = 1000;@@ -33,6 +36,7 @@ pub struct CrawlCoordinator {db: CrawlDb,num_crawled_urls: AtomicU64,num_urls_to_crawl: u64,+ call_counter: Mutex<CallCounter>,}impl CrawlCoordinator {@@ -58,10 +62,22 @@ impl CrawlCoordinator {db,num_urls_to_crawl,num_crawled_urls: AtomicU64::new(0),+ call_counter: Mutex::new(CallCounter::new(Duration::from_secs(60))),})}+ fn log_crawls_per_second(&self, num_urls: usize) {+ let mut call_counter = self.call_counter.lock().unwrap();++ for _ in 0..num_urls {+ call_counter.count();+ }++ tracing::info!("avg crawls per second: {}", call_counter.avg_per_second());+ }+pub fn add_response(&self, response: &JobResponse) -> Result<()> {+ self.log_crawls_per_second(response.url_responses.len());self.num_crawled_urls.fetch_add(1, Ordering::SeqCst);let tx = self.db.transaction()?;diff --git a/core/src/lib.rs b/core/src/lib.rsindex 65ab5e20..1c3ce7e1 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -37,6 +37,7 @@ pub mod alice;mod api;mod autosuggest;mod bangs;+mod call_counter;mod collector;mod crawler;mod directory;