web search done right
split live index into more manageable files
Mikkel Denker72ac62202846
diff --git a/crates/core/src/live_index.rs b/crates/core/src/live_index.rsdeleted file mode 100644index db0d1312..00000000--- a/crates/core/src/live_index.rs+++ /dev/null@@ -1,412 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use anyhow::Result;-use std::{- path::Path,- sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard},- time::{Duration, SystemTime},-};--use chrono::{DateTime, Utc};-use url::Url;--use crate::{- config::{CrawlerConfig, LiveIndexConfig},- crawler::{- reqwest_client, CrawlDatum, DatumStream, JobExecutor, RetrieableUrl, WeightedUrl, WorkerJob,- },- entrypoint::indexer::{self, IndexingWorker},- feed::{- self,- scheduler::{Domain, DomainFeeds, Split},- },-};--const TTL: Duration = Duration::from_secs(60 * 60 * 24 * 60); // 60 days-const PRUNE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour-const FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutes-const AUTO_COMMIT_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 minutes-const EVENT_LOOP_INTERVAL: Duration = Duration::from_secs(5);-const BATCH_SIZE: usize = 512;--#[derive(Debug, Clone)]-struct Feeds {- last_checked: DateTime<Utc>,- feed: DomainFeeds,-}--impl From<Split> for Vec<Feeds> {- fn from(split: Split) -> Self {- split- .feeds- .into_iter()- .map(|feed| Feeds {- last_checked: Utc::now(),- feed,- })- .collect()- }-}--struct DownloadedDb {- db: rocksdb::DB,-}--impl DownloadedDb {- fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let mut options = rocksdb::Options::default();- options.create_if_missing(true);-- // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)- options.set_level_compaction_dynamic_level_bytes(true);- options.set_bytes_per_sync(1048576);- let mut block_options = rocksdb::BlockBasedOptions::default();- block_options.set_block_size(16 * 1024);- block_options.set_format_version(5);- block_options.set_cache_index_and_filter_blocks(true);- block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);-- block_options.set_ribbon_filter(10.0);-- options.set_block_based_table_factory(&block_options);- options.optimize_for_point_lookup(512); // 512 mb-- let db = rocksdb::DB::open_with_ttl(&options, path, TTL)?;- Ok(Self { db })- }-- fn has_downloaded(&self, url: &Url) -> Result<bool> {- let key = url.as_str().as_bytes();-- if self.db.key_may_exist(key) {- self.db.get(key).map(|v| v.is_some()).map_err(Into::into)- } else {- Ok(false)- }- }-- fn insert(&self, url: &Url) -> Result<()> {- let key = url.as_str().as_bytes();- self.db.put(key, b"")?;- Ok(())- }-}--struct Indexer {- search_index: Arc<RwLock<crate::index::Index>>,- worker: IndexingWorker,- write_batch: Arc<Mutex<Vec<indexer::IndexableWebpage>>>,-}--enum CrawlResults {- None,- HasInserts,-}--struct Crawler {- feeds: Vec<Feeds>,- indexer: Arc<Indexer>,- downloaded_db: DownloadedDb,- config: Arc<CrawlerConfig>,- client: reqwest::Client,-}--impl Crawler {- fn new(- split: Split,- indexer: Arc<Indexer>,- downloaded_db: DownloadedDb,- config: Arc<CrawlerConfig>,- ) -> Result<Self> {- let client = reqwest_client(&config)?;-- Ok(Self {- feeds: split.into(),- indexer,- downloaded_db,- config,- client,- })- }-- async fn process_urls(&self, urls: Vec<Url>) -> Result<bool> {- if urls.is_empty() {- return Ok(false);- }-- let domain = urls.first().unwrap().into();- let job = WorkerJob {- domain,- urls: urls- .clone()- .into_iter()- .map(|url| RetrieableUrl::from(WeightedUrl { url, weight: 1.0 }))- .collect(),- wandering_urls: 0,- };-- let executor = JobExecutor::new(- job,- self.client.clone(),- self.config.clone(),- self.indexer.clone(),- );- executor.run().await;-- for url in &urls {- self.downloaded_db.insert(url)?;- }-- Ok(true)- }-- async fn process_feed(&self, domain_feeds: &DomainFeeds) -> Result<bool> {- let mut urls = Vec::new();-- for feed in domain_feeds.feeds.iter() {- let content = self- .client- .get(feed.url.as_str())- .send()- .await?- .text()- .await?;-- let feed = feed::parse(&content, feed.kind)?;-- for url in feed.links {- if !self.downloaded_db.has_downloaded(&url)?- && Domain::from(&url) == domain_feeds.domain- {- urls.push(url);- }- }- }-- self.process_urls(urls).await- }-- async fn check_feeds(&mut self) -> CrawlResults {- let mut feeds = self.feeds.clone();-- let mut futures = Vec::new();- for feeds in feeds.iter_mut() {- if feeds.last_checked + FEED_CHECK_INTERVAL < Utc::now() {- futures.push(self.process_feed(&feeds.feed));- feeds.last_checked = Utc::now();- }- }-- let res = futures::future::join_all(futures).await;-- self.feeds = feeds;-- if res.iter().filter_map(|r| r.as_ref().ok()).any(|r| *r) {- CrawlResults::HasInserts- } else {- CrawlResults::None- }- }-}--impl Indexer {- fn maybe_write_batch_to_index(&self) {- let batch = self.write_batch.lock().unwrap_or_else(|e| e.into_inner());-- if batch.len() < BATCH_SIZE {- return;- }-- let prepared = self.worker.prepare_webpages(&batch);-- let search_index = self.search_index.write().unwrap_or_else(|e| e.into_inner());- for webpage in &prepared {- search_index.insert(webpage).ok();- }- }-- fn write_batch_to_index(&self) {- let batch = self.write_batch.lock().unwrap_or_else(|e| e.into_inner());-- if batch.is_empty() {- return;- }-- let prepared = self.worker.prepare_webpages(&batch);-- let search_index = self.search_index.write().unwrap_or_else(|e| e.into_inner());- for webpage in &prepared {- search_index.insert(webpage).ok();- }- }-}--pub struct Index {- search_index: Arc<RwLock<crate::index::Index>>,-}--impl Index {- pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {- let mut search_index = crate::index::Index::open(path.as_ref().join("index"))?;- search_index.prepare_writer()?;- search_index.set_auto_merge_policy();-- let search_index = Arc::new(RwLock::new(search_index));-- Ok(Self { search_index })- }-- fn commit(&self) {- self.search_index- .write()- .unwrap_or_else(|e| e.into_inner())- .commit()- .ok();- }-- fn prune(&self) {- self.search_index- .write()- .unwrap_or_else(|e| e.into_inner())- .delete_all_before(SystemTime::now() - TTL)- .ok();- }-- fn clone_inner_index(&self) -> Arc<RwLock<crate::index::Index>> {- self.search_index.clone()- }-- pub fn read(&self) -> RwLockReadGuard<'_, crate::index::Index> {- self.search_index.read().unwrap_or_else(|e| e.into_inner())- }-- pub fn write(&self) -> RwLockWriteGuard<'_, crate::index::Index> {- self.search_index.write().unwrap_or_else(|e| e.into_inner())- }-}--impl DatumStream for Indexer {- async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {- self.write_batch- .lock()- .unwrap_or_else(|e| e.into_inner())- .push(crawl_datum.into());-- self.maybe_write_batch_to_index();-- Ok(())- }-- async fn finish(&self) -> Result<()> {- self.write_batch_to_index();-- self.search_index- .write()- .unwrap_or_else(|e| e.into_inner())- .commit()?;-- Ok(())- }-}--impl From<&LiveIndexConfig> for CrawlerConfig {- fn from(live: &LiveIndexConfig) -> Self {- Self {- num_worker_threads: 1, // no impact- user_agent: live.user_agent.clone(),- robots_txt_cache_sec: live.robots_txt_cache_sec,- politeness_factor: live.politeness_factor,- min_crawl_delay_ms: live.min_crawl_delay_ms,- max_crawl_delay_ms: live.max_crawl_delay_ms,- max_politeness_factor: live.max_politeness_factor,- max_url_slowdown_retry: live.max_url_slowdown_retry,- max_redirects: live.max_redirects,- dry_run: false,- timeout_seconds: live.timeout_seconds,- // no impact- s3: crate::config::S3Config {- bucket: String::new(),- folder: String::new(),- access_key: String::new(),- secret_key: String::new(),- endpoint: String::new(),- },- router_hosts: Vec::new(),- }- }-}--pub struct IndexManager {- index: Arc<Index>,- crawler: Crawler,-}--impl IndexManager {- pub fn new(config: LiveIndexConfig) -> Result<Self> {- let index = Index::new(&config.index_path)?;- let indexer = Arc::new(Indexer {- search_index: index.clone_inner_index(),- worker: IndexingWorker::new(config.clone()),- write_batch: Arc::new(Mutex::new(Vec::new())),- });-- let crawler_config = CrawlerConfig::from(&config);-- let crawler = Crawler::new(- Split::open(&config.split_path)?,- indexer,- DownloadedDb::open(&config.downloaded_db_path)?,- Arc::new(crawler_config),- )?;-- Ok(Self {- index: Arc::new(index),- crawler,- })- }-- pub async fn run(mut self) {- let mut has_inserts = false;- let mut last_commit = Utc::now();- let mut last_prune = Utc::now();-- loop {- if let CrawlResults::HasInserts = self.crawler.check_feeds().await {- has_inserts = true;- }-- if last_prune + PRUNE_INTERVAL < Utc::now() {- self.index.prune();-- last_prune = Utc::now();- }-- if last_commit + AUTO_COMMIT_INTERVAL < Utc::now() && has_inserts {- self.index.commit();-- last_commit = Utc::now();- has_inserts = false;- }-- tokio::time::sleep(EVENT_LOOP_INTERVAL).await;- }- }-- pub fn index(&self) -> Arc<Index> {- self.index.clone()- }-}diff --git a/crates/core/src/live_index/crawler.rs b/crates/core/src/live_index/crawler.rsnew file mode 100644index 00000000..b389d7f9--- /dev/null+++ b/crates/core/src/live_index/crawler.rs@@ -0,0 +1,143 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use crate::Result;+use std::sync::Arc;++use chrono::Utc;+use url::Url;++use crate::{+ config::CrawlerConfig,+ crawler::{reqwest_client, JobExecutor, RetrieableUrl, WeightedUrl, WorkerJob},+ feed::{+ self,+ scheduler::{Domain, DomainFeeds, Split},+ },+};++use super::{downloaded_db::DownloadedDb, indexer::Indexer, Feeds, FEED_CHECK_INTERVAL};++pub enum CrawlResults {+ None,+ HasInserts,+}++pub struct Crawler {+ feeds: Vec<Feeds>,+ indexer: Arc<Indexer>,+ downloaded_db: DownloadedDb,+ config: Arc<CrawlerConfig>,+ client: reqwest::Client,+}++impl Crawler {+ pub fn new(+ split: Split,+ indexer: Arc<Indexer>,+ downloaded_db: DownloadedDb,+ config: Arc<CrawlerConfig>,+ ) -> Result<Self> {+ let client = reqwest_client(&config)?;++ Ok(Self {+ feeds: split.into(),+ indexer,+ downloaded_db,+ config,+ client,+ })+ }++ async fn process_urls(&self, urls: Vec<Url>) -> Result<bool> {+ if urls.is_empty() {+ return Ok(false);+ }++ let domain = urls.first().unwrap().into();+ let job = WorkerJob {+ domain,+ urls: urls+ .clone()+ .into_iter()+ .map(|url| RetrieableUrl::from(WeightedUrl { url, weight: 1.0 }))+ .collect(),+ wandering_urls: 0,+ };++ let executor = JobExecutor::new(+ job,+ self.client.clone(),+ self.config.clone(),+ self.indexer.clone(),+ );+ executor.run().await;++ for url in &urls {+ self.downloaded_db.insert(url)?;+ }++ Ok(true)+ }++ async fn process_feed(&self, domain_feeds: &DomainFeeds) -> Result<bool> {+ let mut urls = Vec::new();++ for feed in domain_feeds.feeds.iter() {+ let content = self+ .client+ .get(feed.url.as_str())+ .send()+ .await?+ .text()+ .await?;++ let feed = feed::parse(&content, feed.kind)?;++ for url in feed.links {+ if !self.downloaded_db.has_downloaded(&url)?+ && Domain::from(&url) == domain_feeds.domain+ {+ urls.push(url);+ }+ }+ }++ self.process_urls(urls).await+ }++ pub async fn check_feeds(&mut self) -> CrawlResults {+ let mut feeds = self.feeds.clone();++ let mut futures = Vec::new();+ for feeds in feeds.iter_mut() {+ if feeds.last_checked + FEED_CHECK_INTERVAL < Utc::now() {+ futures.push(self.process_feed(&feeds.feed));+ feeds.last_checked = Utc::now();+ }+ }++ let res = futures::future::join_all(futures).await;++ self.feeds = feeds;++ if res.iter().filter_map(|r| r.as_ref().ok()).any(|r| *r) {+ CrawlResults::HasInserts+ } else {+ CrawlResults::None+ }+ }+}diff --git a/crates/core/src/live_index/downloaded_db.rs b/crates/core/src/live_index/downloaded_db.rsnew file mode 100644index 00000000..cca209cc--- /dev/null+++ b/crates/core/src/live_index/downloaded_db.rs@@ -0,0 +1,66 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use url::Url;++use crate::Result;+use std::path::Path;++use super::TTL;++pub struct DownloadedDb {+ db: rocksdb::DB,+}++impl DownloadedDb {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let mut options = rocksdb::Options::default();+ options.create_if_missing(true);++ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)+ options.set_level_compaction_dynamic_level_bytes(true);+ options.set_bytes_per_sync(1048576);+ let mut block_options = rocksdb::BlockBasedOptions::default();+ block_options.set_block_size(16 * 1024);+ block_options.set_format_version(5);+ block_options.set_cache_index_and_filter_blocks(true);+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);++ block_options.set_ribbon_filter(10.0);++ options.set_block_based_table_factory(&block_options);+ options.optimize_for_point_lookup(512); // 512 mb++ let db = rocksdb::DB::open_with_ttl(&options, path, TTL)?;+ Ok(Self { db })+ }++ pub fn has_downloaded(&self, url: &Url) -> Result<bool> {+ let key = url.as_str().as_bytes();++ if self.db.key_may_exist(key) {+ self.db.get(key).map(|v| v.is_some()).map_err(Into::into)+ } else {+ Ok(false)+ }+ }++ pub fn insert(&self, url: &Url) -> Result<()> {+ let key = url.as_str().as_bytes();+ self.db.put(key, b"")?;+ Ok(())+ }+}diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsnew file mode 100644index 00000000..ec0b09ac--- /dev/null+++ b/crates/core/src/live_index/index.rs@@ -0,0 +1,69 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++use std::{+ path::Path,+ sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},+ time::SystemTime,+};++use crate::Result;++use super::TTL;++pub struct Index {+ search_index: Arc<RwLock<crate::index::Index>>,+}++impl Index {+ pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {+ let mut search_index = crate::index::Index::open(path.as_ref().join("index"))?;+ search_index.prepare_writer()?;+ search_index.set_auto_merge_policy();++ let search_index = Arc::new(RwLock::new(search_index));++ Ok(Self { search_index })+ }++ pub fn commit(&self) {+ self.search_index+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .commit()+ .ok();+ }++ pub fn prune(&self) {+ self.search_index+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .delete_all_before(SystemTime::now() - TTL)+ .ok();+ }++ pub fn clone_inner_index(&self) -> Arc<RwLock<crate::index::Index>> {+ self.search_index.clone()+ }++ pub fn read(&self) -> RwLockReadGuard<'_, crate::index::Index> {+ self.search_index.read().unwrap_or_else(|e| e.into_inner())+ }++ pub fn write(&self) -> RwLockWriteGuard<'_, crate::index::Index> {+ self.search_index.write().unwrap_or_else(|e| e.into_inner())+ }+}diff --git a/crates/core/src/live_index/index_manager.rs b/crates/core/src/live_index/index_manager.rsnew file mode 100644index 00000000..217cd2f9--- /dev/null+++ b/crates/core/src/live_index/index_manager.rs@@ -0,0 +1,89 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::sync::Arc;++use chrono::Utc;++use crate::{+ config::{CrawlerConfig, LiveIndexConfig},+ feed::scheduler::Split,+};++use super::{+ crawler::{CrawlResults, Crawler},+ downloaded_db::DownloadedDb,+ indexer::Indexer,+ Index, AUTO_COMMIT_INTERVAL, EVENT_LOOP_INTERVAL, PRUNE_INTERVAL,+};+use crate::Result;++pub struct IndexManager {+ index: Arc<Index>,+ crawler: Crawler,+}++impl IndexManager {+ pub fn new(config: LiveIndexConfig) -> Result<Self> {+ let index = Index::new(&config.index_path)?;+ let indexer = Arc::new(Indexer::new(index.clone_inner_index(), config.clone()));++ let crawler_config = CrawlerConfig::from(&config);++ let crawler = Crawler::new(+ Split::open(&config.split_path)?,+ indexer,+ DownloadedDb::open(&config.downloaded_db_path)?,+ Arc::new(crawler_config),+ )?;++ Ok(Self {+ index: Arc::new(index),+ crawler,+ })+ }++ pub async fn run(mut self) {+ let mut has_inserts = false;+ let mut last_commit = Utc::now();+ let mut last_prune = Utc::now();++ loop {+ if let CrawlResults::HasInserts = self.crawler.check_feeds().await {+ has_inserts = true;+ }++ if last_prune + PRUNE_INTERVAL < Utc::now() {+ self.index.prune();++ last_prune = Utc::now();+ }++ if last_commit + AUTO_COMMIT_INTERVAL < Utc::now() && has_inserts {+ self.index.commit();++ last_commit = Utc::now();+ has_inserts = false;+ }++ tokio::time::sleep(EVENT_LOOP_INTERVAL).await;+ }+ }++ pub fn index(&self) -> Arc<Index> {+ self.index.clone()+ }+}diff --git a/crates/core/src/live_index/indexer.rs b/crates/core/src/live_index/indexer.rsnew file mode 100644index 00000000..6ba6fc15--- /dev/null+++ b/crates/core/src/live_index/indexer.rs@@ -0,0 +1,96 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::sync::{Arc, Mutex, RwLock};++use crate::{+ config::LiveIndexConfig,+ crawler::{CrawlDatum, DatumStream},+ entrypoint::indexer::IndexingWorker,+ Result,+};++use super::BATCH_SIZE;++pub struct Indexer {+ search_index: Arc<RwLock<crate::index::Index>>,+ worker: IndexingWorker,+ write_batch: Arc<Mutex<Vec<crate::entrypoint::indexer::IndexableWebpage>>>,+}++impl Indexer {+ pub fn new(search_index: Arc<RwLock<crate::index::Index>>, config: LiveIndexConfig) -> Self {+ Self {+ search_index,+ worker: IndexingWorker::new(config),+ write_batch: Arc::new(Mutex::new(Vec::new())),+ }+ }++ pub fn maybe_write_batch_to_index(&self) {+ let batch = self.write_batch.lock().unwrap_or_else(|e| e.into_inner());++ if batch.len() < BATCH_SIZE {+ return;+ }++ let prepared = self.worker.prepare_webpages(&batch);++ let search_index = self.search_index.write().unwrap_or_else(|e| e.into_inner());+ for webpage in &prepared {+ search_index.insert(webpage).ok();+ }+ }++ pub fn write_batch_to_index(&self) {+ let batch = self.write_batch.lock().unwrap_or_else(|e| e.into_inner());++ if batch.is_empty() {+ return;+ }++ let prepared = self.worker.prepare_webpages(&batch);++ let search_index = self.search_index.write().unwrap_or_else(|e| e.into_inner());+ for webpage in &prepared {+ search_index.insert(webpage).ok();+ }+ }+}++impl DatumStream for Indexer {+ async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {+ self.write_batch+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .push(crawl_datum.into());++ self.maybe_write_batch_to_index();++ Ok(())+ }++ async fn finish(&self) -> Result<()> {+ self.write_batch_to_index();++ self.search_index+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .commit()?;++ Ok(())+ }+}diff --git a/crates/core/src/live_index/mod.rs b/crates/core/src/live_index/mod.rsnew file mode 100644index 00000000..65a70bbc--- /dev/null+++ b/crates/core/src/live_index/mod.rs@@ -0,0 +1,85 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::time::Duration;++use chrono::{DateTime, Utc};++use crate::{+ config::{CrawlerConfig, LiveIndexConfig},+ feed::scheduler::{DomainFeeds, Split},+};++pub use self::index::Index;+pub use self::index_manager::IndexManager;++mod crawler;+mod downloaded_db;+mod index;+mod index_manager;+mod indexer;++const TTL: Duration = Duration::from_secs(60 * 60 * 24 * 60); // 60 days+const PRUNE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour+const FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutes+const AUTO_COMMIT_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 minutes+const EVENT_LOOP_INTERVAL: Duration = Duration::from_secs(5);+const BATCH_SIZE: usize = 512;++#[derive(Debug, Clone)]+struct Feeds {+ last_checked: DateTime<Utc>,+ feed: DomainFeeds,+}++impl From<Split> for Vec<Feeds> {+ fn from(split: Split) -> Self {+ split+ .feeds+ .into_iter()+ .map(|feed| Feeds {+ last_checked: Utc::now(),+ feed,+ })+ .collect()+ }+}++impl From<&LiveIndexConfig> for CrawlerConfig {+ fn from(live: &LiveIndexConfig) -> Self {+ Self {+ num_worker_threads: 1, // no impact+ user_agent: live.user_agent.clone(),+ robots_txt_cache_sec: live.robots_txt_cache_sec,+ politeness_factor: live.politeness_factor,+ min_crawl_delay_ms: live.min_crawl_delay_ms,+ max_crawl_delay_ms: live.max_crawl_delay_ms,+ max_politeness_factor: live.max_politeness_factor,+ max_url_slowdown_retry: live.max_url_slowdown_retry,+ max_redirects: live.max_redirects,+ dry_run: false,+ timeout_seconds: live.timeout_seconds,+ // no impact+ s3: crate::config::S3Config {+ bucket: String::new(),+ folder: String::new(),+ access_key: String::new(),+ secret_key: String::new(),+ endpoint: String::new(),+ },+ router_hosts: Vec::new(),+ }+ }+}