Skip to content

web search done right

Live index without replication (#221)

Mikkel Denker21ba4cde65b9
diff --git a/Cargo.lock b/Cargo.lock
index 7626b520..c5ad6596 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5228,6 +5228,7 @@ dependencies = [
"scylla",
"serde",
"serde_json",
+ "simple_wal",
"speedy_kv",
"strum 0.26.3",
"tantivy",
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index a6a63085..dd532d45 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -29,7 +29,8 @@ base64.workspace = true
bincode.workspace = true
bitflags.workspace = true
bitvec.workspace = true
-bloom = {path = "../bloom"}
+bloom = { path = "../bloom" }
+simple_wal = { path = "../simple-wal" }
bytecount.workspace = true
bytemuck.workspace = true
candle-core.workspace = true
@@ -48,7 +49,7 @@ dirs.workspace = true
encoding_rs.workspace = true
enum_dispatch.workspace = true
fend-core.workspace = true
-file_store = {path = "../file-store"}
+file_store = { path = "../file-store" }
flate2.workspace = true
fnv.workspace = true
fst.workspace = true
@@ -58,7 +59,7 @@ http.workspace = true
image.workspace = true
indicatif.workspace = true
itertools.workspace = true
-kuchiki = {path = "../kuchiki"}
+kuchiki = { path = "../kuchiki" }
lending-iter.workspace = true
log.workspace = true
logos.workspace = true
@@ -71,7 +72,7 @@ nom.workspace = true
num-bigint.workspace = true
num-rational.workspace = true
openraft.workspace = true
-optics = {path = "../optics"}
+optics = { path = "../optics" }
percent-encoding.workspace = true
postcard.workspace = true
publicsuffix.workspace = true
@@ -84,14 +85,14 @@ reqwest.workspace = true
ring.workspace = true
rio_api.workspace = true
rio_turtle.workspace = true
-robotstxt = {path = "../robotstxt"}
+robotstxt = { path = "../robotstxt" }
rust-s3.workspace = true
rust-stemmers.workspace = true
rustc-hash.workspace = true
scylla.workspace = true
serde.workspace = true
serde_json.workspace = true
-speedy_kv = {path = "../speedy-kv"}
+speedy_kv = { path = "../speedy-kv" }
strum.workspace = true
tantivy.workspace = true
thiserror.workspace = true
@@ -110,7 +111,7 @@ utoipa-swagger-ui.workspace = true
utoipa.workspace = true
uuid.workspace = true
whatlang.workspace = true
-zimba = {path = "../zimba"}
+zimba = { path = "../zimba" }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator.workspace = true
diff --git a/crates/core/src/entrypoint/feed_indexer.rs b/crates/core/src/entrypoint/feed_indexer.rs
index 1b99bb27..01f38444 100644
--- a/crates/core/src/entrypoint/feed_indexer.rs
+++ b/crates/core/src/entrypoint/feed_indexer.rs
@@ -48,7 +48,7 @@ impl From<String> for IndexPointer {
}
}
-pub struct IndexingWorker {}
+pub struct IndexingWorker;
impl IndexingWorker {
fn process_job(&self, job: &Job) -> FeedIndex {
let name = job.warc_path.split('/').last().unwrap().to_string();
@@ -105,7 +105,7 @@ pub fn build(config: FeedIndexingConfig) -> Result<()> {
let job_config: WarcSource = config.warc_source.clone();
- let worker = IndexingWorker {};
+ let worker = IndexingWorker;
let indexes = warc_paths
.into_par_iter()
diff --git a/crates/core/src/entrypoint/indexer/indexable_webpage.rs b/crates/core/src/entrypoint/indexer/indexable_webpage.rs
index 290d8ff9..13b07309 100644
--- a/crates/core/src/entrypoint/indexer/indexable_webpage.rs
+++ b/crates/core/src/entrypoint/indexer/indexable_webpage.rs
@@ -17,6 +17,7 @@
use crate::crawler::CrawlDatum;
use crate::warc::WarcRecord;
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct IndexableWebpage {
pub url: String,
pub body: String,
diff --git a/crates/core/src/entrypoint/live_index.rs b/crates/core/src/entrypoint/live_index.rs
index d46ccaf1..e9ab5145 100644
--- a/crates/core/src/entrypoint/live_index.rs
+++ b/crates/core/src/entrypoint/live_index.rs
@@ -14,40 +14,44 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{path::Path, sync::Arc};
+use std::sync::Arc;
use crate::{
- config::{LiveIndexConfig, LiveIndexSchedulerConfig},
+ config::LiveIndexConfig,
distributed::{
cluster::Cluster,
member::{Member, Service},
sonic::{self, service::sonic_service},
},
- feed::{self, index::FeedIndex},
inverted_index,
- live_index::{Index, IndexManager},
+ live_index::{IndexManager, LiveIndex},
searcher::{InitialWebsiteResult, LocalSearcher},
- webgraph::WebgraphBuilder,
};
use anyhow::Result;
use tracing::info;
-use super::search_server::{RetrieveWebsites, Search};
+use super::{
+ indexer::IndexableWebpage,
+ search_server::{RetrieveWebsites, Search},
+};
-sonic_service!(SearchService, [RetrieveWebsites, Search]);
+sonic_service!(LiveIndexService, [RetrieveWebsites, Search, IndexWebpages]);
-pub struct SearchService {
- local_searcher: LocalSearcher<Arc<Index>>,
+pub struct LiveIndexService {
+ local_searcher: LocalSearcher<Arc<LiveIndex>>,
+ index: Arc<LiveIndex>,
// dropping the handle leaves the cluster
#[allow(unused)]
cluster_handle: Cluster,
}
-impl SearchService {
+impl LiveIndexService {
async fn new(config: LiveIndexConfig) -> Result<Self> {
let manager = IndexManager::new(config.clone())?;
let local_searcher = LocalSearcher::new(manager.index());
+ let index = manager.index();
+
tokio::task::spawn(manager.run());
let cluster_handle = Cluster::join(
@@ -66,13 +70,14 @@ impl SearchService {
Ok(Self {
local_searcher,
cluster_handle,
+ index,
})
}
}
-impl sonic::service::Message<SearchService> for RetrieveWebsites {
+impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {
type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
- async fn handle(self, server: &SearchService) -> Self::Response {
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
server
.local_searcher
.retrieve_websites(&self.websites, &self.query)
@@ -80,17 +85,34 @@ impl sonic::service::Message<SearchService> for RetrieveWebsites {
}
}
-impl sonic::service::Message<SearchService> for Search {
+impl sonic::service::Message<LiveIndexService> for Search {
type Response = Option<InitialWebsiteResult>;
- async fn handle(self, server: &SearchService) -> Self::Response {
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
server.local_searcher.search_initial(&self.query, true).ok()
}
}
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct IndexWebpages {
+ pages: Vec<IndexableWebpage>,
+}
+
+impl sonic::service::Message<LiveIndexService> for IndexWebpages {
+ type Response = ();
+
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ server.index.insert(&self.pages)
+ }
+}
+
pub async fn serve(config: LiveIndexConfig) -> Result<()> {
let addr = config.host;
- let server = SearchService::new(config).await?.bind(&addr).await.unwrap();
+ let server = LiveIndexService::new(config)
+ .await?
+ .bind(&addr)
+ .await
+ .unwrap();
info!("live index is ready to accept requests on {}", addr);
@@ -100,18 +122,3 @@ pub async fn serve(config: LiveIndexConfig) -> Result<()> {
}
}
}
-
-pub fn schedule(config: LiveIndexSchedulerConfig) -> Result<()> {
- let feed_index = FeedIndex::open(config.feed_index_path)?;
- let host_harmonic = speedy_kv::Db::open_or_create(
- Path::new(&config.host_centrality_store_path).join("harmonic"),
- )
- .unwrap();
- let host_graph = WebgraphBuilder::new(config.host_graph_path).open();
-
- let schedule =
- feed::scheduler::schedule(&feed_index, &host_harmonic, &host_graph, config.num_splits);
- schedule.save(config.schedule_path)?;
-
- Ok(())
-}
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rs
index 60f2a7df..267db096 100644
--- a/crates/core/src/inverted_index/indexing.rs
+++ b/crates/core/src/inverted_index/indexing.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
+use tantivy::index::SegmentId;
use tantivy::merge_policy::NoMergePolicy;
use tantivy::{IndexWriter, SegmentMeta};
@@ -152,6 +153,23 @@ impl InvertedIndex {
Ok(())
}
+ #[allow(clippy::missing_panics_doc)] // cannot panic as writer is prepared
+ pub fn merge_segments_by_id(&mut self, segments: &[SegmentId]) -> Result<()> {
+ self.prepare_writer()?;
+
+ if segments.is_empty() {
+ return Ok(());
+ }
+
+ self.writer
+ .as_mut()
+ .expect("writer has not been prepared")
+ .merge(segments)
+ .wait()?;
+
+ Ok(())
+ }
+
#[must_use]
pub fn merge(mut self, mut other: InvertedIndex) -> Self {
self.prepare_writer().expect("failed to prepare writer");
@@ -228,6 +246,49 @@ impl InvertedIndex {
res
}
+ #[allow(clippy::missing_panics_doc)] // cannot panic as writer is prepared
+ pub fn delete_segments_by_id(&mut self, segment_ids: &[SegmentId]) -> Result<()> {
+ if segment_ids.is_empty() {
+ return Ok(());
+ }
+
+ let segments: HashSet<_> = segment_ids.iter().copied().collect();
+ let to_delete: HashSet<_> = self
+ .tantivy_index
+ .searchable_segments()?
+ .into_iter()
+ .filter(|seg| segments.contains(&seg.id()))
+ .flat_map(|seg| seg.meta().list_files().into_iter())
+ .collect();
+
+ let mut index_meta = self.tantivy_index.load_metas()?;
+
+ index_meta.segments = index_meta
+ .segments
+ .clone()
+ .into_iter()
+ .filter(|seg| !segments.contains(&seg.id()))
+ .collect();
+
+ let living_files: HashSet<_> = self
+ .tantivy_index
+ .directory()
+ .list_managed_files()
+ .difference(&to_delete)
+ .cloned()
+ .collect();
+
+ self.tantivy_index
+ .directory_mut()
+ .garbage_collect(|| living_files)?;
+
+ self.tantivy_index.save_metas(&index_meta)?;
+
+ self.reader.reload()?;
+
+ Ok(())
+ }
+
pub fn stop(mut self) {
self.writer
.take()
@@ -236,3 +297,94 @@ impl InvertedIndex {
.unwrap()
}
}
+
+#[cfg(test)]
+mod test {
+ use crate::{
+ config::CollectorConfig,
+ inverted_index::tests::search,
+ query::Query,
+ ranking::{LocalRanker, SignalComputer},
+ searcher::SearchQuery,
+ };
+
+ use super::*;
+
+ #[test]
+ fn test_delete_segments() {
+ let mut index = InvertedIndex::temporary().expect("Unable to open index");
+
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ TEST
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ index
+ .insert(
+ &Webpage::test_parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ TEST
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+
+ let segments = index.segment_ids();
+
+ assert_eq!(segments.len(), 2);
+ index.delete_segments_by_id(&segments).unwrap();
+
+ let segments = index.segment_ids();
+ assert!(segments.is_empty());
+
+ let ctx = index.local_search_ctx();
+ let query = Query::parse(
+ &ctx,
+ &SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ },
+ &index,
+ )
+ .expect("Failed to parse query");
+
+ let ranker = LocalRanker::new(
+ SignalComputer::new(Some(&query)),
+ ctx.columnfield_reader.clone(),
+ CollectorConfig::default(),
+ );
+
+ let result =
+ search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
+ assert_eq!(result.documents.len(), 0);
+ }
+}
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 3eec73f6..621084ed 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -37,6 +37,7 @@ pub use retrieved_webpage::RetrievedWebpage;
use tantivy::directory::MmapDirectory;
+use tantivy::index::SegmentId;
use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
use tantivy::{IndexReader, IndexWriter};
@@ -202,6 +203,10 @@ impl InvertedIndex {
&self.schema
}
+ pub fn segment_ids(&self) -> Vec<SegmentId> {
+ self.tantivy_index.searchable_segment_ids().unwrap()
+ }
+
pub fn num_segments(&self) -> usize {
self.tantivy_index.searchable_segments().unwrap().len()
}
@@ -249,7 +254,7 @@ mod tests {
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
- fn search(
+ pub fn search(
index: &InvertedIndex,
query: &Query,
ctx: &Ctx,
diff --git a/crates/core/src/live_index/downloaded_db.rs b/crates/core/src/live_index/crawler/downloaded_db.rs
similarity index 99%
rename from crates/core/src/live_index/downloaded_db.rs
rename to crates/core/src/live_index/crawler/downloaded_db.rs
index 69a51d37..30bfc9e7 100644
--- a/crates/core/src/live_index/downloaded_db.rs
+++ b/crates/core/src/live_index/crawler/downloaded_db.rs
@@ -20,7 +20,7 @@ use url::Url;
use crate::Result;
use std::{path::Path, time::Duration};
-use super::TTL;
+use crate::live_index::TTL;
#[derive(Debug, Clone)]
struct InsertionTime {
diff --git a/crates/core/src/live_index/crawler.rs b/crates/core/src/live_index/crawler/mod.rs
similarity index 72%
rename from crates/core/src/live_index/crawler.rs
rename to crates/core/src/live_index/crawler/mod.rs
index b389d7f9..84637499 100644
--- a/crates/core/src/live_index/crawler.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -14,6 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/
+mod downloaded_db;
+
use crate::Result;
use std::sync::Arc;
@@ -22,14 +24,16 @@ use url::Url;
use crate::{
config::CrawlerConfig,
- crawler::{reqwest_client, JobExecutor, RetrieableUrl, WeightedUrl, WorkerJob},
+ crawler::reqwest_client,
feed::{
self,
scheduler::{Domain, DomainFeeds, Split},
},
};
-use super::{downloaded_db::DownloadedDb, indexer::Indexer, Feeds, FEED_CHECK_INTERVAL};
+use downloaded_db::DownloadedDb;
+
+use super::{Feeds, FEED_CHECK_INTERVAL};
pub enum CrawlResults {
None,
@@ -38,16 +42,14 @@ pub enum CrawlResults {
pub struct Crawler {
feeds: Vec<Feeds>,
- indexer: Arc<Indexer>,
downloaded_db: DownloadedDb,
- config: Arc<CrawlerConfig>,
+ _config: Arc<CrawlerConfig>,
client: reqwest::Client,
}
impl Crawler {
pub fn new(
split: Split,
- indexer: Arc<Indexer>,
downloaded_db: DownloadedDb,
config: Arc<CrawlerConfig>,
) -> Result<Self> {
@@ -55,42 +57,43 @@ impl Crawler {
Ok(Self {
feeds: split.into(),
- indexer,
downloaded_db,
- config,
+ _config: config,
client,
})
}
- async fn process_urls(&self, urls: Vec<Url>) -> Result<bool> {
- if urls.is_empty() {
- return Ok(false);
- }
-
- let domain = urls.first().unwrap().into();
- let job = WorkerJob {
- domain,
- urls: urls
- .clone()
- .into_iter()
- .map(|url| RetrieableUrl::from(WeightedUrl { url, weight: 1.0 }))
- .collect(),
- wandering_urls: 0,
- };
-
- let executor = JobExecutor::new(
- job,
- self.client.clone(),
- self.config.clone(),
- self.indexer.clone(),
- );
- executor.run().await;
-
- for url in &urls {
- self.downloaded_db.insert(url)?;
- }
-
- Ok(true)
+ async fn process_urls(&self, _urls: Vec<Url>) -> Result<bool> {
+ todo!()
+
+ // if urls.is_empty() {
+ // return Ok(false);
+ // }
+
+ // let domain = urls.first().unwrap().into();
+ // let job = WorkerJob {
+ // domain,
+ // urls: urls
+ // .clone()
+ // .into_iter()
+ // .map(|url| RetrieableUrl::from(WeightedUrl { url, weight: 1.0 }))
+ // .collect(),
+ // wandering_urls: 0,
+ // };
+
+ // let executor = JobExecutor::new(
+ // job,
+ // self.client.clone(),
+ // self.config.clone(),
+ // self.indexer.clone(),
+ // );
+ // executor.run().await;
+
+ // for url in &urls {
+ // self.downloaded_db.insert(url)?;
+ // }
+
+ // Ok(true)
}
async fn process_feed(&self, domain_feeds: &DomainFeeds) -> Result<bool> {
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index d1813909..9110b16e 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -15,48 +15,274 @@
// along with this program. If not, see <https://www.gnu.org/licenses/
use std::{
- path::Path,
- sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
+ io::{BufReader, BufWriter},
+ path::{Path, PathBuf},
+ sync::{Arc, RwLock, RwLockReadGuard},
};
-use crate::Result;
+use chrono::{DateTime, NaiveDate, Utc};
+use itertools::Itertools;
+use simple_wal::Wal;
+use tantivy::index::SegmentId;
-pub struct Index {
- search_index: Arc<RwLock<crate::index::Index>>,
+use std::collections::{HashMap, HashSet};
+
+use crate::{
+ config::{LiveIndexConfig, SnippetConfig},
+ entrypoint::indexer::{IndexableWebpage, IndexingWorker},
+ live_index::{BATCH_SIZE, TTL},
+ searcher::SearchableIndex,
+ Result,
+};
+
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
+struct Segment {
+ id: SegmentId,
+ created: DateTime<Utc>,
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]
+struct Meta {
+ segments: Vec<Segment>,
}
-impl Index {
- pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {
- let mut search_index = crate::index::Index::open(path.as_ref().join("index"))?;
- search_index.prepare_writer()?;
- search_index.set_auto_merge_policy();
+impl Meta {
+ pub fn open_or_create<P: AsRef<Path>>(path: P) -> Self {
+ if !path.as_ref().exists() {
+ let meta = Meta::default();
+ meta.save(path);
+
+ meta
+ } else {
+ let file = std::fs::File::open(path).unwrap();
+ let reader = BufReader::new(file);
+ serde_json::from_reader(reader).unwrap()
+ }
+ }
+
+ pub fn save<P: AsRef<Path>>(&self, path: P) {
+ let file = std::fs::OpenOptions::new()
+ .create(true)
+ .write(true)
+ .read(true)
+ .truncate(true)
+ .open(path)
+ .unwrap();
+ let writer = BufWriter::new(file);
+
+ serde_json::to_writer_pretty(writer, self).unwrap();
+ }
+}
+
+pub struct InnerIndex {
+ index: crate::index::Index,
+ write_ahead_log: Wal<crate::entrypoint::indexer::IndexableWebpage>,
+ has_inserts: bool,
+ indexing_worker: IndexingWorker,
+ path: PathBuf,
+ meta: Meta,
+}
+
+impl InnerIndex {
+ pub fn new(config: LiveIndexConfig) -> Result<Self> {
+ let path = Path::new(&config.index_path);
+ let mut index = crate::index::Index::open(path.join("index"))?;
+ index.prepare_writer()?;
+
+ let write_ahead_log = Wal::open(path.join("wal"))?;
+ let wal_count = write_ahead_log.iter()?.count();
- let search_index = Arc::new(RwLock::new(search_index));
+ let worker = IndexingWorker::new(config.clone());
- Ok(Self { search_index })
+ let meta = Meta::open_or_create(path.join("meta.json"));
+
+ Ok(Self {
+ index,
+ write_ahead_log,
+ indexing_worker: worker,
+ has_inserts: wal_count > 0,
+ meta,
+ path: path.to_path_buf(),
+ })
+ }
+
+ pub fn prune_segments(&mut self) {
+ let old_segments: Vec<_> = self
+ .meta
+ .segments
+ .iter()
+ .filter_map(|segment| {
+ if segment.created + TTL < Utc::now() {
+ Some(segment.id)
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ self.index
+ .inverted_index
+ .delete_segments_by_id(&old_segments)
+ .unwrap();
+
+ self.update_meta();
+ }
+
+ pub fn compact_segments_by_date(&mut self) {
+ let mut segments_by_date: HashMap<NaiveDate, Vec<SegmentId>> = HashMap::new();
+
+ for segment in self.meta.segments.clone() {
+ segments_by_date
+ .entry(segment.created.date_naive())
+ .or_default()
+ .push(segment.id);
+ }
+
+ for (_, segments) in segments_by_date {
+ if segments.len() <= 1 {
+ continue;
+ }
+
+ self.index
+ .inverted_index
+ .merge_segments_by_id(&segments)
+ .unwrap();
+ }
+
+ self.update_meta();
+ }
+
+ fn update_meta(&mut self) {
+ let segments_in_index: HashSet<_> = self
+ .index
+ .inverted_index
+ .segment_ids()
+ .into_iter()
+ .collect();
+
+ let segments_in_meta: HashSet<_> = self
+ .meta
+ .segments
+ .clone()
+ .into_iter()
+ .map(|segment| segment.id)
+ .collect();
+
+ // remove all segments from meta that is not present in the index
+ let to_remove: HashSet<_> = segments_in_meta
+ .iter()
+ .filter(|segment| !segments_in_index.contains(segment))
+ .collect();
+
+ self.meta.segments = self
+ .meta
+ .segments
+ .clone()
+ .into_iter()
+ .filter(|segment| !to_remove.contains(&segment.id))
+ .collect();
+
+ // insert all segments from index that is not already in meta
+ for id in segments_in_index
+ .into_iter()
+ .filter(|segment| !segments_in_meta.contains(segment))
+ {
+ self.meta.segments.push(Segment {
+ id,
+ created: Utc::now(),
+ })
+ }
+
+ self.meta.save(self.path.join("meta.json"));
+ }
+
+ pub fn index(&self) -> &crate::index::Index {
+ &self.index
+ }
+
+ pub fn insert(&mut self, pages: &[IndexableWebpage]) {
+ self.write_ahead_log.batch_write(pages.iter()).unwrap();
+ self.has_inserts = true;
+ }
+
+ pub fn commit(&mut self) {
+ for batch in self
+ .write_ahead_log
+ .iter()
+ .unwrap()
+ .chunks(BATCH_SIZE)
+ .into_iter()
+ {
+ let batch: Vec<_> = batch.collect();
+ for webpage in self.indexing_worker.prepare_webpages(&batch) {
+ self.index.insert(&webpage).unwrap();
+ }
+ }
+ self.index.commit().unwrap();
+ self.update_meta();
+ self.has_inserts = false;
+ }
+
+ pub fn has_inserts(&self) -> bool {
+ self.has_inserts
+ }
+}
+
+pub struct LiveIndex {
+ inner: Arc<RwLock<InnerIndex>>,
+}
+
+impl LiveIndex {
+ pub fn new(config: LiveIndexConfig) -> Result<Self> {
+ Ok(Self {
+ inner: Arc::new(RwLock::new(InnerIndex::new(config)?)),
+ })
}
pub fn commit(&self) {
- self.search_index
+ self.inner
.write()
.unwrap_or_else(|e| e.into_inner())
- .commit()
- .ok();
+ .commit();
+ }
+
+ pub fn prune_segments(&self) {
+ self.inner
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .prune_segments()
+ }
+
+ pub fn has_inserts(&self) -> bool {
+ self.inner
+ .read()
+ .unwrap_or_else(|e| e.into_inner())
+ .has_inserts()
}
- pub fn prune(&self) {
- todo!("delete index files older than TTL")
+ pub fn compact_segments_by_date(&self) {
+ self.inner
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .compact_segments_by_date()
}
- pub fn clone_inner_index(&self) -> Arc<RwLock<crate::index::Index>> {
- self.search_index.clone()
+ pub fn insert(&self, pages: &[IndexableWebpage]) {
+ self.inner
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .insert(pages)
}
- pub fn read(&self) -> RwLockReadGuard<'_, crate::index::Index> {
- self.search_index.read().unwrap_or_else(|e| e.into_inner())
+ pub fn read(&self) -> RwLockReadGuard<'_, InnerIndex> {
+ self.inner.read().unwrap_or_else(|e| e.into_inner())
}
- pub fn write(&self) -> RwLockWriteGuard<'_, crate::index::Index> {
- self.search_index.write().unwrap_or_else(|e| e.into_inner())
+ pub fn set_snippet_config(&self, config: SnippetConfig) {
+ self.inner
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .index
+ .set_snippet_config(config)
}
}
diff --git a/crates/core/src/live_index/index_manager.rs b/crates/core/src/live_index/index_manager.rs
index 217cd2f9..dca7a6ac 100644
--- a/crates/core/src/live_index/index_manager.rs
+++ b/crates/core/src/live_index/index_manager.rs
@@ -18,72 +18,49 @@ use std::sync::Arc;
use chrono::Utc;
-use crate::{
- config::{CrawlerConfig, LiveIndexConfig},
- feed::scheduler::Split,
-};
+use crate::config::LiveIndexConfig;
use super::{
- crawler::{CrawlResults, Crawler},
- downloaded_db::DownloadedDb,
- indexer::Indexer,
- Index, AUTO_COMMIT_INTERVAL, EVENT_LOOP_INTERVAL, PRUNE_INTERVAL,
+ LiveIndex, AUTO_COMMIT_INTERVAL, COMPACT_INTERVAL, EVENT_LOOP_INTERVAL, PRUNE_INTERVAL,
};
use crate::Result;
pub struct IndexManager {
- index: Arc<Index>,
- crawler: Crawler,
+ index: Arc<LiveIndex>,
}
impl IndexManager {
pub fn new(config: LiveIndexConfig) -> Result<Self> {
- let index = Index::new(&config.index_path)?;
- let indexer = Arc::new(Indexer::new(index.clone_inner_index(), config.clone()));
-
- let crawler_config = CrawlerConfig::from(&config);
-
- let crawler = Crawler::new(
- Split::open(&config.split_path)?,
- indexer,
- DownloadedDb::open(&config.downloaded_db_path)?,
- Arc::new(crawler_config),
- )?;
-
- Ok(Self {
- index: Arc::new(index),
- crawler,
- })
+ let index = Arc::new(LiveIndex::new(config.clone())?);
+ Ok(Self { index })
}
- pub async fn run(mut self) {
- let mut has_inserts = false;
+ pub async fn run(self) {
let mut last_commit = Utc::now();
let mut last_prune = Utc::now();
+ let mut last_compact = Utc::now();
loop {
- if let CrawlResults::HasInserts = self.crawler.check_feeds().await {
- has_inserts = true;
- }
-
if last_prune + PRUNE_INTERVAL < Utc::now() {
- self.index.prune();
-
+ self.index.prune_segments();
last_prune = Utc::now();
}
- if last_commit + AUTO_COMMIT_INTERVAL < Utc::now() && has_inserts {
+ if last_commit + AUTO_COMMIT_INTERVAL < Utc::now() && self.index.has_inserts() {
self.index.commit();
-
last_commit = Utc::now();
- has_inserts = false;
+ }
+
+ if last_compact + COMPACT_INTERVAL < Utc::now() {
+ self.index.compact_segments_by_date();
+ last_compact = Utc::now();
}
tokio::time::sleep(EVENT_LOOP_INTERVAL).await;
}
}
- pub fn index(&self) -> Arc<Index> {
+ pub fn index(&self) -> Arc<LiveIndex> {
self.index.clone()
}
}
diff --git a/crates/core/src/live_index/indexer.rs b/crates/core/src/live_index/indexer.rs
deleted file mode 100644
index 77b00e06..00000000
--- a/crates/core/src/live_index/indexer.rs
+++ /dev/null
@@ -1,96 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>
-
-use std::sync::{Arc, Mutex, RwLock};
-
-use crate::{
- config::LiveIndexConfig,
- crawler::{self, CrawlDatum, DatumStream},
- entrypoint::indexer::IndexingWorker,
- Result,
-};
-
-use super::BATCH_SIZE;
-
-pub struct Indexer {
- search_index: Arc<RwLock<crate::index::Index>>,
- worker: IndexingWorker,
- write_batch: Arc<Mutex<Vec<crate::entrypoint::indexer::IndexableWebpage>>>,
-}
-
-impl Indexer {
- pub fn new(search_index: Arc<RwLock<crate::index::Index>>, config: LiveIndexConfig) -> Self {
- Self {
- search_index,
- worker: IndexingWorker::new(config),
- write_batch: Arc::new(Mutex::new(Vec::new())),
- }
- }
-
- pub fn maybe_write_batch_to_index(&self) {
- let batch = self.write_batch.lock().unwrap_or_else(|e| e.into_inner());
-
- if batch.len() < BATCH_SIZE {
- return;
- }
-
- let prepared = self.worker.prepare_webpages(&batch);
-
- let search_index = self.search_index.write().unwrap_or_else(|e| e.into_inner());
- for webpage in &prepared {
- search_index.insert(webpage).ok();
- }
- }
-
- pub fn write_batch_to_index(&self) {
- let batch = self.write_batch.lock().unwrap_or_else(|e| e.into_inner());
-
- if batch.is_empty() {
- return;
- }
-
- let prepared = self.worker.prepare_webpages(&batch);
-
- let search_index = self.search_index.write().unwrap_or_else(|e| e.into_inner());
- for webpage in &prepared {
- search_index.insert(webpage).ok();
- }
- }
-}
-
-impl DatumStream for Indexer {
- async fn write(&self, crawl_datum: CrawlDatum) -> Result<(), crawler::Error> {
- self.write_batch
- .lock()
- .unwrap_or_else(|e| e.into_inner())
- .push(crawl_datum.into());
-
- self.maybe_write_batch_to_index();
-
- Ok(())
- }
-
- async fn finish(&self) -> Result<(), crawler::Error> {
- self.write_batch_to_index();
-
- self.search_index
- .write()
- .unwrap_or_else(|e| e.into_inner())
- .commit()?;
-
- Ok(())
- }
-}
diff --git a/crates/core/src/live_index/mod.rs b/crates/core/src/live_index/mod.rs
index 98ec42e8..221155f7 100644
--- a/crates/core/src/live_index/mod.rs
+++ b/crates/core/src/live_index/mod.rs
@@ -15,47 +15,43 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::time::Duration;
-use chrono::{DateTime, Utc};
-
use crate::{
config::{CrawlerConfig, LiveIndexConfig},
- feed::scheduler::{DomainFeeds, Split},
};
-pub use self::index::Index;
+pub use self::index::LiveIndex;
pub use self::index_manager::IndexManager;
-mod crawler;
-mod downloaded_db;
-mod index;
+// mod crawler;
+pub mod index;
mod index_manager;
-mod indexer;
const TTL: Duration = Duration::from_secs(60 * 60 * 24 * 60); // 60 days
const PRUNE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour
-const FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutes
+const COMPACT_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour
+// const FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutes
const AUTO_COMMIT_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 minutes
const EVENT_LOOP_INTERVAL: Duration = Duration::from_secs(5);
const BATCH_SIZE: usize = 512;
-#[derive(Debug, Clone)]
-struct Feeds {
- last_checked: DateTime<Utc>,
- feed: DomainFeeds,
-}
-
-impl From<Split> for Vec<Feeds> {
- fn from(split: Split) -> Self {
- split
- .feeds
- .into_iter()
- .map(|feed| Feeds {
- last_checked: Utc::now(),
- feed,
- })
- .collect()
- }
-}
+// #[derive(Debug, Clone)]
+// struct Feeds {
+// last_checked: DateTime<Utc>,
+// feed: DomainFeeds,
+// }
+//
+// impl From<Split> for Vec<Feeds> {
+// fn from(split: Split) -> Self {
+// split
+// .feeds
+// .into_iter()
+// .map(|feed| Feeds {
+// last_checked: Utc::now(),
+// feed,
+// })
+// .collect()
+// }
+// }
impl From<&LiveIndexConfig> for CrawlerConfig {
fn from(live: &LiveIndexConfig) -> Self {
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index eec25e6b..7170ab2d 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -175,9 +175,6 @@ enum AdminIndexOptions {
#[derive(Subcommand)]
enum LiveIndex {
- /// Create a schedule of which feeds should go to which index.
- Schedule { config_path: String },
-
/// Serve the live index.
Serve { config_path: String },
}
@@ -451,10 +448,6 @@ fn main() -> Result<()> {
}
},
Commands::LiveIndex { options } => match options {
- LiveIndex::Schedule { config_path } => {
- let config = load_toml_config(config_path);
- entrypoint::live_index::schedule(config)?;
- }
LiveIndex::Serve { config_path } => {
let config = load_toml_config(config_path);
diff --git a/crates/core/src/query/plan/mod.rs b/crates/core/src/query/plan/mod.rs
index ed01658f..0eca5598 100644
--- a/crates/core/src/query/plan/mod.rs
+++ b/crates/core/src/query/plan/mod.rs
@@ -217,10 +217,6 @@ fn process_tantivy_term<T: TextField>(
});
}
- if terms.is_empty() {
- dbg!(field.name());
- }
-
terms
}
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index 8d19a8a4..a8dbfd1c 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -76,7 +76,7 @@ impl<'a> SearchGuard<'a> for NormalIndexSearchGuard<'a> {
}
}
-impl SearchableIndex for Arc<live_index::Index> {
+impl SearchableIndex for Arc<live_index::LiveIndex> {
type SearchGuard<'a> = LiveIndexSearchGuard<'a>;
fn guard(&self) -> Self::SearchGuard<'_> {
@@ -86,17 +86,17 @@ impl SearchableIndex for Arc<live_index::Index> {
}
fn set_snippet_config(&mut self, config: SnippetConfig) {
- self.write().inverted_index.set_snippet_config(config);
+ live_index::LiveIndex::set_snippet_config(self, config)
}
}
pub struct LiveIndexSearchGuard<'a> {
- lock_guard: RwLockReadGuard<'a, crate::index::Index>,
+ lock_guard: RwLockReadGuard<'a, live_index::index::InnerIndex>,
}
impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {
fn search_index(&self) -> &'_ Index {
- &self.lock_guard
+ self.lock_guard.index()
}
}
diff --git a/crates/simple-wal/src/lib.rs b/crates/simple-wal/src/lib.rs
index 5d6aabb7..d23d52c3 100644
--- a/crates/simple-wal/src/lib.rs
+++ b/crates/simple-wal/src/lib.rs
@@ -58,6 +58,16 @@ where
Ok(())
}
+
+ pub fn batch_write<'a>(&'a mut self, items: impl Iterator<Item = &'a T>) -> Result<()> {
+ for item in items {
+ self.writer.write(item)?;
+ }
+
+ self.writer.flush()?;
+
+ Ok(())
+ }
}
impl<T> Wal<T>
diff --git a/crates/tantivy/src/index/index.rs b/crates/tantivy/src/index/index.rs
index 5d589878..2e55e97d 100644
--- a/crates/tantivy/src/index/index.rs
+++ b/crates/tantivy/src/index/index.rs
@@ -545,6 +545,11 @@ impl Index {
load_metas(self.directory(), &self.inventory)
}
+ /// Saves the index meta file to the directory (Advanced user only)
+ pub fn save_metas(&self, metas: &IndexMeta) -> crate::Result<()> {
+ save_metas(metas, self.directory())
+ }
+
/// Open a new index writer. Attempts to acquire a lockfile.
///
/// The lockfile should be deleted on drop, but it is possible
diff --git a/crates/tantivy/src/indexer/segment_updater.rs b/crates/tantivy/src/indexer/segment_updater.rs
index a34ba8d0..88793188 100644
--- a/crates/tantivy/src/indexer/segment_updater.rs
+++ b/crates/tantivy/src/indexer/segment_updater.rs
@@ -31,8 +31,6 @@ const NUM_MERGE_THREADS: usize = 4;
/// and the `meta.json` remains untouched,
/// - it success, and `meta.json` is written
/// and flushed.
-///
-/// This method is not part of tantivy's public API
pub(crate) fn save_metas(metas: &IndexMeta, directory: &dyn Directory) -> crate::Result<()> {
info!("save metas");
let mut buffer = serde_json::to_vec_pretty(metas)?;