Skip to content

web search done right

12.0 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/

use std::{
    io::{BufReader, BufWriter},
    path::{Path, PathBuf},
    sync::Arc,
};

use tokio::sync::RwLock;

use chrono::{DateTime, NaiveDate, Utc};
use itertools::Itertools;
use simple_wal::Wal;
use tantivy::{
    index::SegmentId,
    indexer::{MergeOperation, SegmentEntry},
};

use std::collections::{HashMap, HashSet};

use crate::{
    config::SnippetConfig,
    entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},
    inverted_index::{InvertedIndex, ShardId},
    live_index::{BATCH_SIZE, TTL},
    Result,
};

#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
pub struct Segment {
    id: SegmentId,
    created: DateTime<Utc>,
}

impl Segment {
    pub fn id(&self) -> SegmentId {
        self.id
    }

    pub fn created(&self) -> DateTime<Utc> {
        self.created
    }
}

pub struct CompactOperation {
    segments: Vec<Segment>,
    entry: Option<SegmentEntry>,
    merge_op: MergeOperation,
}

impl CompactOperation {
    pub fn end(self, index: &mut InvertedIndex) -> Result<Option<SegmentId>> {
        let res = index.end_merge_segments_by_id(self.merge_op, self.entry)?;
        Ok(res)
    }
}

#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]
pub struct Meta {
    segments: Vec<Segment>,
}

impl Meta {
    pub fn open_or_create<P: AsRef<Path>>(path: P) -> Self {
        if !path.as_ref().exists() {
            let meta = Meta::default();
            meta.save(path);

            meta
        } else {
            let file = std::fs::File::open(path).unwrap();
            let reader = BufReader::new(file);
            serde_json::from_reader(reader).unwrap()
        }
    }

    pub fn save<P: AsRef<Path>>(&self, path: P) {
        let file = std::fs::OpenOptions::new()
            .create(true)
            .write(true)
            .read(true)
            .truncate(true)
            .open(path)
            .unwrap();
        let writer = BufWriter::new(file);

        serde_json::to_writer_pretty(writer, self).unwrap();
    }

    pub fn segments(&self) -> &[Segment] {
        &self.segments
    }
}

pub struct InnerIndex {
    index: Arc<RwLock<crate::index::Index>>,
    write_ahead_log: Wal<crate::entrypoint::indexer::IndexableWebpage>,
    has_inserts: bool,
    indexing_worker: IndexingWorker,
    path: PathBuf,
    meta: Meta,
}

impl InnerIndex {
    pub async fn new<P: AsRef<Path>>(
        path: P,
        shard_id: ShardId,
        indexer_worker_config: indexer::worker::Config,
    ) -> Result<Self> {
        let mut index = crate::index::Index::open(path.as_ref())?;
        index.prepare_writer()?;
        index.set_shard_id(shard_id);

        let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;
        let wal_count = write_ahead_log.iter()?.count();

        let worker = IndexingWorker::new(indexer_worker_config).await;

        let meta = Meta::open_or_create(path.as_ref().join("meta.json"));

        Ok(Self {
            index: Arc::new(RwLock::new(index)),
            write_ahead_log,
            indexing_worker: worker,
            has_inserts: wal_count > 0,
            meta,
            path: path.as_ref().to_path_buf(),
        })
    }

    pub async fn prune_segments(&mut self) {
        let old_segments: Vec<_> = self
            .meta
            .segments
            .iter()
            .filter_map(|segment| {
                if segment.created + TTL < Utc::now() {
                    Some(segment.id)
                } else {
                    None
                }
            })
            .collect();

        self.index
            .write()
            .await
            .inverted_index
            .delete_segments_by_id(&old_segments)
            .unwrap();

        self.sync_meta_with_index().await;
        self.re_open().await;
    }

    pub async fn start_compact_segments_by_date(&self) -> Result<Vec<CompactOperation>> {
        let segments_to_compact = self.prepare_segments_for_compaction();

        let mut operations = Vec::new();

        for (_, segments) in segments_to_compact {
            if segments.len() <= 1 {
                continue;
            }

            let segment_ids: Vec<SegmentId> = segments.iter().map(|s| s.id).collect();

            let (entry, merge_op) = self
                .index
                .write()
                .await
                .inverted_index
                .start_merge_segments_by_id(&segment_ids)
                .await?;

            operations.push(CompactOperation {
                segments,
                entry,
                merge_op,
            });
        }

        Ok(operations)
    }

    async fn end_compact_segments_by_date(
        &mut self,
        operations: Vec<CompactOperation>,
    ) -> Result<()> {
        for op in operations {
            let newest_creation_date = op.segments.iter().map(|s| s.created).max().unwrap();
            let segment_ids: Vec<SegmentId> = op.segments.iter().map(|s| s.id).collect();

            let mut index = self.index.write().await;
            if let Ok(Some(new_segment_id)) = op.end(&mut index.inverted_index) {
                drop(index);

                self.update_meta_after_compaction(
                    segment_ids,
                    new_segment_id,
                    newest_creation_date,
                );
            }
        }

        self.save_meta();
        self.re_open().await;

        Ok(())
    }

    fn prepare_segments_for_compaction(&self) -> HashMap<NaiveDate, Vec<Segment>> {
        let mut segments_by_date: HashMap<NaiveDate, Vec<Segment>> = HashMap::new();

        for segment in self.meta.segments.clone() {
            segments_by_date
                .entry(segment.created.date_naive())
                .or_default()
                .push(segment);
        }

        segments_by_date
    }

    fn update_meta_after_compaction(
        &mut self,
        old_segment_ids: Vec<SegmentId>,
        new_segment_id: SegmentId,
        newest_creation_date: DateTime<Utc>,
    ) {
        self.meta
            .segments
            .retain(|s| !old_segment_ids.contains(&s.id));
        self.meta.segments.push(Segment {
            id: new_segment_id,
            created: newest_creation_date,
        });
    }

    async fn re_open(&mut self) {
        let mut index = self.index.write().await;
        let shard_id = index.shard_id();
        index.inverted_index.re_open().unwrap();
        index.prepare_writer().unwrap();

        if let Some(shard_id) = shard_id {
            index.set_shard_id(shard_id);
        }
    }

    async fn sync_meta_with_index(&mut self) {
        let segments_in_index: HashSet<_> = self
            .index
            .write()
            .await
            .inverted_index
            .segment_ids()
            .into_iter()
            .collect();

        let segments_in_meta: HashSet<_> = self
            .meta
            .segments
            .clone()
            .into_iter()
            .map(|segment| segment.id)
            .collect();

        // remove all segments from meta that is not present in the index
        let to_remove: HashSet<_> = segments_in_meta
            .iter()
            .filter(|segment| !segments_in_index.contains(segment))
            .collect();

        self.meta.segments = self
            .meta
            .segments
            .clone()
            .into_iter()
            .filter(|segment| !to_remove.contains(&segment.id))
            .collect();

        // insert all segments from index that is not already in meta
        for id in segments_in_index
            .into_iter()
            .filter(|segment| !segments_in_meta.contains(segment))
        {
            self.meta.segments.push(Segment {
                id,
                created: Utc::now(),
            })
        }

        self.save_meta();
    }

    fn save_meta(&self) {
        self.meta.save(self.path.join("meta.json"));
    }

    pub async fn delete_all_pages(&mut self) {
        let mut index = self.index.write().await;
        let segments = index.inverted_index.segment_ids();
        index
            .inverted_index
            .delete_segments_by_id(&segments)
            .unwrap();
        drop(index);

        self.meta = Meta::default();
        self.save_meta();
        self.re_open().await;
    }

    pub fn insert(&mut self, pages: &[IndexableWebpage]) {
        self.write_ahead_log.batch_write(pages.iter()).unwrap();
        self.has_inserts = true;
    }

    pub async fn commit(&mut self) {
        let mut index = self.index.write().await;
        for batch in self
            .write_ahead_log
            .iter()
            .unwrap()
            .unique_by(|page| page.url.clone())
            .chunks(BATCH_SIZE)
            .into_iter()
        {
            let batch: Vec<_> = batch.collect();
            for webpage in self.indexing_worker.prepare_webpages(&batch).await {
                index.insert(&webpage).unwrap();
            }
        }
        index.commit().unwrap();
        drop(index);

        self.write_ahead_log.clear().unwrap();
        self.sync_meta_with_index().await;
        self.has_inserts = false;
        self.re_open().await;
    }

    pub fn has_inserts(&self) -> bool {
        self.has_inserts
    }
}

pub struct LiveIndex {
    inner: Arc<RwLock<InnerIndex>>,
}

impl LiveIndex {
    pub async fn new<P: AsRef<Path>>(
        path: P,
        shard_id: u64,
        indexer_worker_config: indexer::worker::Config,
    ) -> Result<Self> {
        Ok(Self {
            inner: Arc::new(RwLock::new(
                InnerIndex::new(path, ShardId::Live(shard_id), indexer_worker_config).await?,
            )),
        })
    }

    pub async fn commit(&self) {
        tracing::debug!("committing index");
        self.inner.write().await.commit().await;
    }

    pub async fn prune_segments(&self) {
        tracing::debug!("pruning segments");
        self.inner.write().await.prune_segments().await
    }

    pub async fn has_inserts(&self) -> bool {
        self.inner.read().await.has_inserts()
    }

    pub async fn compact_segments_by_date(&self) -> Result<()> {
        tracing::debug!("compacting segments by date");
        let operations = self
            .inner
            .read()
            .await
            .start_compact_segments_by_date()
            .await?;

        self.inner
            .write()
            .await
            .end_compact_segments_by_date(operations)
            .await?;

        Ok(())
    }

    pub async fn index(&self) -> Arc<RwLock<crate::index::Index>> {
        self.inner.read().await.index.clone()
    }

    pub async fn insert(&self, pages: &[IndexableWebpage]) {
        tracing::debug!("inserting {} pages into index", pages.len());
        self.inner.write().await.insert(pages)
    }

    pub async fn set_snippet_config(&self, config: SnippetConfig) {
        self.inner
            .write()
            .await
            .index
            .write()
            .await
            .inverted_index
            .set_snippet_config(config)
    }

    pub async fn path(&self) -> PathBuf {
        self.inner.read().await.path.to_path_buf()
    }

    pub async fn delete_all_pages(&self) {
        self.inner.write().await.delete_all_pages().await
    }

    pub async fn re_open(&self) -> Result<()> {
        self.inner.write().await.re_open().await;

        Ok(())
    }

    pub async fn meta(&self) -> Meta {
        self.inner.read().await.meta.clone()
    }
}