Skip to content

web search done right

9.8 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use std::{collections::HashMap, sync::Arc};

use tantivy::{columnar::ColumnValues, index::SegmentId, DocId};

use crate::{
    enum_map::EnumMap,
    schema::{
        numerical_field::{NumericalField, Orientation},
        DataType, Field, NumericalFieldEnum,
    },
};

#[derive(Default)]
struct InnerNumericalFieldReader {
    segments: HashMap<SegmentId, SegmentReader>,
}

#[derive(Default, Clone)]
pub struct NumericalFieldReader {
    inner: Arc<InnerNumericalFieldReader>,
}

impl NumericalFieldReader {
    pub fn borrow_segment(&self, segment: &SegmentId) -> &SegmentReader {
        self.inner.segments.get(segment).as_ref().unwrap()
    }
}

impl NumericalFieldReader {
    pub fn new(tv_searcher: &tantivy::Searcher) -> Self {
        let mut segments = HashMap::new();

        for reader in tv_searcher.segment_readers() {
            let mut field_ids = EnumMap::new();
            let columnfield_readers = reader.column_fields();

            let mut u64s = EnumMap::new();
            let mut u128s = EnumMap::new();
            let mut bytes = EnumMap::new();
            let mut bools = EnumMap::new();
            let mut f64s = EnumMap::new();

            for field in Field::all().filter_map(|f| f.as_numerical()) {
                match field.data_type() {
                    DataType::U64 => {
                        if let Ok(reader) = columnfield_readers.u64(field.name()) {
                            u64s.insert(field, reader.values);
                        }
                    }
                    DataType::U128 => {
                        if let Ok(reader) = columnfield_readers.u128(field.name()) {
                            u128s.insert(field, reader.values);
                        }
                    }
                    DataType::F64 => {
                        if let Ok(reader) = columnfield_readers.f64(field.name()) {
                            f64s.insert(field, reader.values);
                        }
                    }
                    DataType::Bool => {
                        if let Ok(reader) = columnfield_readers.bool(field.name()) {
                            bools.insert(field, reader.values);
                        }
                    }
                    DataType::Bytes => {
                        if let Ok(Some(reader)) = columnfield_readers.bytes(field.name()) {
                            bytes.insert(field, reader);
                        }
                    }
                };

                field_ids.insert(
                    field,
                    reader.schema().get_field(field.name()).unwrap().field_id(),
                );
            }

            segments.insert(
                reader.segment_id(),
                SegmentReader {
                    row_reader: reader.row_fields().clone(),
                    columnar_readers: ColumnarReaders {
                        u64s,
                        u128s,
                        bytes,
                        bools,
                        f64s,
                    },
                    field_ids,
                    row: None,
                },
            );
        }

        Self {
            inner: Arc::new(InnerNumericalFieldReader { segments }),
        }
    }
}

#[derive(Clone)]
struct ColumnarReaders {
    u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,
    u128s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u128>>>,
    f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,
    bools: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<bool>>>,
    bytes: EnumMap<NumericalFieldEnum, tantivy::columnar::BytesColumn>,
}

pub enum Value {
    U64(u64),
    U128(u128),
    F64(f64),
    Bytes(Vec<u8>),
    Bool(bool),
}

impl Value {
    pub fn as_u64(&self) -> Option<u64> {
        match self {
            Value::U64(val) => Some(*val),
            _ => None,
        }
    }

    pub fn as_u128(&self) -> Option<u128> {
        match self {
            Value::U128(val) => Some(*val),
            _ => None,
        }
    }

    pub fn as_f64(&self) -> Option<f64> {
        match self {
            Value::F64(val) => Some(*val),
            _ => None,
        }
    }

    pub fn as_bytes(&self) -> Option<&[u8]> {
        match self {
            Value::Bytes(val) => Some(val),
            _ => None,
        }
    }

    pub fn as_bool(&self) -> Option<bool> {
        match self {
            Value::Bool(val) => Some(*val),
            _ => None,
        }
    }
}

impl From<u64> for Value {
    fn from(val: u64) -> Self {
        Value::U64(val)
    }
}

impl From<u128> for Value {
    fn from(val: u128) -> Self {
        Value::U128(val)
    }
}

impl From<f64> for Value {
    fn from(val: f64) -> Self {
        Value::F64(val)
    }
}

impl From<Vec<u8>> for Value {
    fn from(val: Vec<u8>) -> Self {
        Value::Bytes(val)
    }
}

impl From<bool> for Value {
    fn from(val: bool) -> Self {
        Value::Bool(val)
    }
}

impl From<Value> for Option<u64> {
    fn from(val: Value) -> Self {
        val.as_u64()
    }
}

impl<'a> From<&'a Value> for Option<&'a [u8]> {
    fn from(val: &'a Value) -> Self {
        val.as_bytes()
    }
}

impl From<Value> for Option<Vec<u8>> {
    fn from(val: Value) -> Self {
        match val {
            Value::Bytes(val) => Some(val),
            _ => None,
        }
    }
}

impl From<Value> for Option<bool> {
    fn from(val: Value) -> Self {
        val.as_bool()
    }
}

pub struct FieldReader<'a> {
    columnar_readers: &'a ColumnarReaders,
    row: Option<&'a tantivy::roworder::Row>,
    field_ids: &'a EnumMap<NumericalFieldEnum, u32>,
    doc: DocId,
}

impl FieldReader<'_> {
    pub fn get(&self, field: NumericalFieldEnum) -> Option<Value> {
        if field.orientation().contains(Orientation::ROW) {
            let field_id = self.field_ids.get(field)?;

            self.row.as_ref().and_then(|row| match field.data_type() {
                DataType::U64 => row.get_u64(field_id).map(Value::U64),
                DataType::F64 => row.get_f64(field_id).map(Value::F64),
                DataType::Bool => row.get_bool(field_id).map(Value::Bool),
                DataType::U128 => row.get_u128(field_id).map(Value::U128),
                DataType::Bytes => unimplemented!("bytes fields cannot be row oriented"),
            })
        } else if field.orientation().contains(Orientation::COLUMNAR) {
            match field.data_type() {
                DataType::U64 => Some(
                    self.columnar_readers
                        .u64s
                        .get(field)?
                        .get_val(self.doc)
                        .into(),
                ),
                DataType::U128 => Some(
                    self.columnar_readers
                        .u128s
                        .get(field)?
                        .get_val(self.doc)
                        .into(),
                ),
                DataType::F64 => Some(
                    self.columnar_readers
                        .f64s
                        .get(field)?
                        .get_val(self.doc)
                        .into(),
                ),

                DataType::Bool => Some(
                    self.columnar_readers
                        .bools
                        .get(field)?
                        .get_val(self.doc)
                        .into(),
                ),

                DataType::Bytes => {
                    let reader = self.columnar_readers.bytes.get(field)?;
                    let ord = reader.ords().values.get_val(self.doc);

                    if ord > reader.num_terms() as u64 || reader.num_terms() == 0 {
                        return None;
                    }

                    let mut bytes = Vec::new();
                    reader.ord_to_bytes(ord, &mut bytes).unwrap();

                    if bytes.is_empty() {
                        None
                    } else {
                        Some(bytes.into())
                    }
                }
            }
        } else {
            None
        }
    }
}

struct Row {
    doc: DocId,
    row: tantivy::roworder::Row,
}

pub struct SegmentReader {
    columnar_readers: ColumnarReaders,
    row_reader: tantivy::roworder::readers::RowFieldReaders,
    field_ids: EnumMap<NumericalFieldEnum, u32>,
    row: Option<Row>,
}

impl Clone for SegmentReader {
    fn clone(&self) -> Self {
        Self {
            columnar_readers: self.columnar_readers.clone(),
            row_reader: self.row_reader.clone(),
            field_ids: self.field_ids.clone(),
            row: None,
        }
    }
}

impl SegmentReader {
    pub fn prepare_row_for_doc(&mut self, doc: DocId) {
        self.row = Some(Row {
            doc,
            row: self.row_reader.row_index().get_row(doc as usize).unwrap(),
        });
    }

    pub fn get_field_reader(&self, doc: DocId) -> FieldReader<'_> {
        debug_assert!(
            self.row.as_ref().map(|r| r.doc == doc).unwrap_or(true),
            "Row prepared for wrong doc"
        );

        FieldReader {
            columnar_readers: &self.columnar_readers,
            field_ids: &self.field_ids,
            row: self.row.as_ref().map(|r| &r.row),
            doc,
        }
    }
}