Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

23.5 KB
Raw Blame
//! The whole pipeline in one call: [`run`] compiles a statement, drives an
//! [`Engine`] through a [`RowSource`] for its reads and an [`OpsSink`] for its
//! writes, and answers with an [`Outcome`].
//!
//! The source and sink are the only I/O; everything else is pure. Paging
//! and the row cap live in the engine rather than in the source so a fake
//! source can prove them.

#[cfg(test)]
mod test;

use std::future::Future;

use maybe_send::MaybeSend;
use serde::{Deserialize, Serialize};
use specta::Type;
use uuid::Uuid;

use models_databases::views::ViewProblem;
use models_databases::{
    ColumnResult, DatabaseId, DatabaseOp, OpResult, RowId, RowsResult, TableId, TableResult,
    ViewResult,
};

use crate::catalog::{Catalog, ColumnKind};
use crate::engine::{Engine, Step};
use crate::fold::{Bin, Row, Table};
use crate::parse::ParseError;
use crate::resolve::{
    AggregateFunction, Binding, CompileError, Relation, ResolveError, SelectItem, SelectQuery,
    binding, row_id_key,
};
use crate::split::{GqlQuery, column_of, virtual_column_of};

/// The most rows one statement reads before the fold. Past it the answer
/// is still returned, marked truncated, so aggregates are visibly partial
/// rather than silently wrong.
pub const ROW_CAP: usize = 20_000;

/// The most rows asked for in one page.
pub const PAGE_LIMIT: usize = 500;

/// One page of rows from the server.
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, Type)]
#[serde(rename_all = "camelCase")]
pub struct Page {
    /// The rows, carrying at least the columns asked for.
    pub rows: Vec<Row>,
    /// The cursor for the next page, if there is one.
    pub next: Option<String>,
}

/// Where rows come from: the Soup GraphQL API on the server, the normalized
/// cache in the browser.
pub trait RowSource {
    /// Why the source could not answer.
    type Error: std::error::Error + MaybeSend + 'static;

    /// One page of the query, from `cursor` (the start when `None`), at most
    /// `limit` rows. `needs` names the columns the rows must carry.
    fn page(
        &self,
        query: &GqlQuery,
        needs: &[Uuid],
        cursor: Option<String>,
        limit: usize,
    ) -> impl Future<Output = Result<Page, Self::Error>> + MaybeSend;

    /// The bins of a `GqlQuery::GroupSoup`.
    fn bins(
        &self,
        query: &GqlQuery,
    ) -> impl Future<Output = Result<Vec<Bin>, Self::Error>> + MaybeSend;
}

/// Where writes go: a statement's ops, applied together to one database.
pub trait OpsSink {
    /// Why a write did not land.
    type Error: std::error::Error + MaybeSend + 'static;

    /// Apply `ops` to `database`; one result per op, in order.
    fn apply(
        &self,
        database: DatabaseId,
        ops: Vec<DatabaseOp>,
    ) -> impl Future<Output = Result<Vec<OpResult>, Self::Error>> + MaybeSend;
}

/// Why [`run`] did not produce an outcome: the engine refused the
/// statement, or its source or sink failed.
#[derive(Debug, PartialEq, thiserror::Error)]
pub enum RunFailure<SourceFailure, WriteFailure> {
    /// The engine refused the statement or what it was fed.
    #[error(transparent)]
    Engine(#[from] RunError),
    /// The source could not answer a read.
    #[error("could not read rows")]
    Source(#[source] SourceFailure),
    /// The sink did not apply the statement's ops.
    #[error(transparent)]
    Write(WriteFailure),
}

/// Why a statement did not run, as one typed union: each failure is a
/// value the browser reads by its `stage` (and, for resolution, `kind`), and
/// its `Display` text is what an agent reads.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error, Serialize, Type)]
#[serde(
    tag = "stage",
    rename_all = "camelCase",
    rename_all_fields = "camelCase"
)]
pub enum RunError {
    /// The text was not a statement of the supported grammar.
    #[error(transparent)]
    Parse(ParseError),
    /// The statement named something the catalog does not have, or used a
    /// column in a way its type does not allow.
    #[error(transparent)]
    Resolve(ResolveError),
    /// A view does not fit the table it shows.
    #[error(transparent)]
    View(ViewProblem),
    /// An `UPDATE` or `DELETE` named a row by id that the table does not
    /// have.
    #[error("row {position}: no row {row} in this table")]
    NoSuchRow {
        /// Where the row is named in the statement's list of ids, from 1.
        #[specta(type = u32)]
        position: usize,
        /// The row.
        row: RowId,
    },
    /// An `UPDATE` or `DELETE` matched more rows than a statement reads.
    #[error("the WHERE matches more than {limit} rows; narrow it and run the statement again")]
    TooManyRows {
        /// The most rows a statement reads.
        #[specta(type = u32)]
        limit: usize,
    },
    /// A feed answered the outstanding request with the wrong kind of
    /// answer.
    #[error("request {request} wants {expected}, but {fed} was fed")]
    WrongAnswer {
        /// The outstanding request.
        request: u32,
        /// What it asked for.
        expected: Answer,
        /// What was fed.
        fed: Answer,
    },
    /// Bins were folded for a statement that counting groups does not
    /// answer.
    #[error("the statement is not answered by counting groups")]
    NotAnsweredByBins,
    /// The sink answered a different number of results than ops sent.
    #[error("one op was sent, but {received} results came back")]
    OpResultCount {
        /// The results that came back.
        #[specta(type = u32)]
        received: usize,
    },
    /// The sink answered an op with a result of another kind.
    #[error("{sent} was sent, but {received} came back")]
    UnexpectedOpResult {
        /// The op sent.
        sent: SentOp,
        /// The result that came back.
        received: OpResultKind,
    },
    /// A feed quoted a request the engine is not waiting on.
    #[error("fed request {fed}, but request {expected} is outstanding")]
    WrongRequest {
        /// The outstanding request.
        expected: u32,
        /// The id fed.
        fed: u32,
    },
    /// A feed arrived when nothing was outstanding.
    #[error("fed request {fed}, but nothing is outstanding")]
    NothingOutstanding {
        /// The id fed.
        fed: u32,
    },
    /// A value handed to the engine is not the shape it reads.
    #[error("{what} is not readable: {message}")]
    Unreadable {
        /// What was handed in: the catalog, a page, the bins, ….
        what: Input,
        /// Why it could not be read.
        message: String,
    },
    /// A value the engine hands back could not be written out.
    #[error("the engine's answer could not be written out: {message}")]
    Unwritable {
        /// Why it could not be written.
        message: String,
    },
    /// The first step was asked for twice.
    #[error("the query has already started")]
    AlreadyStarted,
    /// The statement names a table its catalog does not have: it was
    /// resolved against another catalog.
    #[error("table {table} is not in the catalog the statement was resolved against")]
    TableNotInCatalog {
        /// The table.
        table: TableId,
    },
    /// The statement names a column its table does not have: it was
    /// resolved against another catalog.
    #[error(
        "table {table} has no column {column}; the statement was resolved against another catalog"
    )]
    ColumnNotInTable {
        /// The table.
        table: TableId,
        /// The column's definition.
        column: Uuid,
    },
    /// A select item names a key nothing in the statement binds.
    #[error("select item {key} is bound to no column of the statement")]
    UnboundColumn {
        /// The key.
        key: Uuid,
    },
    /// A value for a column is not of the column's type.
    #[error("a value written to \"{column}\" is not of that column's type")]
    MismatchedValue {
        /// The column's name.
        column: String,
    },
    /// The engine broke one of its own invariants: a bug in the engine, not
    /// a problem with the statement.
    #[error(
        "the query engine failed internally ({invariant}); this is a bug, not a problem with the statement"
    )]
    Internal {
        /// The invariant broken.
        invariant: &'static str,
    },
}

/// A failure as it crosses the wasm boundary: the typed error, and the words
/// the engine would give an agent for it.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Type)]
pub struct EngineError {
    /// What went wrong.
    pub error: RunError,
    /// The error in words.
    pub message: String,
}

impl From<RunError> for EngineError {
    fn from(error: RunError) -> Self {
        Self {
            message: error.to_string(),
            error,
        }
    }
}

/// What a request is answered with.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Type, strum::Display)]
#[serde(rename_all = "camelCase")]
pub enum Answer {
    /// A page of rows, through [`Engine::feed_page`].
    #[strum(serialize = "a page")]
    Page,
    /// The bins of a grouped read, through [`Engine::feed_bins`].
    #[strum(serialize = "bins")]
    Bins,
    /// The results of ops, through [`Engine::feed_ops`].
    #[strum(serialize = "op results")]
    OpResults,
}

/// The op a statement sent.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Type, strum::Display)]
#[serde(rename_all = "camelCase")]
pub enum SentOp {
    /// Rows inserted, updated or deleted.
    #[strum(serialize = "a row write")]
    RowWrite,
    /// A column's type change.
    #[strum(serialize = "a column type change")]
    ColumnTypeChange,
}

/// The kind of an [`OpResult`], with what happened to its resource.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Type, strum::Display)]
#[serde(rename_all = "camelCase")]
pub enum OpResultKind {
    /// [`TableResult::Created`] of an [`OpResult::Table`].
    #[strum(serialize = "a table created")]
    TableCreated,
    /// [`TableResult::Renamed`] of an [`OpResult::Table`].
    #[strum(serialize = "a table renamed")]
    TableRenamed,
    /// [`TableResult::Deleted`] of an [`OpResult::Table`].
    #[strum(serialize = "a table deleted")]
    TableDeleted,
    /// [`TableResult::ColumnsReordered`] of an [`OpResult::Table`].
    #[strum(serialize = "columns reordered")]
    ColumnsReordered,
    /// [`TableResult::ViewsReordered`] of an [`OpResult::Table`].
    #[strum(serialize = "views reordered")]
    ViewsReordered,
    /// [`ColumnResult::Created`] of an [`OpResult::Column`].
    #[strum(serialize = "a column created")]
    ColumnCreated,
    /// [`ColumnResult::Renamed`] of an [`OpResult::Column`].
    #[strum(serialize = "a column renamed")]
    ColumnRenamed,
    /// [`ColumnResult::TypeChanged`] of an [`OpResult::Column`].
    #[strum(serialize = "a column type changed")]
    ColumnTypeChanged,
    /// [`ColumnResult::Deleted`] of an [`OpResult::Column`].
    #[strum(serialize = "a column deleted")]
    ColumnDeleted,
    /// [`ColumnResult::OptionsAdded`] of an [`OpResult::Column`].
    #[strum(serialize = "options added")]
    OptionsAdded,
    /// [`ColumnResult::OptionUpdated`] of an [`OpResult::Column`].
    #[strum(serialize = "an option updated")]
    OptionUpdated,
    /// [`ColumnResult::OptionDeleted`] of an [`OpResult::Column`].
    #[strum(serialize = "an option deleted")]
    OptionDeleted,
    /// [`RowsResult::Inserted`] of an [`OpResult::Rows`].
    #[strum(serialize = "rows inserted")]
    RowsInserted,
    /// [`RowsResult::Updated`] of an [`OpResult::Rows`].
    #[strum(serialize = "rows updated")]
    RowsUpdated,
    /// [`RowsResult::Deleted`] of an [`OpResult::Rows`].
    #[strum(serialize = "rows deleted")]
    RowsDeleted,
    /// [`ViewResult::Created`] of an [`OpResult::View`].
    #[strum(serialize = "a view created")]
    ViewCreated,
    /// [`ViewResult::Updated`] of an [`OpResult::View`].
    #[strum(serialize = "a view updated")]
    ViewUpdated,
    /// [`ViewResult::Deleted`] of an [`OpResult::View`].
    #[strum(serialize = "a view deleted")]
    ViewDeleted,
    /// [`ViewResult::CardMoved`] of an [`OpResult::View`].
    #[strum(serialize = "a card moved")]
    CardMoved,
    /// [`OpResult::ReorderTables`].
    #[strum(serialize = "tables reordered")]
    TablesReordered,
    /// [`ColumnResult::FormulaSet`] of an [`OpResult::Column`].
    #[strum(serialize = "a formula set")]
    FormulaSet,
}

impl From<&OpResult> for OpResultKind {
    fn from(result: &OpResult) -> Self {
        match result {
            OpResult::Table {
                change: TableResult::Created,
                ..
            } => OpResultKind::TableCreated,
            OpResult::Table {
                change: TableResult::Renamed,
                ..
            } => OpResultKind::TableRenamed,
            OpResult::Table {
                change: TableResult::Deleted,
                ..
            } => OpResultKind::TableDeleted,
            OpResult::Table {
                change: TableResult::ColumnsReordered,
                ..
            } => OpResultKind::ColumnsReordered,
            OpResult::Table {
                change: TableResult::ViewsReordered { .. },
                ..
            } => OpResultKind::ViewsReordered,
            OpResult::Column {
                change: ColumnResult::Created,
                ..
            } => OpResultKind::ColumnCreated,
            OpResult::Column {
                change: ColumnResult::Renamed,
                ..
            } => OpResultKind::ColumnRenamed,
            OpResult::Column {
                change: ColumnResult::TypeChanged,
                ..
            } => OpResultKind::ColumnTypeChanged,
            OpResult::Column {
                change: ColumnResult::Deleted,
                ..
            } => OpResultKind::ColumnDeleted,
            OpResult::Column {
                change: ColumnResult::FormulaSet,
                ..
            } => OpResultKind::FormulaSet,
            OpResult::Column {
                change: ColumnResult::OptionsAdded { .. },
                ..
            } => OpResultKind::OptionsAdded,
            OpResult::Column {
                change: ColumnResult::OptionUpdated,
                ..
            } => OpResultKind::OptionUpdated,
            OpResult::Column {
                change: ColumnResult::OptionDeleted,
                ..
            } => OpResultKind::OptionDeleted,
            OpResult::Rows {
                change: RowsResult::Inserted { .. },
                ..
            } => OpResultKind::RowsInserted,
            OpResult::Rows {
                change: RowsResult::Updated { .. },
                ..
            } => OpResultKind::RowsUpdated,
            OpResult::Rows {
                change: RowsResult::Deleted { .. },
                ..
            } => OpResultKind::RowsDeleted,
            OpResult::View {
                change: ViewResult::Created { .. },
                ..
            } => OpResultKind::ViewCreated,
            OpResult::View {
                change: ViewResult::Updated { .. },
                ..
            } => OpResultKind::ViewUpdated,
            OpResult::View {
                change: ViewResult::Deleted,
                ..
            } => OpResultKind::ViewDeleted,
            OpResult::View {
                change: ViewResult::CardMoved { .. },
                ..
            } => OpResultKind::CardMoved,
            OpResult::ReorderTables { .. } => OpResultKind::TablesReordered,
        }
    }
}

/// A value a driver hands the engine.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Type, strum::Display)]
#[serde(rename_all = "camelCase")]
#[strum(serialize_all = "lowercase")]
pub enum Input {
    /// The catalog a statement compiles against.
    Catalog,
    /// The schema a catalog is built from.
    Schema,
    /// The database a catalog is scoped to.
    Scope,
    /// A page of rows.
    Page,
    /// The bins of a grouped read.
    Bins,
    /// A view.
    View,
    /// What a view's read produced.
    Outcome,
    /// The stored positions of a board's cards.
    Positions,
    /// A derived column's formula, or the table or column it is for.
    Formula,
}

impl From<CompileError> for RunError {
    fn from(error: CompileError) -> Self {
        match error {
            CompileError::Parse(error) => RunError::Parse(error),
            CompileError::Resolve(error) => RunError::Resolve(error),
        }
    }
}

impl From<ViewProblem> for RunError {
    fn from(problem: ViewProblem) -> Self {
        RunError::View(problem)
    }
}

/// What a statement produced.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Type, Default)]
#[serde(rename_all = "camelCase")]
pub struct Outcome {
    /// The result columns, in select-list order; empty for writes.
    pub columns: Vec<OutcomeColumn>,
    /// The result rows.
    pub rows: Table,
    /// For a row-shaped result, the row entity id behind each result row.
    pub row_ids: Vec<RowId>,
    /// The tables read, so a caller can watch them for changes.
    pub read_tables: Vec<TableId>,
    /// Whether the read hit [`ROW_CAP`], making aggregates partial.
    pub truncated: bool,
    /// Rows an `INSERT` created, in statement order for the rows that landed.
    pub inserted_row_ids: Vec<RowId>,
    /// Rows a write changed.
    pub changes_applied: u32,
    /// The column an `ALTER COLUMN` changed.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub altered_column: Option<AlteredColumn>,
}

/// A column whose type an `ALTER COLUMN` changed.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)]
#[serde(rename_all = "camelCase")]
pub struct AlteredColumn {
    /// The table.
    pub table: TableId,
    /// The column's property definition before the change.
    pub column: Uuid,
    /// The type it became, as SQL spells it.
    pub to: String,
}

/// One result column.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Type)]
#[serde(rename_all = "camelCase")]
pub struct OutcomeColumn {
    /// The name as the statement would call it: the column's display name,
    /// or `SUM(amount)`.
    pub name: String,
    /// The column behind the values, when there is one.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub column: Option<Uuid>,
    /// What the values are.
    pub kind: OutcomeKind,
    /// For `row_id`, the table whose rows its cells are.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub table: Option<TableId>,
}

/// The value kind of a result column.
#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))]
#[cfg_attr(feature = "openapi", schema(as = SqlOutcomeKind))]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub enum OutcomeKind {
    /// Text or link.
    Text,
    /// A number, including every aggregate but `MIN`/`MAX` of dates.
    Number,
    /// A checkbox.
    Boolean,
    /// A date-time.
    Date,
    /// A span from one date-time to another.
    DateRange,
    /// Select option ids; the caller labels them from the catalog.
    Select,
    /// Entity ids; the caller hydrates them.
    Entity,
    /// The `row_id` column: the ids of the table's own rows.
    Row,
}

/// Compile and execute one statement.
pub async fn run<Source: RowSource, Sink: OpsSink>(
    catalog: &Catalog,
    sql: &str,
    source: &Source,
    sink: &Sink,
) -> Result<Outcome, RunFailure<Source::Error, Sink::Error>> {
    let (mut engine, mut step) = Engine::start(catalog, sql)?;
    loop {
        step = match step {
            Step::Done(outcome) => return Ok(outcome),
            Step::Fetch(request) => {
                let page = source
                    .page(
                        &request.query,
                        &request.needs,
                        request.cursor,
                        request.limit,
                    )
                    .await
                    .map_err(RunFailure::Source)?;
                engine.feed_page(request.id, page)?
            }
            Step::Bins(request) => {
                let bins = source
                    .bins(&request.query)
                    .await
                    .map_err(RunFailure::Source)?;
                engine.feed_bins(request.id, bins)?
            }
            Step::Ops { id, database, ops } => {
                let results = sink.apply(database, ops).await.map_err(RunFailure::Write)?;
                engine.feed_ops(id, results)?
            }
        };
    }
}

/// The columns a resolved `SELECT` returns, named and typed as its outcome
/// will be.
pub fn result_columns(
    catalog: &Catalog,
    select: &SelectQuery,
) -> Result<Vec<OutcomeColumn>, RunError> {
    describe(
        catalog,
        &select.items,
        &select.labels,
        &select.bindings,
        &select.relations,
    )
}

/// Name and type each select item.
pub(crate) fn describe(
    catalog: &Catalog,
    items: &[SelectItem],
    labels: &[(usize, String)],
    bindings: &[Binding],
    relations: &[Relation],
) -> Result<Vec<OutcomeColumn>, RunError> {
    let column = |key: Uuid| {
        column_of(catalog, bindings, relations, key)
            .cloned()
            .or_else(|| virtual_column_of(bindings, relations, key))
            .ok_or(RunError::UnboundColumn { key })
    };
    let mut columns: Vec<OutcomeColumn> = items
        .iter()
        .map(|item| match item {
            SelectItem::Column(key) => {
                let column = column(*key)?;
                let bound = binding(bindings, *key);
                let row_table = bound
                    .and_then(|bound| relations.get(bound.relation))
                    .map(|relation| relation.table)
                    .filter(|table| *key == row_id_key(*table));
                Ok(OutcomeColumn {
                    column: bound.and_then(|bound| bound.column),
                    table: row_table,
                    kind: match column.kind {
                        _ if row_table.is_some() => OutcomeKind::Row,
                        ColumnKind::Text | ColumnKind::Link => OutcomeKind::Text,
                        ColumnKind::Number => OutcomeKind::Number,
                        ColumnKind::Boolean => OutcomeKind::Boolean,
                        ColumnKind::Date => OutcomeKind::Date,
                        ColumnKind::DateRange => OutcomeKind::DateRange,
                        ColumnKind::Select { .. } => OutcomeKind::Select,
                        ColumnKind::Entity { .. } => OutcomeKind::Entity,
                    },
                    name: column.name,
                })
            }
            SelectItem::Aggregate {
                function,
                column: None,
            } => Ok(OutcomeColumn {
                name: format!("{}(*)", function.name()),
                column: None,
                kind: OutcomeKind::Number,
                table: None,
            }),
            SelectItem::Aggregate {
                function,
                column: Some(key),
            } => {
                let column = column(*key)?;
                Ok(OutcomeColumn {
                    name: format!("{}({})", function.name(), column.name),
                    column: None,
                    kind: match (function, &column.kind) {
                        (AggregateFunction::Min | AggregateFunction::Max, ColumnKind::Date) => {
                            OutcomeKind::Date
                        }
                        _ => OutcomeKind::Number,
                    },
                    table: None,
                })
            }
        })
        .collect::<Result<_, RunError>>()?;
    for (index, label) in labels {
        if let Some(column) = columns.get_mut(*index) {
            column.name = label.clone();
        }
    }
    Ok(columns)
}