World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
//! Text is a write-time convenience, never a second stored rule language (0036).
//! Resolve keys, compile an allowlisted scalar expression, then let the ordinary
//! rule writer check the UUID references in the current base again.
use serde_json::Value;
use sqlparser::ast::{
BinaryOperator, CastKind, Expr as SqlExpr, FunctionArg, FunctionArgExpr, FunctionArguments,
UnaryOperator, Value as SqlValue,
};
use sqlparser::{dialect::GenericDialect, parser::Parser, tokenizer::Token};
use std::collections::HashMap;
use utopia_core::error::{AppError, AppResult};
use utopia_reason::expressions::{DatePart, Scalar};
use utopia_reason::rules::{Arith, Expr, MAX_EXPR_DEPTH};
use utopia_store::business_rules::ConditionInput;
use uuid::Uuid;
const MAX_INPUT_BYTES: usize = 8192;
pub(super) async fn compile_inputs(
pool: &sqlx::PgPool,
kb_id: Uuid,
conclusion: Option<&mut Value>,
conditions: &mut [ConditionInput],
) -> AppResult<()> {
let mut inputs: Vec<&mut Value> = conclusion
.into_iter()
.chain(
conditions
.iter_mut()
.filter(|c| matches!(c.op.as_str(), "gt" | "gte" | "lt" | "lte"))
.filter_map(|c| c.operand.as_mut()),
)
.filter(|value| value.get("expression").is_some())
.collect();
if inputs.is_empty() {
// Metadata-only PATCH must not acquire new semantic requirements.
return Ok(());
}
let rows: Vec<(String, Uuid)> =
sqlx::query_as("SELECT key, id FROM relation_types WHERE kb_id=$1 AND kind='attribute'")
.bind(kb_id)
.fetch_all(pool)
.await?;
let names = rows.into_iter().collect();
for value in &mut inputs {
let text = value
.as_object()
.filter(|o| o.len() == 1)
.and_then(|o| o.get("expression"))
.and_then(Value::as_str)
.ok_or_else(|| {
invalid("Text input is an object containing only an expression string.")
})?;
**value = compile(text, &names)?;
}
Ok(())
}
fn invalid(message: impl Into<String>) -> AppError {
AppError::invalid("bad_expression", message)
}
fn unsupported() -> AppError {
invalid("Use attributes, literals, arithmetic, CAST AS DOUBLE PRECISION, a simple literal CASE, or date_trunc with year, month or day.")
}
fn compile(text: &str, names: &HashMap<String, Uuid>) -> AppResult<Value> {
if text.trim().is_empty() || text.len() > MAX_INPUT_BYTES {
return Err(invalid(
"An expression must contain between one and 8192 bytes.",
));
}
// Bound parser recursion too: parentheses do not count as expression nodes,
// but accepting arbitrarily many of them would still consume a parser stack.
let dialect = GenericDialect {};
let mut parser = Parser::new(&dialect)
.with_recursion_limit(32)
.try_with_sql(text)
.map_err(|e| invalid(e.to_string()))?;
let ast = parser.parse_expr().map_err(|e| invalid(e.to_string()))?;
if parser.peek_token().token != Token::EOF {
return Err(invalid(
"Supply one scalar expression, without a statement, alias or trailing clause.",
));
}
let expr = compile_ast(&ast, names, 0)?;
let tree = expr.to_json();
// The JSON and text entry points must accept the same case shapes and depth.
Expr::from_json(&tree).map_err(|e| AppError::invalid(e.code(), e.message()))?;
Ok(tree)
}
fn literal(ast: &SqlExpr) -> AppResult<Scalar> {
match ast {
SqlExpr::Nested(inner) => literal(inner),
SqlExpr::Value(v) => match &v.value {
SqlValue::Number(text, false) => text
.parse::<f64>()
.ok()
.filter(|n| n.is_finite())
.map(Scalar::Number)
.ok_or_else(unsupported),
SqlValue::SingleQuotedString(text) => Ok(Scalar::Text(text.clone())),
_ => Err(unsupported()),
},
SqlExpr::UnaryOp { op, expr }
if matches!(op, UnaryOperator::Plus | UnaryOperator::Minus) =>
{
let Scalar::Number(n) = literal(expr)? else {
return Err(unsupported());
};
Ok(Scalar::Number(if *op == UnaryOperator::Minus {
-n
} else {
n
}))
}
_ => Err(invalid(
"CASE keys and results must be literal numbers or text.",
)),
}
}
fn compile_ast(ast: &SqlExpr, names: &HashMap<String, Uuid>, depth: usize) -> AppResult<Expr> {
if depth > MAX_EXPR_DEPTH {
let error = utopia_reason::expressions::ExprError::TooDeep;
return Err(AppError::invalid(error.code(), error.message()));
}
match ast {
SqlExpr::Nested(inner) => compile_ast(inner, names, depth),
SqlExpr::Identifier(name) => {
names
.get(&name.value)
.copied()
.map(Expr::Attr)
.ok_or_else(|| {
AppError::invalid(
"unknown_expression_attribute",
format!("There is no attribute key {:?} in this base.", name.value),
)
})
}
SqlExpr::Value(_) => match literal(ast)? {
Scalar::Number(n) => Ok(Expr::Const(n)),
Scalar::Text(s) => Ok(Expr::Text(s)),
},
SqlExpr::BinaryOp { left, op, right } => {
let op = match op {
BinaryOperator::Plus => Arith::Add,
BinaryOperator::Minus => Arith::Sub,
BinaryOperator::Multiply => Arith::Mul,
BinaryOperator::Divide => Arith::Div,
_ => return Err(unsupported()),
};
Ok(Expr::Arith {
op,
l: Box::new(compile_ast(left, names, depth + 1)?),
r: Box::new(compile_ast(right, names, depth + 1)?),
})
}
SqlExpr::UnaryOp { op, expr }
if matches!(op, UnaryOperator::Plus | UnaryOperator::Minus) =>
{
if let Ok(Scalar::Number(n)) = literal(ast) {
return Ok(Expr::Const(n));
}
if *op == UnaryOperator::Plus {
return compile_ast(expr, names, depth);
}
Ok(Expr::Arith {
op: Arith::Sub,
l: Box::new(Expr::Const(0.0)),
r: Box::new(compile_ast(expr, names, depth + 1)?),
})
}
SqlExpr::Cast {
kind,
expr,
data_type,
array,
format,
} if matches!(kind, CastKind::Cast | CastKind::DoubleColon)
&& !array
&& format.is_none()
&& matches!(
data_type.to_string().as_str(),
"DOUBLE" | "DOUBLE PRECISION"
) =>
{
Ok(Expr::Number(Box::new(compile_ast(expr, names, depth + 1)?)))
}
SqlExpr::Case {
operand: Some(operand),
conditions,
else_result,
..
} => {
let arms = conditions
.iter()
.map(|arm| Ok((literal(&arm.condition)?, literal(&arm.result)?)))
.collect::<AppResult<_>>()?;
Ok(Expr::Case {
expr: Box::new(compile_ast(operand, names, depth + 1)?),
arms,
otherwise: else_result.as_ref().map(|v| literal(v)).transpose()?,
})
}
SqlExpr::Function(function)
if function.name.to_string().eq_ignore_ascii_case("date_trunc") =>
{
if function.uses_odbc_syntax
|| !matches!(function.parameters, FunctionArguments::None)
|| function.filter.is_some()
|| function.null_treatment.is_some()
|| function.over.is_some()
|| !function.within_group.is_empty()
{
return Err(unsupported());
}
let FunctionArguments::List(args) = &function.args else {
return Err(unsupported());
};
if args.duplicate_treatment.is_some() || !args.clauses.is_empty() {
return Err(unsupported());
}
let [FunctionArg::Unnamed(FunctionArgExpr::Expr(part)), FunctionArg::Unnamed(FunctionArgExpr::Expr(expr))] =
args.args.as_slice()
else {
return Err(unsupported());
};
let Scalar::Text(part) = literal(part)? else {
return Err(unsupported());
};
let part = DatePart::parse(&part.to_ascii_lowercase()).ok_or_else(unsupported)?;
Ok(Expr::DateTrunc {
part,
expr: Box::new(compile_ast(expr, names, depth + 1)?),
})
}
_ => Err(unsupported()),
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn names() -> HashMap<String, Uuid> {
["amt_pay", "chnl", "dt_crt", "净额"]
.into_iter()
.enumerate()
.map(|(i, name)| (name.to_owned(), Uuid::from_u128(i as u128 + 1)))
.collect()
}
#[test]
fn the_wide_corpus_conversions_compile_to_ids_and_run() {
let names = names();
let inputs = HashMap::from([
(names["amt_pay"], Scalar::Text("12345".into())),
(names["chnl"], Scalar::Number(2.0)),
(
names["dt_crt"],
Scalar::Text("2024-03-01T00:30:00+08:00".into()),
),
(names["净额"], Scalar::Number(250.0)),
]);
for (text, expected) in [
("CAST(amt_pay AS DOUBLE PRECISION) / 100", json!(123.45)),
(
"CASE chnl WHEN 1 THEN 'app' WHEN 2 THEN 'web' ELSE 'other' END",
json!("web"),
),
("date_trunc('month', dt_crt)", json!("2024-02-01")),
("(\"净额\" - 50) / (10 - 2)", json!(25.0)),
("-amt_pay / -1e2", json!(123.45)),
(
"CASE chnl WHEN 2 THEN 'customer''s app' END",
json!("customer's app"),
),
] {
let tree = compile(text, &names).unwrap_or_else(|e| panic!("{text}: {e}"));
let expr = Expr::from_json(&tree).unwrap();
assert_eq!(
expr.evaluate(&|p| inputs.get(&p).cloned())
.unwrap()
.to_json(),
expected,
"{text}"
);
assert_eq!(Expr::from_json(&expr.to_json()).unwrap(), expr);
}
assert!(compile("CASE chnl WHEN 3 THEN 'web' END", &names).is_ok());
}
#[test]
fn unsupported_sql_cannot_be_silently_shortened_into_a_conversion() {
for text in [
"unknown / 100",
"other.amt_pay",
"amt_pay; DELETE FROM jobs",
"amt_pay AS amount",
"amt_pay FROM jobs",
"(SELECT 1)",
"SUM(amt_pay)",
"amt_pay % 100",
"amt_pay > 0",
"CAST(amt_pay AS DECIMAL(10, 2))",
"CAST(amt_pay AS INTEGER)",
"TRY_CAST(amt_pay AS DOUBLE)",
"CASE WHEN chnl = 1 THEN 'app' END",
"CASE chnl WHEN 1 THEN amt_pay END",
"CASE chnl WHEN 1 THEN 'app' ELSE 0 END",
"CASE chnl WHEN NULL THEN 'app' END",
"date_trunc('hour', dt_crt)",
"date_trunc('month', dt_crt, 'UTC')",
"date_trunc(DISTINCT 'month', dt_crt)",
"date_trunc('month', dt_crt) OVER ()",
"date_trunc('month', dt_crt) FILTER (WHERE chnl = 1)",
"date_trunc('month', dt_crt ORDER BY chnl)",
"pg_catalog.date_trunc('month', dt_crt)",
"date_trunc(unit => 'month', value => dt_crt)",
"1e999",
"true",
"NULL",
"",
":parameter",
] {
assert!(compile(text, &names()).is_err(), "accepted {text}");
}
}
#[test]
fn text_and_json_share_the_existing_depth_bound() {
let mut text = "amt_pay".to_owned();
for _ in 0..MAX_EXPR_DEPTH {
text = format!("({text} + 1)");
}
assert!(compile(&text, &names()).is_ok());
assert!(compile(&format!("{text} + 1"), &names()).is_err());
assert!(compile(
&format!("{}amt_pay{}", "(".repeat(1000), ")".repeat(1000)),
&names()
)
.is_err());
assert!(compile(&" ".repeat(MAX_INPUT_BYTES + 1), &names()).is_err());
}
}