Apache DataFusion SQL Query Engine
feat: parse `JsonAccess` as a binary operator, add `Operator::Colon` (#20628)
Samyak Sarnayak5d278604381e
diff --git a/datafusion/expr-common/src/operator.rs b/datafusion/expr-common/src/operator.rsindex 33512b0c35..427069b326 100644--- a/datafusion/expr-common/src/operator.rs+++ b/datafusion/expr-common/src/operator.rs@@ -140,6 +140,10 @@ pub enum Operator {////// Not implemented in DataFusion yet.QuestionPipe,+ /// Colon operator, like `:`+ ///+ /// Not implemented in DataFusion yet.+ Colon,}impl Operator {@@ -188,7 +192,8 @@ impl Operator {| Operator::AtQuestion| Operator::Question| Operator::QuestionAnd- | Operator::QuestionPipe => None,+ | Operator::QuestionPipe+ | Operator::Colon => None,}}@@ -283,7 +288,8 @@ impl Operator {| Operator::AtQuestion| Operator::Question| Operator::QuestionAnd- | Operator::QuestionPipe => None,+ | Operator::QuestionPipe+ | Operator::Colon => None,}}@@ -323,7 +329,8 @@ impl Operator {| Operator::AtQuestion| Operator::Question| Operator::QuestionAnd- | Operator::QuestionPipe => 30,+ | Operator::QuestionPipe+ | Operator::Colon => 30,Operator::Plus | Operator::Minus => 40,Operator::Multiply | Operator::Divide | Operator::Modulo => 45,}@@ -369,7 +376,8 @@ impl Operator {| Operator::AtQuestion| Operator::Question| Operator::QuestionAnd- | Operator::QuestionPipe => true,+ | Operator::QuestionPipe+ | Operator::Colon => true,// E.g. `TRUE OR NULL` is `TRUE`Operator::Or@@ -429,6 +437,7 @@ impl fmt::Display for Operator {Operator::Question => "?",Operator::QuestionAnd => "?&",Operator::QuestionPipe => "?|",+ Operator::Colon => ":",};write!(f, "{display}")}diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rsindex dd3db8e40b..fa109e38a4 100644--- a/datafusion/expr-common/src/type_coercion/binary.rs+++ b/datafusion/expr-common/src/type_coercion/binary.rs@@ -324,6 +324,9 @@ impl<'a> BinaryTypeCoercer<'a> {)}},+ Colon => {+ Ok(Signature { lhs: lhs.clone(), rhs: rhs.clone(), ret: lhs.clone() })+ },IntegerDivide | Arrow | LongArrow | HashArrow | HashLongArrow| HashMinus | AtQuestion | Question | QuestionAnd | QuestionPipe => {not_impl_err!("Operator {} is not yet supported", self.op)diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rsindex 72eae396e6..02628b405e 100644--- a/datafusion/physical-expr/src/expressions/binary.rs+++ b/datafusion/physical-expr/src/expressions/binary.rs@@ -715,7 +715,7 @@ impl BinaryExpr {StringConcat => concat_elements(&left, &right),AtArrow | ArrowAt | Arrow | LongArrow | HashArrow | HashLongArrow | AtAt| HashMinus | AtQuestion | Question | QuestionAnd | QuestionPipe- | IntegerDivide => {+ | IntegerDivide | Colon => {not_impl_err!("Binary operator '{:?}' is not supported in the physical expr",self.opdiff --git a/datafusion/sql/src/expr/binary_op.rs b/datafusion/sql/src/expr/binary_op.rsindex edad5bbc6d..4e9025e02e 100644--- a/datafusion/sql/src/expr/binary_op.rs+++ b/datafusion/sql/src/expr/binary_op.rs@@ -22,7 +22,7 @@ use sqlparser::ast::BinaryOperator;impl<S: ContextProvider> SqlToRel<'_, S> {pub(crate) fn parse_sql_binary_op(&self, op: &BinaryOperator) -> Result<Operator> {- match *op {+ match op {BinaryOperator::Gt => Ok(Operator::Gt),BinaryOperator::GtEq => Ok(Operator::GtEq),BinaryOperator::Lt => Ok(Operator::Lt),@@ -68,6 +68,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {BinaryOperator::Question => Ok(Operator::Question),BinaryOperator::QuestionAnd => Ok(Operator::QuestionAnd),BinaryOperator::QuestionPipe => Ok(Operator::QuestionPipe),+ BinaryOperator::Custom(s) if s == ":" => Ok(Operator::Colon),_ => not_impl_err!("Unsupported binary operator: {:?}", op),}}diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rsindex 9aa5be8131..7902eed1e6 100644--- a/datafusion/sql/src/expr/mod.rs+++ b/datafusion/sql/src/expr/mod.rs@@ -22,8 +22,8 @@ use datafusion_expr::planner::{use sqlparser::ast::{AccessExpr, BinaryOperator, CastFormat, CastKind, CeilFloorKind,DataType as SQLDataType, DateTimeField, DictionaryField, Expr as SQLExpr,- ExprWithAlias as SQLExprWithAlias, MapEntry, StructField, Subscript, TrimWhereField,- TypedString, Value, ValueWithSpan,+ ExprWithAlias as SQLExprWithAlias, JsonPath, MapEntry, StructField, Subscript,+ TrimWhereField, TypedString, Value, ValueWithSpan,};use datafusion_common::{@@ -651,10 +651,36 @@ impl<S: ContextProvider> SqlToRel<'_, S> {options: Box::new(WildcardOptions::default()),}),SQLExpr::Tuple(values) => self.parse_tuple(schema, planner_context, values),+ SQLExpr::JsonAccess { value, path } => {+ self.parse_json_access(schema, planner_context, value, &path)+ }_ => not_impl_err!("Unsupported ast node in sqltorel: {sql:?}"),}}+ fn parse_json_access(+ &self,+ schema: &DFSchema,+ planner_context: &mut PlannerContext,+ value: Box<SQLExpr>,+ path: &JsonPath,+ ) -> Result<Expr> {+ let json_path = path.to_string();+ let json_path = if let Some(json_path) = json_path.strip_prefix(":") {+ // sqlparser's JsonPath display adds an extra `:` at the beginning.+ json_path.to_owned()+ } else {+ json_path+ };+ self.build_logical_expr(+ BinaryOperator::Custom(":".to_owned()),+ self.sql_to_expr(*value, schema, planner_context)?,+ // pass json path as a string literal, let the impl parse it when needed.+ Expr::Literal(ScalarValue::Utf8(Some(json_path)), None),+ schema,+ )+ }+/// Parses a struct(..) expression and plans it creationfn parse_struct(&self,diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rsindex bbe1f3dd9d..503048bb3c 100644--- a/datafusion/sql/src/unparser/expr.rs+++ b/datafusion/sql/src/unparser/expr.rs@@ -1094,6 +1094,7 @@ impl Unparser<'_> {Operator::Question => Ok(BinaryOperator::Question),Operator::QuestionAnd => Ok(BinaryOperator::QuestionAnd),Operator::QuestionPipe => Ok(BinaryOperator::QuestionPipe),+ Operator::Colon => Ok(BinaryOperator::Custom(":".to_owned())),}}diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rsindex 4717b843ab..670046f164 100644--- a/datafusion/sql/tests/cases/plan_to_sql.rs+++ b/datafusion/sql/tests/cases/plan_to_sql.rs@@ -2821,3 +2821,39 @@ fn test_struct_expr3() {@r#"SELECT test.c1."metadata".product."name" FROM (SELECT {"metadata": {product: {"name": 'Product Name'}}} AS c1) AS test"#);}++#[test]+fn test_json_access_1() {+ let statement = generate_round_trip_statement(+ GenericDialect {},+ r#"SELECT j1_string:field FROM j1"#,+ );+ assert_snapshot!(+ statement,+ @r#"SELECT (j1.j1_string : 'field') FROM j1"#+ );+}++#[test]+fn test_json_access_2() {+ let statement = generate_round_trip_statement(+ GenericDialect {},+ r#"SELECT j1_string:field[0] FROM j1"#,+ );+ assert_snapshot!(+ statement,+ @r#"SELECT (j1.j1_string : 'field[0]') FROM j1"#+ );+}++#[test]+fn test_json_access_3() {+ let statement = generate_round_trip_statement(+ GenericDialect {},+ r#"SELECT j1_string:field.inner1['inner2'] FROM j1"#,+ );+ assert_snapshot!(+ statement,+ @r#"SELECT (j1.j1_string : 'field.inner1[''inner2'']') FROM j1"#+ );+}diff --git a/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rsindex bd8a9d9a99..9f70e903a0 100644--- a/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs+++ b/datafusion/substrait/src/logical_plan/producer/expr/scalar_function.rs@@ -344,5 +344,6 @@ pub fn operator_to_name(op: Operator) -> &'static str {Operator::BitwiseXor => "bitwise_xor",Operator::BitwiseShiftRight => "bitwise_shift_right",Operator::BitwiseShiftLeft => "bitwise_shift_left",+ Operator::Colon => "colon",}}