Skip to content

Apache DataFusion SQL Query Engine

fix: change token consumption to pick to test on EOF in parser (#19927)

Albert Skalt1897924b7a40
diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs
index 27db2b0f97..7ce1f977e9 100644
--- a/datafusion/sql/src/parser.rs
+++ b/datafusion/sql/src/parser.rs
@@ -658,7 +658,7 @@ impl<'a> DFParser<'a> {
}
}
} else {
- let token = self.parser.next_token();
+ let token = self.parser.peek_token();
if token == Token::EOF || token == Token::SemiColon {
break;
} else {
@@ -1079,7 +1079,7 @@ impl<'a> DFParser<'a> {
}
}
} else {
- let token = self.parser.next_token();
+ let token = self.parser.peek_token();
if token == Token::EOF || token == Token::SemiColon {
break;
} else {
@@ -2026,6 +2026,48 @@ mod tests {
);
}
+ #[test]
+ fn test_multistatement() {
+ let sql = "COPY foo TO bar STORED AS CSV; \
+ CREATE EXTERNAL TABLE t(c1 int) STORED AS CSV LOCATION 'foo.csv'; \
+ RESET var;";
+ let statements = DFParser::parse_sql(sql).unwrap();
+ assert_eq!(
+ statements,
+ vec![
+ Statement::CopyTo(CopyToStatement {
+ source: object_name("foo"),
+ target: "bar".to_string(),
+ partitioned_by: vec![],
+ stored_as: Some("CSV".to_owned()),
+ options: vec![],
+ }),
+ {
+ let name = ObjectName::from(vec![Ident::from("t")]);
+ let display = None;
+ Statement::CreateExternalTable(CreateExternalTable {
+ name: name.clone(),
+ columns: vec![make_column_def("c1", DataType::Int(display))],
+ file_type: "CSV".to_string(),
+ location: "foo.csv".into(),
+ table_partition_cols: vec![],
+ order_exprs: vec![],
+ if_not_exists: false,
+ or_replace: false,
+ temporary: false,
+ unbounded: false,
+ options: vec![],
+ constraints: vec![],
+ })
+ },
+ {
+ let name = ObjectName::from(vec![Ident::from("var")]);
+ Statement::Reset(ResetStatement::Variable(name))
+ }
+ ]
+ );
+ }
+
fn expect_parse_expr_ok(sql: &str, expected: ExprWithAlias) {
let expr = DFParser::parse_sql_into_expr(sql).unwrap();
assert_eq!(expr, expected, "actual:\n{expr:#?}");