Apache DataFusion SQL Query Engine
fix(spark): align parse_url empty FILE path (#21969)
Kumar Ujjawal52f23ae8f7f9
diff --git a/datafusion/spark/src/function/url/parse_url.rs b/datafusion/spark/src/function/url/parse_url.rsindex 2374d6c084..18f0bb1e0d 100644--- a/datafusion/spark/src/function/url/parse_url.rs+++ b/datafusion/spark/src/function/url/parse_url.rs@@ -77,7 +77,7 @@ impl ParseUrl {/// # Returns////// * `Ok(Some(String))` - The extracted URL component as a string- /// * `Ok(None)` - If the requested component doesn't exist or is empty+ /// * `Ok(None)` - If the requested component doesn't exist/// * `Err(DataFusionError)` - If the URL is malformed and cannot be parsedfn parse(value: &str, part: &str, key: Option<&str>) -> Result<Option<String>> {let url: std::result::Result<Url, ParseError> = Url::parse(value);@@ -97,12 +97,7 @@ impl ParseUrl {"PATH" => Some(path.to_string()),"QUERY" => match key {None => query.map(String::from),- Some(key) => query.and_then(|q| {- q.split('&')- .filter_map(|pair| pair.split_once('='))- .find(|(k, _)| *k == key)- .map(|(_, v)| v.to_string())- }),+ Some(key) => Self::query_value(query, key).map(String::from),},"REF" => fragment.map(String::from),"FILE" => {@@ -122,21 +117,17 @@ impl ParseUrl {.map(|url| match part {"HOST" => url.host_str().map(String::from),"PATH" => {- let path: String = url.path().to_string();- let path: String = if path == "/" { "".to_string() } else { path };- Some(path)+ let path = Self::path(value, &url);+ Some(path.to_string())}"QUERY" => match key {None => url.query().map(String::from),- Some(key) => url- .query_pairs()- .find(|(k, _)| k == key)- .map(|(_, v)| v.into_owned()),+ Some(key) => Self::query_value(url.query(), key).map(String::from),},"REF" => url.fragment().map(String::from),"PROTOCOL" => Some(url.scheme().to_string()),"FILE" => {- let path = url.path();+ let path = Self::path(value, &url);match url.query() {Some(query) => Some(format!("{path}?{query}")),None => Some(path.to_string()),@@ -156,6 +147,36 @@ impl ParseUrl {_ => None,})}++ fn path<'a>(value: &str, url: &'a Url) -> &'a str {+ let path = url.path();+ if path == "/" && Self::absolute_url_has_empty_path(value) {+ ""+ } else {+ path+ }+ }++ fn absolute_url_has_empty_path(value: &str) -> bool {+ let Some(authority_start) = value.find("://").map(|index| index + 3) else {+ return false;+ };+ let after_authority = &value[authority_start..];+ match after_authority.find(['/', '?', '#']) {+ None => true,+ Some(index) => matches!(after_authority.as_bytes()[index], b'?' | b'#'),+ }+ }++ fn query_value<'a>(query: Option<&'a str>, key: &str) -> Option<&'a str> {+ query.and_then(|query| {+ query+ .split('&')+ .filter_map(|pair| pair.split_once('='))+ .find(|(query_key, _)| *query_key == key)+ .map(|(_, value)| value)+ })+ }}impl ScalarUDFImpl for ParseUrl {@@ -382,9 +403,79 @@ mod tests {}#[test]- fn test_parse_path_root_is_empty_string() -> Result<()> {- let got = ParseUrl::parse("https://example.com/", "PATH", None)?;- assert_eq!(got, Some("".to_string()));+ fn test_parse_path_empty_vs_root() -> Result<()> {+ assert_eq!(+ ParseUrl::parse("https://example.com", "PATH", None)?,+ Some("".to_string())+ );+ assert_eq!(+ ParseUrl::parse("https://example.com/", "PATH", None)?,+ Some("/".to_string())+ );+ assert_eq!(+ ParseUrl::parse("https://ex.com/dir%20/pa%20th.HTML", "PATH", None)?,+ Some("/dir%20/pa%20th.HTML".to_string())+ );+ Ok(())+ }++ #[test]+ fn test_parse_query_key_is_raw() -> Result<()> {+ let url = "https://use%20r:pas%[email protected]/dir%20/pa%20th.HTML?query=x%20y&q2=2#Ref%20two";+ assert_eq!(+ ParseUrl::parse(url, "QUERY", None)?,+ Some("query=x%20y&q2=2".to_string())+ );+ assert_eq!(+ ParseUrl::parse(url, "QUERY", Some("query"))?,+ Some("x%20y".to_string())+ );+ assert_eq!(+ ParseUrl::parse("http://ex.com?key=", "QUERY", Some("key"))?,+ Some("".to_string())+ );+ assert_eq!(+ ParseUrl::parse("http://ex.com?keyonly", "QUERY", Some("keyonly"))?,+ None+ );+ assert_eq!(+ ParseUrl::parse("http://ex.com?a=1&a=2", "QUERY", Some("a"))?,+ Some("1".to_string())+ );+ assert_eq!(+ ParseUrl::parse("http://ex.com?a%20b=1", "QUERY", Some("a b"))?,+ None+ );+ Ok(())+ }++ #[test]+ fn test_parse_empty_path_file() -> Result<()> {+ assert_eq!(ParseUrl::parse("", "PATH", None)?, Some("".to_string()));+ assert_eq!(+ ParseUrl::parse("http://example.com", "FILE", None)?,+ Some("".to_string())+ );+ assert_eq!(+ ParseUrl::parse("http://example.com?foo=bar", "FILE", None)?,+ Some("?foo=bar".to_string())+ );+ assert_eq!(+ ParseUrl::parse("http://example.com#fragment", "FILE", None)?,+ Some("".to_string())+ );+ assert_eq!(+ ParseUrl::parse("http://example.com/?foo=bar", "FILE", None)?,+ Some("/?foo=bar".to_string())+ );+ assert_eq!(+ ParseUrl::parse("http://ex.com/?", "FILE", None)?,+ Some("/?".to_string())+ );+ assert_eq!(+ ParseUrl::parse("http://ex.com?", "FILE", None)?,+ Some("?".to_string())+ );Ok(())}@@ -482,7 +573,7 @@ mod tests {assert_eq!(out_sa.len(), 2);assert_eq!(out_sa.value(0), "example.com");- assert_eq!(out_sa.value(1), "");+ assert_eq!(out_sa.value(1), "/");Ok(())}diff --git a/datafusion/sqllogictest/test_files/spark/url/parse_url.slt b/datafusion/sqllogictest/test_files/spark/url/parse_url.sltindex 7a5051d50e..7803084693 100644--- a/datafusion/sqllogictest/test_files/spark/url/parse_url.slt+++ b/datafusion/sqllogictest/test_files/spark/url/parse_url.slt@@ -235,6 +235,86 @@ SELECT parse_url('https://example.com', 'PATH');----(empty)+query T+SELECT parse_url('https://example.com/', 'PATH');+----+/++query T+SELECT parse_url('https://ex.com/dir%20/pa%20th.HTML', 'PATH');+----+/dir%20/pa%20th.HTML++query T+SELECT parse_url('', 'PATH');+----+(empty)++query T+SELECT parse_url('http://example.com', 'FILE');+----+(empty)++query T+SELECT parse_url('http://example.com/', 'FILE');+----+/++query T+SELECT parse_url('http://example.com?foo=bar', 'FILE');+----+?foo=bar++query T+SELECT parse_url('http://example.com#fragment', 'FILE');+----+(empty)++query T+SELECT parse_url('http://example.com/?foo=bar', 'FILE');+----+/?foo=bar++query T+SELECT parse_url('http://ex.com/?', 'FILE');+----+/?++query T+SELECT parse_url('http://ex.com?', 'FILE');+----+?++query T+SELECT parse_url('https://use%20r:pas%[email protected]/dir%20/pa%20th.HTML?query=x%20y&q2=2#Ref%20two', 'QUERY');+----+query=x%20y&q2=2++query T+SELECT parse_url('https://use%20r:pas%[email protected]/dir%20/pa%20th.HTML?query=x%20y&q2=2#Ref%20two', 'QUERY', 'query');+----+x%20y++query T+SELECT parse_url('http://ex.com?key=', 'QUERY', 'key');+----+(empty)++query T+SELECT parse_url('http://ex.com?keyonly', 'QUERY', 'keyonly');+----+NULL++query T+SELECT parse_url('http://ex.com?a=1&a=2', 'QUERY', 'a');+----+1++query T+SELECT parse_url('http://ex.com?a%20b=1', 'QUERY', 'a b');+----+NULL+query TSELECT parse_url('https://example.com', 'path');----diff --git a/datafusion/sqllogictest/test_files/spark/url/try_parse_url.slt b/datafusion/sqllogictest/test_files/spark/url/try_parse_url.sltindex a0e42a1648..4b105f1450 100644--- a/datafusion/sqllogictest/test_files/spark/url/try_parse_url.slt+++ b/datafusion/sqllogictest/test_files/spark/url/try_parse_url.slt@@ -186,6 +186,86 @@ SELECT try_parse_url('https://example.com', 'PATH');----(empty)+query T+SELECT try_parse_url('https://example.com/', 'PATH');+----+/++query T+SELECT try_parse_url('https://ex.com/dir%20/pa%20th.HTML', 'PATH');+----+/dir%20/pa%20th.HTML++query T+SELECT try_parse_url('', 'PATH');+----+(empty)++query T+SELECT try_parse_url('http://example.com', 'FILE');+----+(empty)++query T+SELECT try_parse_url('http://example.com/', 'FILE');+----+/++query T+SELECT try_parse_url('http://example.com?foo=bar', 'FILE');+----+?foo=bar++query T+SELECT try_parse_url('http://example.com#fragment', 'FILE');+----+(empty)++query T+SELECT try_parse_url('http://example.com/?foo=bar', 'FILE');+----+/?foo=bar++query T+SELECT try_parse_url('http://ex.com/?', 'FILE');+----+/?++query T+SELECT try_parse_url('http://ex.com?', 'FILE');+----+?++query T+SELECT try_parse_url('https://use%20r:pas%[email protected]/dir%20/pa%20th.HTML?query=x%20y&q2=2#Ref%20two', 'QUERY');+----+query=x%20y&q2=2++query T+SELECT try_parse_url('https://use%20r:pas%[email protected]/dir%20/pa%20th.HTML?query=x%20y&q2=2#Ref%20two', 'QUERY', 'query');+----+x%20y++query T+SELECT try_parse_url('http://ex.com?key=', 'QUERY', 'key');+----+(empty)++query T+SELECT try_parse_url('http://ex.com?keyonly', 'QUERY', 'keyonly');+----+NULL++query T+SELECT try_parse_url('http://ex.com?a=1&a=2', 'QUERY', 'a');+----+1++query T+SELECT try_parse_url('http://ex.com?a%20b=1', 'QUERY', 'a b');+----+NULL+query TSELECT try_parse_url('https://example.com', 'path');----