Apache DataFusion SQL Query Engine
fix: keep extension types when skip_metadata clears Parquet field metadata (#26854)
Yury Syrovetsky0ef1b4b45775
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex c825dfb2a8..c50d061977 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -1325,7 +1325,9 @@ config_namespace! {/// (reading) If true, the parquet reader skip the optional embedded metadata that may be in/// the file Schema. This setting can help avoid schema conflicts when querying- /// multiple parquet files with schemas containing compatible types but different metadata+ /// multiple parquet files with schemas containing compatible types but different metadata.+ /// Arrow extension types (the `ARROW:extension:name` and `ARROW:extension:metadata`+ /// field metadata) are kept, because they are part of the type of a columnpub skip_metadata: bool, default = true/// (reading) If specified, the parquet reader will try and fetch the last `size_hint`diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rsindex 26e50d19a2..cfced8bbb9 100644--- a/datafusion/datasource-parquet/src/file_format.rs+++ b/datafusion/datasource-parquet/src/file_format.rs@@ -21,6 +21,7 @@ pub use crate::schema_coercion::{pub use crate::sink::ParquetSink;use arrow::datatypes::{DataType, FieldRef, Fields, Metadata, Schema, SchemaRef};+use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};use datafusion_datasource::TableSchema;use datafusion_datasource::file_compression_type::FileCompressionType;use datafusion_datasource::file_sink_config::FileSinkConfig;@@ -244,7 +245,8 @@ impl ParquetFormat {}/// Clears all metadata (Schema level and field level) on an iterator-/// of Schemas+/// of Schemas, except for the extension type of the fields, which is part+/// of the type of a field.fn clear_metadata(schemas: impl IntoIterator<Item = Schema>,) -> impl Iterator<Item = Schema> {@@ -253,7 +255,16 @@ fn clear_metadata(.fields().iter().map(|field| {- field.as_ref().clone().with_metadata(Metadata::new()) // clear meta+ let metadata = field+ .metadata()+ .iter()+ .filter(|(key, _)| {+ key.as_str() == EXTENSION_TYPE_NAME_KEY+ || key.as_str() == EXTENSION_TYPE_METADATA_KEY+ })+ .map(|(key, value)| (key.clone(), value.clone()))+ .collect::<Metadata>();+ field.as_ref().clone().with_metadata(metadata)}).collect::<Fields>();Schema::new(fields)diff --git a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.sltindex 10e6a753fd..d068da7889 100644--- a/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt+++ b/datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt@@ -106,7 +106,8 @@ FROM (00010203040506070809000102030507 arrow.uuid 00010203040506070809000102030506 arrow.uuid# Extension types of Parquet columns are visible to expressions evaluated in the scan,-# also when the scan replaces a constant column with a literal from the statistics+# also when the scan replaces a constant column with a literal from the statistics.+# skip_metadata = true clears field metadata but keeps extension types.statement okCOPY (SELECT CAST(arrow_cast(X'00010203040506070809000102030506', 'FixedSizeBinary(16)') AS UUID) AS uuid_val@@ -115,7 +116,7 @@ TO 'test_files/scratch/cast_extension_type_metadata/uuid.parquet'STORED AS PARQUET;statement ok-SET datafusion.execution.parquet.skip_metadata = false;+SET datafusion.execution.parquet.skip_metadata = true;statement okCREATE EXTERNAL TABLE uuid_parquet@@ -132,3 +133,46 @@ DROP TABLE uuid_parquet;statement okRESET datafusion.execution.parquet.skip_metadata;++# Schema inference over several files with skip_metadata = true (the default) keeps the+# extension type and clears the other field metadata, even if it conflicts between files.+# A file without the extension type does not remove it.+statement ok+COPY (+ SELECT with_metadata(+ with_metadata(with_metadata(1, 'ARROW:extension:name', 'example.ext'), 'ARROW:extension:metadata', '{"width":5}'),+ 'other', 'bar'+ ) AS id+)+TO 'test_files/scratch/cast_extension_type_metadata/merge/1.parquet'+STORED AS PARQUET;++statement ok+COPY (+ SELECT with_metadata(+ with_metadata(with_metadata(2, 'ARROW:extension:name', 'example.ext'), 'ARROW:extension:metadata', '{"width":5}'),+ 'other', 'baz'+ ) AS id+)+TO 'test_files/scratch/cast_extension_type_metadata/merge/2.parquet'+STORED AS PARQUET;++statement ok+COPY (SELECT 3 AS id)+TO 'test_files/scratch/cast_extension_type_metadata/merge/3.parquet'+STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE merged+STORED AS PARQUET+LOCATION 'test_files/scratch/cast_extension_type_metadata/merge/';++query I?+SELECT id, arrow_metadata(id) FROM merged ORDER BY id;+----+1 {ARROW:extension:metadata: {"width":5}, ARROW:extension:name: example.ext}+2 {ARROW:extension:metadata: {"width":5}, ARROW:extension:name: example.ext}+3 {ARROW:extension:metadata: {"width":5}, ARROW:extension:name: example.ext}++statement ok+DROP TABLE merged;diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex c7939b29d8..f9531f20e8 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -425,7 +425,7 @@ datafusion.execution.parquet.reorder_filters false (reading) If true, filter expdatafusion.execution.parquet.row_group_range_assignment start_offset (reading) Which byte range of a split file reads each row group. `start_offset` picks the range containing the row group's start. `midpoint` picks the range containing its midpoint, as Spark does, which spreads large row groups more evenly across ranges.datafusion.execution.parquet.schema_force_view_types true (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types.datafusion.execution.parquet.skip_arrow_metadata false (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to <https://docs.rs/parquet/53.3.0/parquet/arrow/arrow_writer/struct.ArrowWriterOptions.html#method.with_skip_arrow_metadata>-datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata+datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata. Arrow extension types (the `ARROW:extension:name` and `ARROW:extension:metadata` field metadata) are kept, because they are part of the type of a columndatafusion.execution.parquet.statistics_enabled page (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer settingdatafusion.execution.parquet.statistics_truncate_length 64 (writing) Sets statistics truncate length. If NULL, uses default parquet writer settingdatafusion.execution.parquet.write_batch_size 1024 (writing) Sets write_batch_size in rowsdiff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.mdindex fe163e4add..0f013baeff 100644--- a/docs/source/user-guide/configs.md+++ b/docs/source/user-guide/configs.md@@ -83,7 +83,7 @@ The following configuration settings are available:| datafusion.execution.time_zone | NULL | The default time zone Some functions, e.g. `now`, return timestamps in this time zone. This is also used to interpret timezone-naive timestamps in comparisons and subtraction with timezone-aware timestamps. || datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. || datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file |-| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata |+| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata. Arrow extension types (the `ARROW:extension:name` and `ARROW:extension:metadata` field metadata) are kept, because they are part of the type of a column || datafusion.execution.parquet.metadata_size_hint | 524288 | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer Default setting to 512 KiB, which should be sufficient for most parquet files, it can reduce one I/O operation per parquet file. If the metadata is larger than the hint, two reads will still be performed. || datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". || datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query |