From 8074dfdb9289cb54536355b401b94a243ec1059e Mon Sep 17 00:00:00 2001 From: Artem Medvedev Date: Sat, 25 May 2024 19:44:13 +0200 Subject: [PATCH] fix: pass `quote` parameter to CSV writer Closes #10670 --- .../common/src/file_options/csv_writer.rs | 1 + .../sqllogictest/test_files/csv_files.slt | 41 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/datafusion/common/src/file_options/csv_writer.rs b/datafusion/common/src/file_options/csv_writer.rs index 2904ea0f8f03..4f948a29adc4 100644 --- a/datafusion/common/src/file_options/csv_writer.rs +++ b/datafusion/common/src/file_options/csv_writer.rs @@ -51,6 +51,7 @@ impl TryFrom<&CsvOptions> for CsvWriterOptions { fn try_from(value: &CsvOptions) -> Result { let mut builder = WriterBuilder::default() .with_header(value.has_header.unwrap_or(false)) + .with_quote(value.quote) .with_delimiter(value.delimiter); if let Some(v) = &value.date_format { diff --git a/datafusion/sqllogictest/test_files/csv_files.slt b/datafusion/sqllogictest/test_files/csv_files.slt index 50477e1dab45..f581fa9abcbb 100644 --- a/datafusion/sqllogictest/test_files/csv_files.slt +++ b/datafusion/sqllogictest/test_files/csv_files.slt @@ -161,3 +161,44 @@ physical_plan 01)SortPreservingMergeExec: [int_col@0 ASC NULLS LAST] 02)--SortExec: expr=[int_col@0 ASC NULLS LAST], preserve_partitioning=[true] 03)----CsvExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/csv_files/csv_partitions/1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/csv_files/csv_partitions/2.csv]]}, projection=[int_col, string_col, bigint_col, partition_col], has_header=false + + +# ensure that correct quote character is used when writing to csv +statement ok +CREATE TABLE table_with_necessary_quoting ( + int_col INT, + string_col TEXT, +) AS VALUES +(1, 'e|e|e'), +(2, 'f|f|f'), +(3, 'g|g|g'), +(4, 'h|h|h'); + +# quote is required because `|` is delimiter and part of the data +query IT +COPY table_with_necessary_quoting TO 'test_files/scratch/csv_files/table_with_necessary_quoting.csv' +STORED AS csv +OPTIONS ('format.quote' '~', + 'format.delimiter' '|', + 'format.has_header' 'true'); +---- +4 + +# read the stored csv file with quote character +statement ok +CREATE EXTERNAL TABLE stored_table_with_necessary_quoting ( +c1 VARCHAR, +c2 VARCHAR +) STORED AS CSV +LOCATION 'test_files/scratch/csv_files/table_with_necessary_quoting.csv' +OPTIONS ('format.quote' '~', + 'format.delimiter' '|', + 'format.has_header' 'true'); + +query TT +select * from stored_table_with_necessary_quoting; +---- +1 e|e|e +2 f|f|f +3 g|g|g +4 h|h|h