From e3c39e26ca27668cd0def6a79fee61c170f1cd85 Mon Sep 17 00:00:00 2001 From: Yulei-Yang Date: Mon, 20 Jan 2025 14:08:57 +0800 Subject: [PATCH 1/4] [fix](date_function) fix str_to_date function return wrong microsecond issue (#47129) Issue Number: close #47105 Related PR: #24932 Problem Summary: str_to_date always return microsecond part for datetime even if user does not specfic %f in date format string. This is wrong. mysql> select id,str_to_date(dt, '%Y-%m-%d %H:%i:%s') from test1 limit 1; +------+--------------------------------------+ | id | str_to_date(dt, '%Y-%m-%d %H:%i:%s') | +------+--------------------------------------+ | 2 | 2024-12-28 10:11:12.000000 | +------+--------------------------------------+ and constant fold scenario is wrong too: mysql> select cast(str_to_date('2025-01-17 11:59:30', '%Y-%m-%d %H:%i:%s') as string); +--------------------------------------------------------------------------+ | cast(str_to_date('2025-01-17 11:59:30', '%Y-%m-%d %H:%i:%s') as TEXT) | +--------------------------------------------------------------------------+ | 2025-01-17 11:59:30.000000 | +--------------------------------------------------------------------------+ - Test - [x] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../apache/doris/analysis/DateLiteral.java | 5 ++ .../DateTimeExtractAndTransform.java | 4 +- .../functions/scalar/StrToDate.java | 3 +- .../test_date_function_v2.groovy | 60 +++++++++++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 67d99b1b06ea73..35b8076789415a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -111,6 +111,7 @@ public class DateLiteral extends LiteralExpr { private static Map WEEK_DAY_NAME_DICT = Maps.newHashMap(); private static Set TIME_PART_SET = Sets.newHashSet(); private static final int[] DAYS_IN_MONTH = new int[] {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + private static String MICRO_SECOND_FORMATTER = "%f"; private static final WeekFields weekFields = WeekFields.of(DayOfWeek.SUNDAY, 7); static { @@ -1062,6 +1063,10 @@ public static boolean hasTimePart(String format) { return format.chars().anyMatch(c -> TIME_PART_SET.contains((char) c)); } + public static boolean hasMicroSecondPart(String format) { + return format.indexOf(MICRO_SECOND_FORMATTER) != -1; + } + // Return the date stored in the dateliteral as pattern format. // eg : "%Y-%m-%d" or "%Y-%m-%d %H:%i:%s" public String dateFormat(String pattern) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java index 1f6f3afd7d4086..d0023ed2ff6a03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransform.java @@ -628,8 +628,10 @@ public static Expression strToDate(StringLikeLiteral str, StringLikeLiteral form if (org.apache.doris.analysis.DateLiteral.hasTimePart(format.getStringValue())) { DataType returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME)); if (returnType instanceof DateTimeV2Type) { + boolean hasMicroPart = org.apache.doris.analysis.DateLiteral + .hasMicroSecondPart(format.getStringValue()); return DateTimeV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) - .toFormatter(), str.getValue())); + .toFormatter(), str.getValue()), hasMicroPart ? 6 : 0); } else { return DateTimeLiteral.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue()) .toFormatter(), str.getValue())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java index ff014db6bca7a5..c768a25ba3835e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java @@ -88,7 +88,8 @@ public FunctionSignature computeSignature(FunctionSignature signature) { if (getArgument(1) instanceof StringLikeLiteral) { if (DateLiteral.hasTimePart(((StringLikeLiteral) getArgument(1)).getStringValue())) { returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME)); - if (returnType.isDateTimeV2Type()) { + if (returnType.isDateTimeV2Type() + && DateLiteral.hasMicroSecondPart(((StringLikeLiteral) getArgument(1)).getStringValue())) { returnType = DataType.fromCatalogType(Type.DATETIMEV2_WITH_MAX_SCALAR); } } else { diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy new file mode 100644 index 00000000000000..bdb8af9ae9b0ae --- /dev/null +++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_date_function_v2") { + sql """ + admin set frontend config ("enable_date_conversion"="true"); + """ + + def tableName = "test_date_function_v2" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `id` INT, + `name` varchar(32), + `dt` varchar(32) + ) ENGINE=OLAP + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) + """ + sql """ insert into ${tableName} values (3, 'Carl','2024-12-29 10:11:12') """ + def result1 = try_sql """ + select cast(str_to_date(dt, '%Y-%m-%d %H:%i:%s') as string) from ${tableName}; + """ + assertEquals(result1[0][0], "2024-12-29 10:11:12"); + + def result2 = try_sql """ + select cast(str_to_date(dt, '%Y-%m-%d %H:%i:%s.%f') as string) from ${tableName}; + """ + assertEquals(result2[0][0], "2024-12-29 10:11:12.000000"); + + def result3 = try_sql """ + select cast(str_to_date("2025-01-17 11:59:30", '%Y-%m-%d %H:%i:%s') as string); + """ + assertEquals(result3[0][0], "2025-01-17 11:59:30"); + + def result4 = try_sql """ + select cast(str_to_date("2025-01-17 11:59:30", '%Y-%m-%d %H:%i:%s.%f') as string); + """ + assertEquals(result4[0][0], "2025-01-17 11:59:30.000000"); + + sql """ drop table ${tableName} """ +} From 794beab424d2833d5a953575f0224c398b969a74 Mon Sep 17 00:00:00 2001 From: Yulei-Yang Date: Mon, 20 Jan 2025 19:56:11 +0800 Subject: [PATCH 2/4] [fix](date_function) fix str_to_date function return wrong microsecond issue --- .../apache/doris/analysis/DateLiteral.java | 5 +++- .../doris/analysis/FunctionCallExpr.java | 6 ++++- .../test_date_function_v2.groovy | 24 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 35b8076789415a..414b955604372c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -718,7 +718,7 @@ public String getStringValue() { int scale = ((ScalarType) type).getScalarScale(); long scaledMicroseconds = (long) (microsecond / SCALE_FACTORS[scale]); - if (scaledMicroseconds != 0) { + if (scale > 0) { dateTimeChars[19] = '.'; fillPaddedValue(dateTimeChars, 20, (int) scaledMicroseconds, scale); return new String(dateTimeChars, 0, 20 + scale); @@ -1609,6 +1609,9 @@ public int fromDateFormatStr(String format, String value, boolean hasSubVal) thr case 'T': partUsed |= timePart; break; + case 'f': + partUsed |= fracPart; + break; default: break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 8d71bdc35573a7..c87ef4d144f07e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1993,7 +1993,11 @@ && collectChildReturnTypes()[0].isDecimalV3()) { Expr child1Result = getChild(1).getResultValue(false); if (child1Result instanceof StringLiteral) { if (DateLiteral.hasTimePart(child1Result.getStringValue())) { - this.type = Type.DATETIMEV2_WITH_MAX_SCALAR; + if (DateLiteral.hasMicroSecondPart(child1Result.getStringValue())) { + this.type = Type.DATETIMEV2_WITH_MAX_SCALAR; + } else { + this.type = Type.DEFAULT_DATETIMEV2; + } } else { this.type = Type.DATEV2; } diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy index bdb8af9ae9b0ae..8b24c9d9c3fefd 100644 --- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function_v2.groovy @@ -19,6 +19,7 @@ suite("test_date_function_v2") { sql """ admin set frontend config ("enable_date_conversion"="true"); """ + sql """SET enable_nereids_planner=true""" def tableName = "test_date_function_v2" @@ -56,5 +57,28 @@ suite("test_date_function_v2") { """ assertEquals(result4[0][0], "2025-01-17 11:59:30.000000"); + // test legacy planner + sql """SET enable_nereids_planner=false""" + def result5 = try_sql """ + select cast(str_to_date(dt, '%Y-%m-%d %H:%i:%s') as string) from ${tableName}; + """ + assertEquals(result5[0][0], "2024-12-29 10:11:12"); + + result5 = try_sql """ + select cast(str_to_date(dt, '%Y-%m-%d %H:%i:%s.%f') as string) from ${tableName}; + """ + assertEquals(result5[0][0], "2024-12-29 10:11:12.000000"); + + result5 = try_sql """ + select cast(str_to_date('2025-01-17 11:59:30', '%Y-%m-%d %H:%i:%s') as string); + """ + assertEquals(result5[0][0], "2025-01-17 11:59:30"); + + result5 = try_sql """ + select cast(str_to_date('2025-01-17 11:59:30', '%Y-%m-%d %H:%i:%s.%f') as string); + """ + assertEquals(result5[0][0], "2025-01-17 11:59:30.000000"); + + sql """ drop table ${tableName} """ } From 4bcd4fbfe2ce0639a26e2844fc05ae7b23c67b24 Mon Sep 17 00:00:00 2001 From: Yulei-Yang Date: Tue, 21 Jan 2025 10:47:45 +0800 Subject: [PATCH 3/4] update --- .../src/main/java/org/apache/doris/analysis/DateLiteral.java | 2 +- .../array_functions/test_array_with_scale_type.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 414b955604372c..7685a24ab1b770 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -1673,7 +1673,7 @@ public int fromDateFormatStr(String format, String value, boolean hasSubVal) thr // Compute timestamp type if ((partUsed & datePart) != 0) { // Ymd part only - if ((partUsed & fracPart) != 0) { + if (hasMicroSecondPart(format)) { this.type = Type.DATETIMEV2_WITH_MAX_SCALAR; } else if ((partUsed & timePart) != 0) { this.type = ScalarType.getDefaultDateType(Type.DATETIME); diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out index 446a96fa2ae73c..9388c8018789c5 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out @@ -108,7 +108,7 @@ ["2022-12-02 22:23:25.000", "2022-12-02 22:23:23.998"] -- !select -- -["2022-12-02 22:23:25", "2022-12-02 22:23:23.998"] +["2022-12-02 22:23:25.000", "2022-12-02 22:23:23.998"] -- !select -- [] From a372cc76046aba0590a5a425394c7e171477ae16 Mon Sep 17 00:00:00 2001 From: Yulei-Yang Date: Wed, 22 Jan 2025 11:49:06 +0800 Subject: [PATCH 4/4] fix reg case --- .../external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy index d2b3669ab457a3..cb2633e404c9e9 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy @@ -134,7 +134,7 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d order_qt_date4 """ select * from TEST_DATE where (T1 > '2022-01-21 00:00:00' and T1 < '2022-01-22 00:00:00') or (T1 > '2022-01-20 00:00:00' and T1 < '2022-01-23 00:00:00'); """ order_qt_date5 """ select * from TEST_DATE where T1 < '2022-01-22 00:00:00' or T1 = '2022-01-21 05:23:01'; """ order_qt_date6 """ select * from TEST_DATE where (T1 < '2022-01-22 00:00:00' or T1 > '2022-01-20 00:00:00') and (T1 < '2022-01-23 00:00:00' or T1 > '2022-01-19 00:00:00'); """ - order_qt_date7 """select * from TEST_TIMESTAMP where T2 < str_to_date('2020-12-21 12:34:56', '%Y-%m-%d %H:%i:%s');""" + order_qt_date7 """select * from TEST_TIMESTAMP where T2 < str_to_date('2020-12-21 12:34:56', '%Y-%m-%d %H:%i:%s.%f');""" // test nvl explain {