Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix](datetime) fix unstable str_to_date function result #25707

Merged
merged 2 commits into from
Oct 23, 2023
Merged

[fix](datetime) fix unstable str_to_date function result #25707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b61565f
code
zclllyybb Oct 20, 2023
c914a9b
fix
zclllyybb Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions be/src/vec/functions/function_timestamp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,16 +64,22 @@

namespace doris::vectorized {

template <typename DateType>
struct StrToDate {
static constexpr auto name = "str_to_date";

static bool is_variadic() { return false; }

static DataTypes get_variadic_argument_types() { return {}; }
static DataTypes get_variadic_argument_types() {
return {std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>()};
}

static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
//TODO: it doesn't matter now. maybe sometime we should find the function signature with return_type together
return make_nullable(std::make_shared<DataTypeDateTime>());
if constexpr (IsDateType<DateType> || IsDateV2Type<DateType>) {
return make_nullable(std::make_shared<DataTypeDateV2>());
}
//datetimev2
return make_nullable(std::make_shared<DataTypeDateTimeV2>(6));
}

static StringRef rewrite_specific_format(const char* raw_str, size_t str_size) {
Expand Down Expand Up @@ -1266,7 +1272,8 @@ class FunctionOtherTypesToDateType : public IFunction {
}
};

using FunctionStrToDate = FunctionOtherTypesToDateType<StrToDate>;
using FunctionStrToDate = FunctionOtherTypesToDateType<StrToDate<DataTypeDateV2>>;
using FunctionStrToDatetime = FunctionOtherTypesToDateType<StrToDate<DataTypeDateTimeV2>>;
using FunctionMakeDate = FunctionOtherTypesToDateType<MakeDateImpl>;
using FunctionDateTruncDate = FunctionOtherTypesToDateType<DateTrunc<DataTypeDate>>;
using FunctionDateTruncDateV2 = FunctionOtherTypesToDateType<DateTrunc<DataTypeDateV2>>;
Expand All @@ -1275,6 +1282,7 @@ using FunctionDateTruncDatetimeV2 = FunctionOtherTypesToDateType<DateTrunc<DataT

void register_function_timestamp(SimpleFunctionFactory& factory) {
factory.register_function<FunctionStrToDate>();
factory.register_function<FunctionStrToDatetime>();
factory.register_function<FunctionMakeDate>();
factory.register_function<FromDays>();
factory.register_function<FunctionDateTruncDate>();
Expand Down
19 changes: 0 additions & 19 deletions be/test/vec/function/function_string_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1121,25 +1121,6 @@ TEST(function_string_test, function_coalesce_test) {
}
}

TEST(function_string_test, function_str_to_date_test) {
std::string func_name = "str_to_date";
InputTypeSet input_types = {
TypeIndex::String,
TypeIndex::String,
};
DataSet data_set = {
{{Null(), std::string("%Y-%m-%d %H:%i:%s")}, {Null()}},
{{std::string("2014-12-21 12:34:56"), std::string("%Y-%m-%d %H:%i:%s")},
str_to_date_time("2014-12-21 12:34:56", false)},
{{std::string("2014-12-21 12:34%3A56"), std::string("%Y-%m-%d %H:%i%%3A%s")},
str_to_date_time("2014-12-21 12:34:56", false)},
{{std::string("11/09/2011"), std::string("%m/%d/%Y")},
str_to_date_time("2011-11-09", false)},
{{std::string("2020-09-01"), std::string("%Y-%m-%d %H:%i:%s")},
str_to_date_time("2020-09-01 00:00:00", false)}};
static_cast<void>(check_function<DataTypeDateTime, true>(func_name, input_types, data_set));
}

TEST(function_string_test, function_replace) {
std::string func_name = "replace";
InputTypeSet input_types = {
Expand Down
17 changes: 0 additions & 17 deletions be/test/vec/function/function_time_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1395,23 +1395,6 @@ TEST(VTimestampFunctionsTest, yearweek_v2_test) {
}
}

TEST(VTimestampFunctionsTest, str_to_date_test) {
std::string func_name = "str_to_date";

InputTypeSet input_types = {TypeIndex::String, TypeIndex::String};

{
DataSet data_set = {{{std::string("2021-01-01"), std::string("%Y-%m-%d")},
str_to_date_time("2021-01-01", false)},
{{std::string("2022-01-03"), std::string("%Y-%m-%d")},
str_to_date_time("2022-01-03", false)},
{{std::string("2021-00-01"), std::string("%Y-%m-%d")}, Null()},
{{std::string("2021-01-00"), std::string("%Y-%m-%d")}, Null()}};

static_cast<void>(check_function<DataTypeDate, true>(func_name, input_types, data_set));
}
}

TEST(VTimestampFunctionsTest, from_days_test) {
std::string func_name = "from_days";

Expand Down
19 changes: 15 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1276,9 +1276,20 @@ public int fromDateFormatStr(String format, String value, boolean hasSubVal) thr
break;
// Micro second
case 'f':
tmp = vp + Math.min(6, vend - vp);
intValue = strToLong(value.substring(vp, tmp));
this.microsecond = (long) (intValue * Math.pow(10, 6 - Math.min(6, vend - vp)));
// FIXME: fix same with BE
tmp = vp;
// when there's still something to the end, fix the scale of ms.
while (tmp < vend && Character.isDigit(value.charAt(tmp))) {
tmp += 1;
}

if (tmp - vp > 6) {
int tmp2 = vp + 6;
intValue = strToLong(value.substring(vp, tmp2));
} else {
intValue = strToLong(value.substring(vp, tmp));
}
this.microsecond = (long) (intValue * Math.pow(10, 6 - Math.min(6, tmp - vp)));
timePartUsed = true;
microSecondPartUsed = true;
vp = tmp;
Expand Down Expand Up @@ -1476,7 +1487,7 @@ public int fromDateFormatStr(String format, String value, boolean hasSubVal) thr
// we think it's stable enough
if (datePartUsed) {
if (microSecondPartUsed) {
this.type = Type.DATETIMEV2;
this.type = Type.DATETIMEV2_WITH_MAX_SCALAR;
} else if (timePartUsed) {
this.type = ScalarType.getDefaultDateType(Type.DATETIME);
} else {
Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1900,12 +1900,12 @@ && collectChildReturnTypes()[0].isDecimalV3()) {
Expr child1Result = getChild(1).getResultValue(false);
if (child1Result instanceof StringLiteral) {
if (DateLiteral.hasTimePart(child1Result.getStringValue())) {
this.type = Type.DATETIME;
this.type = Type.DATETIMEV2_WITH_MAX_SCALAR;
} else {
this.type = Type.DATE;
this.type = Type.DATEV2;
}
} else {
this.type = Type.DATETIME;
this.type = Type.DATETIMEV2_WITH_MAX_SCALAR;
}
} else if (TIME_FUNCTIONS_WITH_PRECISION.contains(fnName.getFunction().toLowerCase())
&& fn.getReturnType().isDatetimeV2()) {
Expand Down
7 changes: 5 additions & 2 deletions .../src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ public class StrToDate extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable {

public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT,
FunctionSignature.ret(DateTimeV2Type.MAX).args(VarcharType.SYSTEM_DEFAULT,
VarcharType.SYSTEM_DEFAULT),
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(StringType.INSTANCE, StringType.INSTANCE)
FunctionSignature.ret(DateTimeV2Type.MAX).args(StringType.INSTANCE, StringType.INSTANCE)
);

/**
Expand Down Expand Up @@ -95,6 +95,9 @@ public FunctionSignature computeSignature(FunctionSignature signature) {
}
} else {
returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME));
if (returnType.isDateTimeV2Type()) {
returnType = DataType.fromCatalogType(Type.DATETIMEV2_WITH_MAX_SCALAR);
}
}
return signature.withReturnType(returnType);
}
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ public static StringLiteral dateFormat(LiteralExpr date, StringLiteral fmtLitera
return new StringLiteral(result);
}

@FEFunction(name = "str_to_date", argTypes = { "VARCHAR", "VARCHAR" }, returnType = "DATETIME")
@FEFunction(name = "str_to_date", argTypes = { "VARCHAR", "VARCHAR" }, returnType = "DATETIMEV2")
public static DateLiteral dateParse(StringLiteral date, StringLiteral fmtLiteral) throws AnalysisException {
DateLiteral dateLiteral = new DateLiteral();
try {
Expand Down
4 changes: 2 additions & 2 deletions regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,10 +273,10 @@ February
2014-12-21T12:34:56

-- !sql --
2014-12-21T12:34:56
2014-12-21T12:34:56.789

-- !sql --
2023-07-05T02:09:55
2023-07-05T02:09:55.880

-- !sql --
2004-10-18
Expand Down