From 978189211c13c81fe7569add4405d654bf5eebef Mon Sep 17 00:00:00 2001 From: jacktengg Date: Thu, 3 Sep 2026 18:18:59 +0800 Subject: [PATCH] [fix](function) Preserve subsecond precision in convert_tz DST gaps ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: CONVERT_TZ mapped nonexistent local times in daylight-saving gaps to the transition instant while explicitly discarding the DateTimeV2 fractional component. This made BE execution inconsistent with timezone-bearing casts, and FE constant folding had the same precision loss. Preserve and round the fractional component consistently in BE and FE. Because preserving fractions introduces resets at civil-second boundaries inside a gap, also disable the monotonic partition-pruning shortcut for affected positive-scale ranges. ### Release note CONVERT_TZ now preserves DateTimeV2 fractional seconds when a source local time falls in a daylight-saving gap. ### Check List (For Author) - Test: Unit Test and Regression test - BE unit test: VTimestampFunctionsTest.convert_tz_test - FE unit tests: DateTimeExtractAndTransformTest, ConvertTzTest - Regression test: query_p0/sql_functions/datetime_functions/test_convert_tz - Behavior changed: Yes. CONVERT_TZ preserves fractional seconds in DST gaps. - Does this need documentation: No --- be/src/exprs/function/function_convert_tz.cpp | 21 +++----- be/test/exprs/function/function_time_test.cpp | 5 +- .../functions/scalar/ConvertTz.java | 52 ++++++++++++++++--- .../expressions/literal/DateTimeLiteral.java | 9 ++-- .../DateTimeExtractAndTransformTest.java | 9 +++- .../functions/scalar/ConvertTzTest.java | 18 +++++++ .../datetime_functions/test_convert_tz.out | 8 ++- .../datetime_functions/test_convert_tz.groovy | 19 +++++++ 8 files changed, 111 insertions(+), 30 deletions(-) diff --git a/be/src/exprs/function/function_convert_tz.cpp b/be/src/exprs/function/function_convert_tz.cpp index aa2966cbc3fd44..1c6932e2b66525 100644 --- a/be/src/exprs/function/function_convert_tz.cpp +++ b/be/src/exprs/function/function_convert_tz.cpp @@ -212,19 +212,6 @@ class FunctionConvertTZ : public IFunction { } } - static std::pair unix_timestamp_for_convert_tz( - const DateValueType& ts_value, const cctz::time_zone& from_tz) { - cctz::civil_second civil_time(ts_value.year(), ts_value.month(), ts_value.day(), - ts_value.hour(), ts_value.minute(), ts_value.second()); - const auto lookup = from_tz.lookup(civil_time); - const bool skipped = lookup.kind == cctz::time_zone::civil_lookup::SKIPPED; - const auto tp = skipped ? lookup.trans : lookup.pre; - - // Skipped civil times map to the transition instant. Do not keep the - // input fractional part inside a local time interval that never existed. - return {tp.time_since_epoch().count(), skipped ? 0 : ts_value.microsecond()}; - } - static void execute_tz_const_with_state(ConvertTzState* convert_tz_state, const ColumnType* date_column, ColumnType* result_column, NullMap& result_null_map, @@ -252,7 +239,9 @@ class FunctionConvertTZ : public IFunction { DateValueType ts_value = date_column->get_element(i); DateValueType ts_value2; - ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz); + std::pair timestamp; + ts_value.unix_timestamp(×tamp, from_tz); + ts_value2.from_unixtime(timestamp, to_tz); if (!ts_value2.is_valid_date()) [[unlikely]] { throw_out_of_bound_convert_tz(date_column->get_element(i), @@ -303,7 +292,9 @@ class FunctionConvertTZ : public IFunction { to_tz_name); } - ts_value2.from_unixtime(unix_timestamp_for_convert_tz(ts_value, from_tz), to_tz); + std::pair timestamp; + ts_value.unix_timestamp(×tamp, from_tz); + ts_value2.from_unixtime(timestamp, to_tz); if (!ts_value2.is_valid_date()) [[unlikely]] { throw_out_of_bound_convert_tz(date_column->get_element(index_now), diff --git a/be/test/exprs/function/function_time_test.cpp b/be/test/exprs/function/function_time_test.cpp index 0ddc2771a59e6b..06e4725f3ece9b 100644 --- a/be/test/exprs/function/function_time_test.cpp +++ b/be/test/exprs/function/function_time_test.cpp @@ -332,7 +332,10 @@ TEST(VTimestampFunctionsTest, convert_tz_test) { PrimitiveType::TYPE_VARCHAR}; DataSet data_set = {{{std::string {"2021-03-28 02:15:30.123456"}, std::string {"Europe/Paris"}, std::string {"UTC"}}, - std::string("2021-03-28 01:00:00.000000")}, + std::string("2021-03-28 01:00:00.123456")}, + {{std::string {"2024-03-10 02:30:00.123457"}, + std::string {"America/New_York"}, std::string {"+00:00"}}, + std::string("2024-03-10 07:00:00.123457")}, {{std::string {"2021-03-28 03:00:30.123456"}, std::string {"Europe/Paris"}, std::string {"UTC"}}, std::string("2021-03-28 01:00:30.123456")}, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java index 195ea273af90b8..253dfeccc18147 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTz.java @@ -44,6 +44,8 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneRules; import java.util.List; /** @@ -107,11 +109,8 @@ public boolean isMonotonic(Literal lower, Literal upper) { if (fromZone == null || toZone == null) { return false; } - if (toZone.getRules().isFixedOffset()) { - return true; - } if (lower == null || upper == null) { - return false; + return toZone.getRules().isFixedOffset() && !mayHaveFractionalSecondSourceGap(fromZone); } LocalDateTime lowerDateTime = toLocalDateTime(lower); LocalDateTime upperDateTime = toLocalDateTime(upper); @@ -124,15 +123,24 @@ public boolean isMonotonic(Literal lower, Literal upper) { if (upperDateTime.isBefore(lowerDateTime)) { return false; } + if (mayHaveFractionalSecondSourceGap(fromZone) + && hasGapResetInRange(fromZone, lowerDateTime, upperDateTime)) { + return false; + } + if (toZone.getRules().isFixedOffset()) { + return true; + } /* * convert_tz can be treated as a composition of two mappings: * * source local time x -> instant by from_tz -> target local time by to_tz. * - * After PR #64029, the first mapping is monotonic non-decreasing. A spring gap in from_tz - * flattens skipped local times to the transition instant, and a fall-back overlap uses the - * pre-transition offset before jumping forward at the overlap end. Neither case makes the - * instant move backward as x increases. + * For whole-second values, the first mapping is monotonic non-decreasing. A spring gap in + * from_tz flattens skipped local times to the transition instant, and a fall-back overlap + * uses the pre-transition offset before jumping forward at the overlap end. For fractional + * values, each skipped civil second preserves its fraction, so crossing an integer-second + * boundary inside the gap resets that fraction and makes the mapping non-monotonic. That + * case is rejected above. * * The second mapping, instant -> to_tz local time, is also monotonic non-decreasing except * at a to_tz fall-back transition, where the displayed local time jumps backward. Therefore @@ -150,6 +158,34 @@ public boolean isMonotonic(Literal lower, Literal upper) { return !DateUtils.hasFallbackTransitionInInstantRange(toZone, lowerInstant, upperInstant); } + private boolean mayHaveFractionalSecondSourceGap(ZoneId fromZone) { + return child(0).getDataType() instanceof DateTimeV2Type + && ((DateTimeV2Type) child(0).getDataType()).getScale() > 0 + && !fromZone.getRules().isFixedOffset(); + } + + private boolean hasGapResetInRange(ZoneId fromZone, LocalDateTime lower, LocalDateTime upper) { + ZoneRules rules = fromZone.getRules(); + Instant lowerInstant = DateTimeLiteral.convertLocalToInstant(lower, fromZone); + ZoneOffsetTransition transition = rules.getTransition(lower); + if (transition == null) { + transition = rules.nextTransition(lowerInstant.minusNanos(1)); + } + while (transition != null && !transition.getDateTimeBefore().isAfter(upper)) { + if (transition.isGap()) { + LocalDateTime firstReset = transition.getDateTimeBefore().plusSeconds(1); + LocalDateTime lastReset = transition.getDateTimeAfter(); + LocalDateTime nextReset = lower.isBefore(firstReset) + ? firstReset : lower.withNano(0).plusSeconds(1); + if (!nextReset.isAfter(upper) && !nextReset.isAfter(lastReset)) { + return true; + } + } + transition = rules.nextTransition(transition.getInstant()); + } + return false; + } + @Override public boolean isPositive() { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index c50f98b45a1119..5e2f61d485db4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -283,8 +283,8 @@ private static LocalDateTime convertTimeZone(long year, long month, long day, lo * *

For normal local times, there is one valid offset. For fall-back overlap times, two offsets * are valid and the first one is the pre-transition offset. For spring-forward gap times, the - * local time does not exist, so any value inside the skipped interval maps to the transition - * instant. + * local time does not exist, so the whole-second part maps to the transition instant while the + * fractional-second part is preserved. */ public static Instant convertLocalToInstant(LocalDateTime localDateTime, ZoneId fromZone) { ZoneRules rules = fromZone.getRules(); @@ -295,9 +295,10 @@ public static Instant convertLocalToInstant(LocalDateTime localDateTime, ZoneId if (size == 1 || size == 2) { return localDateTime.atOffset(validOffsets.get(0)).toInstant(); } - // Skipped local time maps to the transition instant, e.g. 2021-03-28 02:15 Europe/Paris. + // Match BE DateV2Value::unix_timestamp by preserving the sub-second part separately when + // the civil second maps to the transition instant. ZoneOffsetTransition transition = rules.getTransition(localDateTime); - return transition.getInstant(); + return transition.getInstant().plusNanos(localDateTime.getNano()); } public boolean checkRange() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java index f32ca713dcd9cf..b606d9c161c57c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/DateTimeExtractAndTransformTest.java @@ -170,7 +170,7 @@ public void testFromUnixTimeOutOfRangeThrows() { @Test void testConvertTzDstTransition() { - // Spring gap maps skipped local times to the transition instant. + // Spring gap maps the whole-second part to the transition instant. Assertions.assertEquals( new DateTimeV2Literal("2021-03-28 01:00:00"), DateTimeExtractAndTransform.convertTz( @@ -189,6 +189,13 @@ void testConvertTzDstTransition() { new DateTimeV2Literal("2021-03-28 03:00:00"), new VarcharLiteral("Europe/Paris"), new VarcharLiteral("UTC"))); + // The fractional-second part is preserved after the whole-second part is normalized. + Assertions.assertEquals( + new DateTimeV2Literal(DateTimeV2Type.of(6), "2024-03-10 07:00:00.123457"), + DateTimeExtractAndTransform.convertTz( + new DateTimeV2Literal(DateTimeV2Type.of(6), "2024-03-10 02:30:00.123456789"), + new VarcharLiteral("America/New_York"), + new VarcharLiteral("+00:00"))); // Fall overlap uses the pre-transition offset. Assertions.assertEquals( new DateTimeV2Literal("2021-10-31 00:15:00"), diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTzTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTzTest.java index f280d831a50d68..3bf4873d95e797 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTzTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConvertTzTest.java @@ -19,8 +19,10 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.DateTimeType; +import org.apache.doris.nereids.types.DateTimeV2Type; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -161,6 +163,22 @@ void testIsMonotonicWithDstGapInSourceZoneOverlap() { new DateTimeLiteral("2021-03-28 04:00:00"))); } + @Test + void testIsNotMonotonicAcrossDstGapForFractionalValues() { + ConvertTz convertTz = new ConvertTz(new SlotReference("ts", DateTimeV2Type.of(6)), + new VarcharLiteral("Europe/Paris"), new VarcharLiteral("UTC")); + + Assertions.assertFalse(convertTz.isMonotonic( + new DateTimeV2Literal(DateTimeV2Type.of(6), "2021-03-28 02:00:00.000000"), + new DateTimeV2Literal(DateTimeV2Type.of(6), "2021-03-28 03:00:00.000000"))); + Assertions.assertFalse(convertTz.isMonotonic( + null, + new DateTimeV2Literal(DateTimeV2Type.of(6), "2021-03-28 03:00:00.000000"))); + Assertions.assertTrue(convertTz.isMonotonic( + new DateTimeV2Literal(DateTimeV2Type.of(6), "2021-03-28 02:00:00.100000"), + new DateTimeV2Literal(DateTimeV2Type.of(6), "2021-03-28 02:00:00.900000"))); + } + @Test void testIsMonotonicWithDstFallbackInSourceZone() { ConvertTz convertTz = new ConvertTz(timestampSlot, diff --git a/regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out b/regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out index 643540d9c7bec4..5126ddc8e055cf 100644 --- a/regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out +++ b/regression-test/data/query_p0/sql_functions/datetime_functions/test_convert_tz.out @@ -22,5 +22,11 @@ 2024-04-18T23:20 -- !spring_gp_with_micro_sec -- -2021-03-28T01:00 +2021-03-28T01:00:00.003230 + +-- !spring_gap_preserve_fraction_be -- +2024-03-10T07:00:00.123457 2024-03-10T07:00:00.123457 + +-- !spring_gap_preserve_fraction_fe -- +2024-03-10T07:00:00.123457 diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy index c9f37b37b4f2ba..d87ab8524184b0 100644 --- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy +++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_convert_tz.groovy @@ -37,4 +37,23 @@ suite("test_convert_tz") { """ sql "set debug_skip_fold_constant=true;" qt_spring_gp_with_micro_sec "select convert_tz('2021-03-28 02:30:00.00323', 'Europe/Paris','UTC');" + sql "set time_zone='+00:00';" + qt_spring_gap_preserve_fraction_be """ + SELECT + CAST('2024-03-10 02:30:00.123456789America/New_York' AS datetimev2(6)), + CONVERT_TZ( + CAST('2024-03-10 02:30:00.123456789' AS datetimev2(6)), + 'America/New_York', + '+00:00' + ); + """ + + sql "set debug_skip_fold_constant=false;" + qt_spring_gap_preserve_fraction_fe """ + SELECT CONVERT_TZ( + CAST('2024-03-10 02:30:00.123456789' AS datetimev2(6)), + 'America/New_York', + '+00:00' + ); + """ }