From fe863a453bf29c136e688c3f4c0e7762fd267e66 Mon Sep 17 00:00:00 2001 From: Aleksandar Trajkovic Date: Tue, 15 Sep 2026 09:14:12 +0000 Subject: [PATCH] [SPARK-58876][SQL][FOLLOWUP] Map sub-microsecond Oracle TIMESTAMP to nanos NTZ Oracle DATE/TIMESTAMP mapping short-circuited the shared TIMESTAMP resolver, regressing TIMESTAMP(7-9) to microsecond TimestampNTZType under the nanos preview. Extract the micro-vs-nanos decision into JdbcUtils.resolveTimestampType and route Oracle's NTZ mapping through it (forcing NTZ). Co-authored-by: Isaac --- .../datasources/jdbc/JdbcUtils.scala | 23 +++++++----- .../apache/spark/sql/jdbc/OracleDialect.scala | 10 +++-- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 37 ++++++++++++++----- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala index 39c606b0ce1d8..0f630b8cc1719 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala @@ -193,6 +193,19 @@ object JdbcUtils extends Logging with SQLConfHelper { if (isTimestampNTZ) TimestampNTZNanosType(precision) else TimestampLTZNanosType(precision) } + // When nanosecond timestamps are requested (and the preview feature is enabled), a driver + // TIMESTAMP that reports a sub-microsecond fractional-second scale (7-9) is mapped to the + // nanosecond-capable type. Otherwise the historical microsecond mapping is preserved. + def resolveTimestampType( + isTimestampNTZ: Boolean, scale: Int, preferTimestampNanos: Boolean): DataType = { + if (preferTimestampNanos && + scale >= TimestampNTZNanosType.MIN_PRECISION && + scale <= TimestampNTZNanosType.MAX_PRECISION && + conf.timestampNanosTypesEnabled) { + getTimestampNanosType(isTimestampNTZ, scale) + } else getTimestampType(isTimestampNTZ) + } + /** * Maps a JDBC type to a Catalyst type. This function is called only when * the JdbcDialect class corresponding to your database driver returns null. @@ -251,15 +264,7 @@ object JdbcUtils extends Logging with SQLConfHelper { TimeType(timePrecision) } else getTimestampType(isTimestampNTZ) case java.sql.Types.TIMESTAMP => - // When nanosecond timestamps are requested (and the preview feature is enabled), a driver - // TIMESTAMP that reports a sub-microsecond fractional-second scale (7-9) is mapped to the - // nanosecond-capable type. Otherwise the historical microsecond mapping is preserved. - if (preferTimestampNanos && - scale >= TimestampNTZNanosType.MIN_PRECISION && - scale <= TimestampNTZNanosType.MAX_PRECISION && - conf.timestampNanosTypesEnabled) { - getTimestampNanosType(isTimestampNTZ, scale) - } else getTimestampType(isTimestampNTZ) + resolveTimestampType(isTimestampNTZ, scale, preferTimestampNanos) case java.sql.Types.TINYINT => IntegerType case java.sql.Types.VARBINARY => BinaryType case java.sql.Types.VARCHAR diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala index 856808258d1f0..678d425ec5b92 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala @@ -187,9 +187,13 @@ private case class OracleDialect() extends JdbcDialect with SQLConfHelper with N // Oracle DATE and TIMESTAMP are zoneless; map to NTZ and mark it wall-clock so a later flag // flip can't desync the read. TZ/LTZ variants are handled above. if (md != null) md.putBoolean(JdbcUtils.READ_TIMESTAMP_NTZ_WALL_CLOCK, value = true) - // TODO: map sub-microsecond TIMESTAMP(7-9) to TimestampNTZNanosType when the nanosecond - // timestamp preview is enabled, instead of truncating to microsecond TimestampNTZType. - Some(TimestampNTZType) + val metadata = if (md != null) md.build() else Metadata.empty + // Absent scale metadata: Oracle TIMESTAMP defaults to TIMESTAMP(6). + val scale = if (metadata.contains("scale")) metadata.getLong("scale").toInt else 6 + val preferNanos = metadata.contains("preferTimestampNanos") && + metadata.getBoolean("preferTimestampNanos") + Some(JdbcUtils.resolveTimestampType( + isTimestampNTZ = true, scale = scale, preferTimestampNanos = preferNanos)) case _ => None } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 77e2ce82d34f7..ece209d2f7033 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -1901,16 +1901,35 @@ class JDBCSuite extends SharedSparkSession { "{ts '2018-07-06 06:00:00.0'}") } - test("SPARK-58876: Oracle TIMESTAMP stays microsecond TimestampNTZType under the nanos preview") { - val oracleDialect = JdbcDialects.get("jdbc:oracle") - // Even with the nanosecond timestamp preview enabled, the Oracle mapping is microsecond - // TimestampNTZType and does not engage that preview (no nanosecond type, no deferral). - withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") { - val md = new MetadataBuilder() - .putBoolean("preferTimestampNanos", value = true).putLong("scale", 9) - assert(oracleDialect.getCatalystType(java.sql.Types.TIMESTAMP, "TIMESTAMP", 0, md) === - Some(TimestampNTZType)) + test("SPARK-58876: Oracle TIMESTAMP(7-9) resolves to nanosecond NTZ under the nanos preview") { + // scale/preferTimestampNanos reach the dialect only as metadata getSchema stamps, so resolve a + // mocked Oracle TIMESTAMP column via getSchema for each (scale, option, preview) combination. + def resolve(scale: Int, preferNanos: Boolean, nanosEnabled: Boolean): DataType = { + val rsmd = mock(classOf[java.sql.ResultSetMetaData]) + when(rsmd.getColumnCount).thenReturn(1) + when(rsmd.getColumnLabel(anyInt())).thenReturn("T") + when(rsmd.getColumnType(anyInt())).thenReturn(java.sql.Types.TIMESTAMP) + when(rsmd.getColumnTypeName(anyInt())).thenReturn("TIMESTAMP") + when(rsmd.getPrecision(anyInt())).thenReturn(0) + when(rsmd.getScale(anyInt())).thenReturn(scale) + when(rsmd.isSigned(anyInt())).thenReturn(false) + when(rsmd.isNullable(anyInt())).thenReturn(java.sql.ResultSetMetaData.columnNullable) + val rs = mock(classOf[ResultSet]) + when(rs.getMetaData).thenReturn(rsmd) + withSQLConf(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> nanosEnabled.toString) { + JdbcUtils.getSchema(mock(classOf[Connection]), rs, OracleDialect(), + preferTimestampNanos = preferNanos).fields.head.dataType + } + } + // Sub-microsecond scales (7-9) widen to the nanosecond NTZ type only when both the read option + // and the preview are on; every coarser scale and either flag off stays microsecond NTZ. + (TimestampNTZNanosType.MIN_PRECISION to TimestampNTZNanosType.MAX_PRECISION).foreach { s => + assert(resolve(s, preferNanos = true, nanosEnabled = true) === TimestampNTZNanosType(s), + s"scale=$s") } + assert(resolve(6, preferNanos = true, nanosEnabled = true) === TimestampNTZType) + assert(resolve(9, preferNanos = false, nanosEnabled = true) === TimestampNTZType) + assert(resolve(9, preferNanos = true, nanosEnabled = false) === TimestampNTZType) } test("SPARK-42469: OracleDialect Limit query test") {