From a887626ee007658755efdb3ecdebd8783b5e6f84 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sat, 14 Nov 2026 05:43:08 +0000 Subject: [PATCH 1/5] Fix Arrow TIME results on Thrift and SEA Handle TIME values and metadata on both Arrow scanners, preserving precision, NULLs, and clock-only nested JSON. Cover database/sql destinations and explicitly opt-in live beta checks. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .github/workflows/nightly-e2e.yml | 8 +- CHANGELOG.md | 1 + README.md | 17 +++ internal/arrowscan/arrowscan.go | 8 ++ internal/arrowscan/arrowscan_test.go | 4 +- internal/arrowscan/coltype.go | 3 + internal/arrowscan/time.go | 16 +++ internal/arrowscan/time_test.go | 39 +++++++ internal/rows/arrowbased/arrowRows.go | 2 + internal/rows/arrowbased/columnValues.go | 27 +++++ internal/rows/arrowbased/time_bench_test.go | 60 ++++++++++ internal/rows/arrowbased/time_test.go | 113 ++++++++++++++++++ internal/rows/rows.go | 16 ++- internal/rows/time_test.go | 74 ++++++++++++ kernel_time_e2e_test.go | 15 +++ pecotesting_creds_test.go | 12 +- time_capability_test.go | 52 +++++++++ time_e2e_test.go | 71 ++++++++++++ time_test.go | 120 ++++++++++++++++++++ 19 files changed, 647 insertions(+), 11 deletions(-) create mode 100644 internal/arrowscan/time.go create mode 100644 internal/arrowscan/time_test.go create mode 100644 internal/rows/arrowbased/time_bench_test.go create mode 100644 internal/rows/arrowbased/time_test.go create mode 100644 internal/rows/time_test.go create mode 100644 kernel_time_e2e_test.go create mode 100644 time_capability_test.go create mode 100644 time_e2e_test.go create mode 100644 time_test.go diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index 41b8e1db..e9177905 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -89,8 +89,8 @@ jobs: DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} run: | - go test -count=1 -v -timeout 15m \ - -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads|TestE2EDecimalMetadata' . + go test -tags e2e -count=1 -v -timeout 15m \ + -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads|TestE2EDecimalMetadata|TestE2ETimeResults' . kernel-e2e: name: E2E (kernel backend) @@ -243,7 +243,7 @@ jobs: DATABRICKS_PECOTESTING_TOKEN: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN }} DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} run: | - CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ + CGO_ENABLED=1 go test -tags databricks_kernel,e2e -count=1 -v -timeout 30m \ -run 'TestKernelE2E|TestKernelThriftParity|TestKernelThriftColumnTypeParity|TestKernelParamsVsThrift' \ -skip 'TestKernelE2EM2M' . @@ -265,6 +265,6 @@ jobs: TEST_WAREHOUSE: reyden DATABRICKS_REYDEN_HTTP_PATH: ${{ secrets.TEST_REYDEN_WAREHOUSE_HTTP_PATH }} run: | - CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ + CGO_ENABLED=1 go test -tags databricks_kernel,e2e -count=1 -v -timeout 30m \ -run 'TestKernelE2E' \ -skip 'TestKernelE2EM2M' . diff --git a/CHANGELOG.md b/CHANGELOG.md index 77a60de2..d782a1af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162). - **Behavior change:** Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results. Tools such as ORM migrators can now act on these qualifiers (databricks/databricks-sql-go#500). +- Support Arrow TIME results on Thrift and SEA, preserving precision as UTC `time.Time` values on `0001-01-01`, with `TIME` column metadata and clock-only nested JSON values. - **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication. - Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439). - **Behavior change:** On the Thrift backend, `ExecuteStatement` is now retried (per `WithRetries`) when the connection to the server or proxy can't be opened, for example connection refused or a connect timeout, because the statement was never sent. Failures after the connection exists are still not retried, so a statement never runs twice. With default retry settings, a refused connection now fails after about 15 seconds instead of immediately (databricks/databricks-sql-go#423). diff --git a/README.md b/README.md index 83207b27..38c6fc50 100644 --- a/README.md +++ b/README.md @@ -369,6 +369,23 @@ STRUCT and VARIANT (as JSON) are supported. BINARY is returned as `sql.RawBytes` Both backends return GEOMETRY / GEOGRAPHY as WKT / EWKT by default. Set `enableGeoSpatialSupport=true` on the kernel path to opt into EWKB. +SQL TIME values in Arrow results scan into `time.Time` (or `any`) on the reference date +`0001-01-01` in UTC, preserving the Arrow second/millisecond/microsecond/nanosecond +precision. The date and location are carriers only; session time zones do not +shift the clock. Scanning into `string` uses `database/sql`'s RFC3339Nano format +(for example, `0001-01-01T09:15:00.123456Z`). NULL scans into `any` as `nil`; +use `sql.NullTime` for nullable typed values. Column metadata reports `TIME` and +`time.Time`. Nested TIME values in ARRAY / MAP / STRUCT JSON use clock-only strings +such as `"09:15:00.123456"`. +On Thrift, TIME column metadata requires a server-supplied Arrow schema; +without it, the legacy Thrift column metadata is retained. + +Live TIME tests require the `e2e` build tag: `go test -tags e2e -run '^TestE2ETimeResults$' .` +or `go test -tags databricks_kernel,e2e -run '^TestKernelE2ETimeResults$' .`. +They require the `DATABRICKS_PECOTESTING_*` credentials and skip only when the server +reports that TIME is disabled; a workspace with the TIME beta enabled is needed +to verify live result conversion. + On Thrift, TIMESTAMP_NTZ handling requires native timestamp fields in the server-supplied Arrow schema. Without that schema, NTZ still reports `TIMESTAMP` and uses session-location handling. String timestamps (`UseArrowNativeTimestamp=false`, diff --git a/internal/arrowscan/arrowscan.go b/internal/arrowscan/arrowscan.go index e3c43920..1a00f33e 100644 --- a/internal/arrowscan/arrowscan.go +++ b/internal/arrowscan/arrowscan.go @@ -210,6 +210,10 @@ func scanCell(col arrow.Array, row int, loc *time.Location, keys *StructKeyCache return inLocation(c.Value(row).ToTime(), loc), nil case *array.Date64: return inLocation(c.Value(row).ToTime(), loc), nil + case *array.Time32: + return timeOfDayValue(int64(c.Value(row)), c.DataType().(*arrow.Time32Type).Unit), nil + case *array.Time64: + return timeOfDayValue(int64(c.Value(row)), c.DataType().(*arrow.Time64Type).Unit), nil case *array.Timestamp: dt, ok := col.DataType().(*arrow.TimestampType) if !ok { @@ -641,6 +645,10 @@ func scalarForJSON(col arrow.Array, row int, loc *time.Location) (any, error) { return c.Value(row), nil case *array.LargeString: return c.Value(row), nil + case *array.Time32: + return timeOfDay(int64(c.Value(row)), c.DataType().(*arrow.Time32Type).Unit).Format("15:04:05.999999999"), nil + case *array.Time64: + return timeOfDay(int64(c.Value(row)), c.DataType().(*arrow.Time64Type).Unit).Format("15:04:05.999999999"), nil } // scanCell, not ScanCell: keyLoc already came from childLocation on the key // Field, so the field-less zone-less fallback must not override it. diff --git a/internal/arrowscan/arrowscan_test.go b/internal/arrowscan/arrowscan_test.go index 137204ea..c43621f5 100644 --- a/internal/arrowscan/arrowscan_test.go +++ b/internal/arrowscan/arrowscan_test.go @@ -106,9 +106,9 @@ func TestScanCellScalars(t *testing.T) { // An unhandled arrow type must error, not return a silently wrong value. // Duration/MonthInterval are now handled (see TestScanCellInterval); use a // type with no scan arm. - b := array.NewTime32Builder(pool, &arrow.Time32Type{Unit: arrow.Second}) + b := array.NewFixedSizeBinaryBuilder(pool, &arrow.FixedSizeBinaryType{ByteWidth: 1}) defer b.Release() - b.Append(1000) + b.Append([]byte{1}) arr := b.NewArray() defer arr.Release() if _, err := ScanCell(arr, 0, nil); err == nil { diff --git a/internal/arrowscan/coltype.go b/internal/arrowscan/coltype.go index 6e0c9d91..5dc4382d 100644 --- a/internal/arrowscan/coltype.go +++ b/internal/arrowscan/coltype.go @@ -41,6 +41,7 @@ var ( scanTypeFloat64 = reflect.TypeOf(float64(0)) scanTypeString = reflect.TypeOf("") scanTypeDateTime = reflect.TypeOf(time.Time{}) + scanTypeTime = reflect.TypeOf(timeOfDayValue(0, arrow.Nanosecond)) scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) scanTypeUnknown = reflect.TypeOf(new(any)) // *interface{}, as on the Thrift path ) @@ -77,6 +78,8 @@ func ColumnTypeInfoFor(dt arrow.DataType) ColumnTypeInfo { return varLen("BINARY", scanTypeRawBytes) case arrow.DATE32, arrow.DATE64: return ColumnTypeInfo{DatabaseTypeName: "DATE", ScanType: scanTypeDateTime} + case arrow.TIME32, arrow.TIME64: + return ColumnTypeInfo{DatabaseTypeName: "TIME", ScanType: scanTypeTime} case arrow.TIMESTAMP: // TIMESTAMP and TIMESTAMP_NTZ both arrive as Arrow timestamp[us]. NTZ is // decided by IsTimestampNTZField: the field's logical-type metadata first diff --git a/internal/arrowscan/time.go b/internal/arrowscan/time.go new file mode 100644 index 00000000..e630a2f2 --- /dev/null +++ b/internal/arrowscan/time.go @@ -0,0 +1,16 @@ +package arrowscan + +import ( + "database/sql/driver" + "time" + + "github.com/apache/arrow-go/v18/arrow" +) + +func timeOfDayValue(value int64, unit arrow.TimeUnit) driver.Value { + return timeOfDay(value, unit) +} + +func timeOfDay(value int64, unit arrow.TimeUnit) time.Time { + return time.Time{}.Add(time.Duration(value) * unit.Multiplier()) +} diff --git a/internal/arrowscan/time_test.go b/internal/arrowscan/time_test.go new file mode 100644 index 00000000..a61abf41 --- /dev/null +++ b/internal/arrowscan/time_test.go @@ -0,0 +1,39 @@ +package arrowscan + +import ( + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/stretchr/testify/require" +) + +func TestScanTimeUnits(t *testing.T) { + for _, test := range []struct { + dataType arrow.DataType + clock string + nanos int + }{ + {arrow.FixedWidthTypes.Time32s, "12:34:56", 0}, + {arrow.FixedWidthTypes.Time32ms, "12:34:56.789", 789000000}, + {arrow.FixedWidthTypes.Time64us, "12:34:56.789123", 789123000}, + {arrow.FixedWidthTypes.Time64ns, "12:34:56.789123456", 789123456}, + } { + t.Run(test.dataType.String(), func(t *testing.T) { + builder := array.NewBuilder(memory.DefaultAllocator, test.dataType) + defer builder.Release() + require.NoError(t, builder.AppendValueFromString(test.clock)) + builder.AppendNull() + values := builder.NewArray() + defer values.Release() + got, err := ScanCell(values, 0, time.FixedZone("east", 9*3600)) + require.NoError(t, err) + require.Equal(t, time.Date(1, time.January, 1, 12, 34, 56, test.nanos, time.UTC), got) + got, err = ScanCell(values, 1, nil) + require.NoError(t, err) + require.Nil(t, got) + }) + } +} diff --git a/internal/rows/arrowbased/arrowRows.go b/internal/rows/arrowbased/arrowRows.go index d50e5ed1..dcdf08b4 100644 --- a/internal/rows/arrowbased/arrowRows.go +++ b/internal/rows/arrowbased/arrowRows.go @@ -657,6 +657,8 @@ func (vcm *arrowValueContainerMaker) makeFieldValueContainer(field arrow.Field, case *arrow.Date32Type: return &dateValueContainer{location: location}, nil + case *arrow.Time32Type, *arrow.Time64Type: + return &timeValueContainer{}, nil case *arrow.TimestampType: // A TIMESTAMP_NTZ carries a wall clock that must not be shifted into the diff --git a/internal/rows/arrowbased/columnValues.go b/internal/rows/arrowbased/columnValues.go index ef1890a5..512fabf2 100644 --- a/internal/rows/arrowbased/columnValues.go +++ b/internal/rows/arrowbased/columnValues.go @@ -6,6 +6,7 @@ import ( "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" + "github.com/databricks/databricks-sql-go/internal/arrowscan" "github.com/databricks/databricks-sql-go/internal/decimalfmt" "github.com/databricks/databricks-sql-go/internal/jsonfmt" "github.com/databricks/databricks-sql-go/internal/rows/rowscanner" @@ -476,6 +477,29 @@ func (dvc *dateValueContainer) SetValueArray(colData arrow.ArrayData) error { return nil } +type timeValueContainer struct { + timeArray arrow.Array +} + +func (container *timeValueContainer) Value(row int) (any, error) { + return arrowscan.ScanCell(container.timeArray, row, nil) +} + +func (container *timeValueContainer) IsNull(row int) bool { + return container.timeArray.IsNull(row) +} + +func (container *timeValueContainer) Release() { + if container.timeArray != nil { + container.timeArray.Release() + } +} + +func (container *timeValueContainer) SetValueArray(data arrow.ArrayData) error { + container.timeArray = array.MakeFromData(data) + return nil +} + type timestampValueContainer struct { timeArray *array.Timestamp location *time.Location @@ -603,6 +627,9 @@ func marshalScalar(dst []byte, holder columnValues, i int, val any) ([]byte, err return append(dst, d.ValueString(i)...), nil } if t, ok := val.(time.Time); ok { + if _, clock := holder.(*timeValueContainer); clock { + return jsonfmt.Append(dst, t.Format("15:04:05.999999999")) + } dst = append(dst, '"') dst = append(dst, t.String()...) return append(dst, '"'), nil diff --git a/internal/rows/arrowbased/time_bench_test.go b/internal/rows/arrowbased/time_bench_test.go new file mode 100644 index 00000000..fb4d652c --- /dev/null +++ b/internal/rows/arrowbased/time_bench_test.go @@ -0,0 +1,60 @@ +package arrowbased + +import ( + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + + "github.com/databricks/databricks-sql-go/internal/arrowscan" +) + +func BenchmarkScanExistingColumns(b *testing.B) { + for _, test := range []struct { + name string + dataType arrow.DataType + value string + }{ + {"int64", arrow.PrimitiveTypes.Int64, "123456789"}, + {"string", arrow.BinaryTypes.String, "existing column"}, + {"timestamp", arrow.FixedWidthTypes.Timestamp_us, "2026-11-14 09:15:00.123456Z"}, + } { + builder := array.NewBuilder(memory.DefaultAllocator, test.dataType) + if err := builder.AppendValueFromString(test.value); err != nil { + b.Fatal(err) + } + values := builder.NewArray() + builder.Release() + b.Run(test.name+"/thrift", func(b *testing.B) { + holder, err := (&arrowValueContainerMaker{}).makeColumnValueContainer(test.dataType, time.UTC, func(timestamp arrow.Timestamp) time.Time { + return timestamp.ToTime(arrow.Microsecond) + }, nil) + if err != nil { + b.Fatal(err) + } + if err := holder.SetValueArray(values.Data()); err != nil { + b.Fatal(err) + } + defer holder.Release() + b.ReportAllocs() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + if _, err := holder.Value(0); err != nil { + b.Fatal(err) + } + } + }) + b.Run(test.name+"/kernel", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for iteration := 0; iteration < b.N; iteration++ { + if _, err := arrowscan.ScanCell(values, 0, time.UTC); err != nil { + b.Fatal(err) + } + } + }) + values.Release() + } +} diff --git a/internal/rows/arrowbased/time_test.go b/internal/rows/arrowbased/time_test.go new file mode 100644 index 00000000..fc84299a --- /dev/null +++ b/internal/rows/arrowbased/time_test.go @@ -0,0 +1,113 @@ +package arrowbased + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/stretchr/testify/require" + + "github.com/databricks/databricks-sql-go/internal/arrowscan" +) + +func TestTimeColumns(t *testing.T) { + for _, test := range []struct { + dataType arrow.DataType + fraction string + nanos int + }{ + {arrow.FixedWidthTypes.Time32s, "", 0}, + {arrow.FixedWidthTypes.Time32ms, ".123", 123000000}, + {arrow.FixedWidthTypes.Time64us, ".123456", 123456000}, + {arrow.FixedWidthTypes.Time64ns, ".123456", 123456000}, + {arrow.FixedWidthTypes.Time64ns, ".123456789", 123456789}, + } { + t.Run(test.dataType.String()+test.fraction, func(t *testing.T) { + pool := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer pool.AssertSize(t, 0) + builder := array.NewBuilder(pool, test.dataType) + defer builder.Release() + for _, clock := range []string{"09:15:00" + test.fraction, "00:00:00", "23:59:59" + test.fraction} { + require.NoError(t, builder.AppendValueFromString(clock)) + } + builder.AppendNull() + values := builder.NewArray() + defer values.Release() + expected := []any{ + time.Date(1, time.January, 1, 9, 15, 0, test.nanos, time.UTC), + time.Time{}, + time.Date(1, time.January, 1, 23, 59, 59, test.nanos, time.UTC), + nil, + } + sliced := array.NewSlice(values, 1, 4) + defer sliced.Release() + for _, location := range []*time.Location{nil, time.UTC, time.FixedZone("west", -7*3600), time.FixedZone("east", 9*3600)} { + holder, err := (&arrowValueContainerMaker{}).makeColumnValueContainer(test.dataType, location, nil, nil) + require.NoError(t, err) + containers := &rowValues{columnValueHolders: []columnValues{holder}} + for _, batch := range []struct { + values arrow.Array + expected []any + }{{values, expected}, {sliced, expected[1:]}} { + require.NoError(t, containers.SetColumnValues(0, batch.values.Data())) + for row, want := range batch.expected { + if want == nil { + require.True(t, holder.IsNull(row)) + } else { + got, err := holder.Value(row) + require.NoError(t, err) + require.Equal(t, want, got) + } + got, err := arrowscan.ScanCell(batch.values, row, location) + require.NoError(t, err) + require.Equal(t, want, got) + } + } + containers.Close() + } + info := arrowscan.ColumnTypeInfoFor(test.dataType) + require.Equal(t, "TIME", info.DatabaseTypeName) + require.Equal(t, reflect.TypeOf(time.Time{}), info.ScanType) + + clock := "09:15:00" + test.fraction + for _, nested := range []struct { + dataType arrow.DataType + input string + expected []string + }{ + {arrow.ListOf(test.dataType), fmt.Sprintf(`[[%q,null],null]`, clock), []string{fmt.Sprintf(`[%q,null]`, clock), ""}}, + {arrow.StructOf(arrow.Field{Name: "clock", Type: test.dataType, Nullable: true}), fmt.Sprintf(`[{"clock":%q},{"clock":null}]`, clock), []string{fmt.Sprintf(`{"clock":%q}`, clock), `{"clock":null}`}}, + {arrow.MapOf(arrow.BinaryTypes.String, test.dataType), fmt.Sprintf(`[[{"key":"clock","value":%q},{"key":"missing","value":null}],null]`, clock), []string{fmt.Sprintf(`{"clock":%q,"missing":null}`, clock), ""}}, + {arrow.MapOf(test.dataType, arrow.BinaryTypes.String), fmt.Sprintf(`[[{"key":%q,"value":"clock"}],null]`, clock), []string{fmt.Sprintf(`{%q:"clock"}`, clock), ""}}, + } { + t.Run(nested.dataType.String(), func(t *testing.T) { + builder := array.NewBuilder(pool, nested.dataType) + defer builder.Release() + require.NoError(t, builder.UnmarshalJSON([]byte(nested.input))) + values := builder.NewArray() + defer values.Release() + holder, err := (&arrowValueContainerMaker{}).makeColumnValueContainer(nested.dataType, time.Local, nil, nil) + require.NoError(t, err) + defer holder.Release() + require.NoError(t, holder.SetValueArray(values.Data())) + for row := range values.Len() { + got, err := arrowscan.ScanCell(values, row, time.Local) + require.NoError(t, err) + if holder.IsNull(row) { + require.Nil(t, got) + } else { + want, err := holder.Value(row) + require.NoError(t, err) + require.JSONEq(t, nested.expected[row], got.(string)) + require.Equal(t, want, got) + } + } + }) + } + }) + } +} diff --git a/internal/rows/rows.go b/internal/rows/rows.go index cffc6367..c617e09e 100644 --- a/internal/rows/rows.go +++ b/internal/rows/rows.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/ipc" "github.com/databricks/databricks-sql-go/driverctx" dbsqlerr "github.com/databricks/databricks-sql-go/errors" @@ -410,6 +411,10 @@ func (r *rows) ColumnTypeScanType(index int) reflect.Type { return nil } + r.cacheDatabaseTypeNameOverrides() + if r.databaseTypeNameOverrides[index] == "TIME" { + return arrowscan.ColumnTypeInfoFor(arrow.FixedWidthTypes.Time64ns).ScanType + } scanType := getScanType(column) return scanType } @@ -436,8 +441,8 @@ func (r *rows) ColumnTypeDatabaseTypeName(index int) string { return dbtype } -// Thrift shares TIMESTAMP_TYPE between TIMESTAMP and TIMESTAMP_NTZ. Cache the -// distinction from server metadata so it survives result-page scanner changes. +// Thrift shares TIMESTAMP_TYPE between TIMESTAMP and TIMESTAMP_NTZ and has no +// TIME id. Cache the Arrow distinctions so they survive result-page scanner changes. // The override is only applied when the result set metadata indicates Arrow // decoding: columnar (TColumn) results keep legacy session-location timestamp // parsing, so they keep the legacy TIMESTAMP type name too. @@ -489,6 +494,9 @@ func (r *rows) buildDatabaseTypeNameOverrides() map[int]string { if arrowscan.IsTimestampNTZField(field) { overrides[i] = "TIMESTAMP_NTZ" } + if field.Type.ID() == arrow.TIME32 || field.Type.ID() == arrow.TIME64 { + overrides[i] = "TIME" + } } return overrides } @@ -522,6 +530,10 @@ func (r *rows) ColumnTypeLength(index int) (length int64, ok bool) { if err != nil { return 0, false } + r.cacheDatabaseTypeNameOverrides() + if r.databaseTypeNameOverrides[index] == "TIME" { + return 0, false + } return columnTypeLengthForID(rowscanner.GetDBTypeID(columnInfo)) } diff --git a/internal/rows/time_test.go b/internal/rows/time_test.go new file mode 100644 index 00000000..f0560c84 --- /dev/null +++ b/internal/rows/time_test.go @@ -0,0 +1,74 @@ +package rows + +import ( + "bytes" + "context" + "sync" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/stretchr/testify/require" + + "github.com/databricks/databricks-sql-go/internal/arrowscan" + "github.com/databricks/databricks-sql-go/internal/cli_service" + "github.com/databricks/databricks-sql-go/internal/client" + "github.com/databricks/databricks-sql-go/internal/config" +) + +func TestTimeColumnMetadata(t *testing.T) { + for _, dataType := range []arrow.DataType{arrow.FixedWidthTypes.Time32s, arrow.FixedWidthTypes.Time32ms, arrow.FixedWidthTypes.Time64us, arrow.FixedWidthTypes.Time64ns} { + t.Run(dataType.String(), func(t *testing.T) { + var schemaBytes bytes.Buffer + writer := ipc.NewWriter(&schemaBytes, ipc.WithSchema(arrow.NewSchema([]arrow.Field{{Name: "clock", Type: dataType, Nullable: true}}, nil))) + require.NoError(t, writer.Close()) + for _, format := range []cli_service.TSparkRowSetType{cli_service.TSparkRowSetType_ARROW_BASED_SET, cli_service.TSparkRowSetType_URL_BASED_SET} { + rowSet := &rows{ + client: &client.TestClient{}, config: config.WithDefaults(), ctx: context.Background(), + schema: &cli_service.TTableSchema{Columns: []*cli_service.TColumnDesc{{ + ColumnName: "clock", TypeDesc: &cli_service.TTypeDesc{Types: []*cli_service.TTypeEntry{{ + PrimitiveEntry: &cli_service.TPrimitiveTypeEntry{Type: cli_service.TTypeId_STRING_TYPE}, + }}}, + }}}, + resultSetMetadata: &cli_service.TGetResultSetMetadataResp{ArrowSchema: schemaBytes.Bytes(), ResultFormat: &format}, + } + want := arrowscan.ColumnTypeInfoFor(dataType) + var readers sync.WaitGroup + for range 8 { + readers.Go(func() { + if got := rowSet.ColumnTypeScanType(0); got != scanTypeDateTime { + t.Errorf("scan type = %v, want time.Time", got) + } + if got := rowSet.ColumnTypeDatabaseTypeName(0); got != "TIME" { + t.Errorf("database type = %q, want TIME", got) + } + }) + } + readers.Wait() + require.Equal(t, "TIME", rowSet.ColumnTypeDatabaseTypeName(0)) + length, hasLength := rowSet.ColumnTypeLength(0) + require.Zero(t, length) + require.False(t, hasLength) + for _, fallback := range []string{"no Arrow", "columnar", "no schema"} { + fallbackRows := &rows{client: rowSet.client, schema: rowSet.schema, ctx: rowSet.ctx, config: config.WithDefaults(), resultSetMetadata: &cli_service.TGetResultSetMetadataResp{ArrowSchema: schemaBytes.Bytes()}} + switch fallback { + case "no Arrow": + fallbackRows.config.UseArrowBatches = false + case "columnar": + columnar := cli_service.TSparkRowSetType_COLUMN_BASED_SET + fallbackRows.resultSetMetadata.ResultFormat = &columnar + case "no schema": + fallbackRows.resultSetMetadata.ArrowSchema = nil + } + require.Equal(t, "STRING", fallbackRows.ColumnTypeDatabaseTypeName(0), fallback) + require.Equal(t, scanTypeString, fallbackRows.ColumnTypeScanType(0), fallback) + } + require.Equal(t, scanTypeDateTime, rowSet.ColumnTypeScanType(0)) + require.Equal(t, want.ScanType, rowSet.ColumnTypeScanType(0)) + require.Equal(t, want.DatabaseTypeName, rowSet.ColumnTypeDatabaseTypeName(0)) + rowSet.RowScanner = nil + require.Equal(t, "TIME", rowSet.ColumnTypeDatabaseTypeName(0)) + } + }) + } +} diff --git a/kernel_time_e2e_test.go b/kernel_time_e2e_test.go new file mode 100644 index 00000000..be4de0be --- /dev/null +++ b/kernel_time_e2e_test.go @@ -0,0 +1,15 @@ +//go:build cgo && databricks_kernel && e2e + +package dbsql + +import "testing" + +func TestKernelE2ETimeResults(t *testing.T) { + options := []ConnOption{WithUseKernel(true), WithSessionParams(map[string]string{"timezone": "Asia/Tokyo"})} + if isReydenLeg() { + options = append(options, WithHTTPPath(reydenHTTPPath())) + } + db := timeTestDB(t, options...) + defer db.Close() //nolint:errcheck + assertTimeResults(t, db) +} diff --git a/pecotesting_creds_test.go b/pecotesting_creds_test.go index 0779989b..38b4fac6 100644 --- a/pecotesting_creds_test.go +++ b/pecotesting_creds_test.go @@ -53,6 +53,15 @@ func tokenFromConfigFile(t *testing.T) string { // the tagged kernel tests and the default-build Thrift tests can share one copy // instead of duplicating the read + skip block at each site. func pecoTestingCreds(t *testing.T) (host, httpPath, token string) { + t.Helper() + host, httpPath, token = readPecoTestingCreds(t) + if host == "" || httpPath == "" || token == "" { + t.Skip("set DATABRICKS_PECOTESTING_SERVER_HOSTNAME, DATABRICKS_PECOTESTING_HTTP_PATH2, and DATABRICKS_PECOTESTING_TOKEN to run") + } + return host, httpPath, token +} + +func readPecoTestingCreds(t *testing.T) (host, httpPath, token string) { t.Helper() host = os.Getenv("DATABRICKS_PECOTESTING_SERVER_HOSTNAME") httpPath = os.Getenv("DATABRICKS_PECOTESTING_HTTP_PATH2") @@ -67,8 +76,5 @@ func pecoTestingCreds(t *testing.T) (host, httpPath, token string) { if token == "" { token = tokenFromConfigFile(t) } - if host == "" || httpPath == "" || token == "" { - t.Skip("set DATABRICKS_PECOTESTING_SERVER_HOSTNAME, DATABRICKS_PECOTESTING_HTTP_PATH2, and DATABRICKS_PECOTESTING_TOKEN to run") - } return host, httpPath, token } diff --git a/time_capability_test.go b/time_capability_test.go new file mode 100644 index 00000000..615cab70 --- /dev/null +++ b/time_capability_test.go @@ -0,0 +1,52 @@ +package dbsql + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + dbsqlerr "github.com/databricks/databricks-sql-go/errors" + dbsqlerrint "github.com/databricks/databricks-sql-go/internal/errors" +) + +func serverTimeUnavailable(err error) bool { + var execution dbsqlerr.DBExecutionError + if !errors.As(err, &execution) || execution.SqlState() != "0A000" { + return false + } + message := strings.ToUpper(execution.Error()) + return strings.Contains(message, "[UNSUPPORTED_TYPED_LITERAL]") && strings.Contains(message, `OF THE TYPE "TIME"`) || + strings.Contains(message, "TIME TYPE IS DISABLED") || strings.Contains(message, "TIME TYPE IS NOT ENABLED") +} + +func TestServerTimeUnavailable(t *testing.T) { + const unsupported = `[UNSUPPORTED_TYPED_LITERAL] Literals of the type "TIME" are not supported.` + for _, test := range []struct { + name string + state string + message string + want bool + }{ + {"TIME literal", "0A000", unsupported, true}, + {"TIME disabled", "0A000", "The TIME type is disabled.", true}, + {"TIME not enabled", "0A000", "The TIME type is not enabled.", true}, + {"other literal", "0A000", `[UNSUPPORTED_TYPED_LITERAL] Literals of the type "DATE" are not supported. Supported types are "TIME".`, false}, + {"other feature", "0A000", "Some other feature is disabled. SELECT TIME '09:15:00'", false}, + {"unauthenticated", "28000", "HTTP 401: " + unsupported, false}, + {"forbidden", "42501", "HTTP 403: " + unsupported, false}, + {"missing state", "", unsupported, false}, + {"scanner failure", "0A000", "databricks: arrow row scanner unhandled type time64[ns]", false}, + } { + t.Run(test.name, func(t *testing.T) { + err := dbsqlerrint.NewExecutionErrorWithState(context.Background(), test.message, nil, "", test.state) + require.Equal(t, test.want, serverTimeUnavailable(err)) + require.Equal(t, test.want, serverTimeUnavailable(fmt.Errorf("query failed: %w", *err))) + }) + } + require.False(t, serverTimeUnavailable(nil)) + require.False(t, serverTimeUnavailable(errors.New(unsupported))) +} diff --git a/time_e2e_test.go b/time_e2e_test.go new file mode 100644 index 00000000..d2a3fbd1 --- /dev/null +++ b/time_e2e_test.go @@ -0,0 +1,71 @@ +//go:build e2e + +package dbsql + +import ( + "context" + "database/sql" + "reflect" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestE2ETimeResults(t *testing.T) { + db := timeTestDB(t, WithSessionParams(map[string]string{"timezone": "America/Los_Angeles"})) + defer db.Close() //nolint:errcheck + assertTimeResults(t, db) +} + +func timeTestDB(t *testing.T, extra ...ConnOption) *sql.DB { + t.Helper() + host, httpPath, token := readPecoTestingCreds(t) + require.NotEmpty(t, host, "live TIME tests require DATABRICKS_PECOTESTING_SERVER_HOSTNAME") + require.NotEmpty(t, httpPath, "live TIME tests require DATABRICKS_PECOTESTING_HTTP_PATH2") + require.NotEmpty(t, token, "live TIME tests require DATABRICKS_PECOTESTING_TOKEN or its configured fallback") + options := append([]ConnOption{WithServerHostname(host), WithHTTPPath(httpPath), WithAccessToken(token)}, extra...) + connector, err := NewConnector(options...) + require.NoError(t, err) + return sql.OpenDB(connector) +} + +func assertTimeResults(t *testing.T, db *sql.DB) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + rows, err := db.QueryContext(ctx, "SELECT clock AS typed, clock AS dynamic, clock AS text, CAST(NULL AS TIME) AS missing FROM (SELECT TIME '09:15:00.123456' AS clock)") + if serverTimeUnavailable(err) { + t.Skip("live TIME verification pending: this workspace has not enabled the TIME beta") + } + require.NoError(t, err) + defer rows.Close() //nolint:errcheck + columns, err := rows.ColumnTypes() + require.NoError(t, err) + for _, column := range columns { + require.Equal(t, "TIME", column.DatabaseTypeName()) + require.Equal(t, reflect.TypeOf(time.Time{}), column.ScanType()) + } + if !rows.Next() { + t.Fatalf("TIME result unavailable: %v", rows.Err()) + } + var typed time.Time + var dynamic any + var text string + var missing sql.NullTime + require.NoError(t, rows.Scan(&typed, &dynamic, &text, &missing)) + want := time.Date(1, time.January, 1, 9, 15, 0, 123456000, time.UTC) + require.Equal(t, want, typed) + require.Equal(t, want, dynamic) + require.Equal(t, "0001-01-01T09:15:00.123456Z", text) + require.False(t, missing.Valid) + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + + var list, structure, mapping string + err = db.QueryRowContext(ctx, "SELECT array(TIME '09:15:00.123456', CAST(NULL AS TIME)), named_struct('clock', TIME '09:15:00.123456', 'missing', CAST(NULL AS TIME)), map(TIME '09:15:00.123456', TIME '09:15:00.123456')").Scan(&list, &structure, &mapping) + require.NoError(t, err) + require.JSONEq(t, `["09:15:00.123456",null]`, list) + require.JSONEq(t, `{"clock":"09:15:00.123456","missing":null}`, structure) + require.JSONEq(t, `{"09:15:00.123456":"09:15:00.123456"}`, mapping) +} diff --git a/time_test.go b/time_test.go new file mode 100644 index 00000000..e72706d2 --- /dev/null +++ b/time_test.go @@ -0,0 +1,120 @@ +package dbsql + +import ( + "bytes" + "context" + "database/sql" + "net/url" + "reflect" + "strconv" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/stretchr/testify/require" + + "github.com/databricks/databricks-sql-go/internal/cli_service" +) + +func TestTimeScanDestinations(t *testing.T) { + for _, test := range []struct { + dataType arrow.DataType + clock string + nanos int + }{ + {arrow.FixedWidthTypes.Time32s, "09:15:00", 0}, + {arrow.FixedWidthTypes.Time32ms, "09:15:00.123", 123000000}, + {arrow.FixedWidthTypes.Time64us, "09:15:00.123456", 123456000}, + {arrow.FixedWidthTypes.Time64ns, "09:15:00.123456", 123456000}, + {arrow.FixedWidthTypes.Time64ns, "09:15:00.123456789", 123456789}, + } { + t.Run(test.dataType.String()+"/"+test.clock, func(t *testing.T) { + state := &callState{} + loadTestData(t, "OpenSessionSuccess.json", &state.openSessionResp) + loadTestData(t, "CloseSessionSuccess.json", &state.closeSessionResp) + loadTestData(t, "CloseOperationSuccess.json", &state.closeOperationResp) + loadTestData(t, "ExecuteStatement5.json", &state.executeStatementResp) + metadata := state.executeStatementResp.DirectResults.ResultSetMetadata + metadata.Schema.Columns = nil + fields := []arrow.Field{ + {Name: "typed", Type: test.dataType}, {Name: "dynamic", Type: test.dataType}, + {Name: "text", Type: test.dataType}, {Name: "missing", Type: test.dataType, Nullable: true}, + {Name: "sibling", Type: arrow.BinaryTypes.String}, + {Name: "missing_any", Type: test.dataType, Nullable: true}, + } + for index, field := range fields { + metadata.Schema.Columns = append(metadata.Schema.Columns, &cli_service.TColumnDesc{ + ColumnName: field.Name, Position: int32(index + 1), + TypeDesc: &cli_service.TTypeDesc{Types: []*cli_service.TTypeEntry{{PrimitiveEntry: &cli_service.TPrimitiveTypeEntry{Type: cli_service.TTypeId_STRING_TYPE}}}}, + }) + } + schema := arrow.NewSchema(fields, nil) + builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) + defer builder.Release() + for index := range 3 { + require.NoError(t, builder.Field(index).AppendValueFromString(test.clock)) + } + builder.Field(3).AppendNull() + require.NoError(t, builder.Field(4).AppendValueFromString("still readable")) + builder.Field(5).AppendNull() + record := builder.NewRecordBatch() + defer record.Release() + var schemaBytes, stream bytes.Buffer + schemaWriter := ipc.NewWriter(&schemaBytes, ipc.WithSchema(schema)) + require.NoError(t, schemaWriter.Close()) + metadata.ArrowSchema = schemaBytes.Bytes()[:schemaBytes.Len()-8] + streamWriter := ipc.NewWriter(&stream, ipc.WithSchema(schema)) + require.NoError(t, streamWriter.Write(record)) + require.NoError(t, streamWriter.Close()) + format := cli_service.TSparkRowSetType_ARROW_BASED_SET + metadata.ResultFormat = &format + state.executeStatementResp.DirectResults.ResultSet.Results = &cli_service.TRowSet{ + ArrowBatches: []*cli_service.TSparkArrowBatch{{Batch: stream.Bytes()[len(metadata.ArrowSchema):], RowCount: 1}}, + } + server := getServer(state) + defer server.Close() + address, err := url.Parse(server.URL) + require.NoError(t, err) + port, err := strconv.Atoi(address.Port()) + require.NoError(t, err) + connector, err := NewConnector(WithServerHostname("localhost"), WithPort(port), WithSessionParams(map[string]string{"timezone": "America/Los_Angeles"})) + require.NoError(t, err) + db := sql.OpenDB(connector) + defer db.Close() //nolint:errcheck + rows, err := db.QueryContext(context.Background(), "SELECT clock") + require.NoError(t, err) + defer rows.Close() //nolint:errcheck + columns, err := rows.ColumnTypes() + require.NoError(t, err) + for _, column := range columns[:4] { + require.Equal(t, "TIME", column.DatabaseTypeName()) + require.Equal(t, reflect.TypeOf(time.Time{}), column.ScanType()) + _, hasLength := column.Length() + require.False(t, hasLength) + } + require.Equal(t, "STRING", columns[4].DatabaseTypeName()) + require.True(t, rows.Next(), "%v", rows.Err()) + var typed time.Time + var dynamic any + var missingAny any = "not null" + var text, sibling string + var missing sql.NullTime + require.NoError(t, rows.Scan(&typed, &dynamic, &text, &missing, &sibling, &missingAny)) + want := time.Date(1, time.January, 1, 9, 15, 0, test.nanos, time.UTC) + require.Equal(t, want, typed) + require.Equal(t, want, dynamic) + require.Equal(t, want.Format(time.RFC3339Nano), text) + require.False(t, missing.Valid) + require.Nil(t, missingAny) + require.Equal(t, "still readable", sibling) + columns, err = rows.ColumnTypes() + require.NoError(t, err) + require.Equal(t, "TIME", columns[0].DatabaseTypeName()) + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + }) + } +} From 9b1039ce005602c9d0ac255dd27d514149c192ed Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sat, 14 Nov 2026 06:26:49 +0000 Subject: [PATCH 2/5] Address round-one TIME review follow-ups Document metadata and zero-time behavior, cover empty and nullable results, broaden precise TIME-unavailable tests, and compile live test configurations in PR CI. Retain the provisional TIME representation. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .github/workflows/go.yml | 8 +++ CHANGELOG.md | 2 +- README.md | 11 ++-- internal/arrowscan/arrowscan.go | 11 ++-- internal/arrowscan/time_test.go | 21 ++++++++ internal/rows/arrowbased/columnValues.go | 3 +- ...{time_bench_test.go => scan_bench_test.go} | 34 +++++++++---- internal/rows/arrowbased/time_test.go | 42 +++++++++++++++ internal/rows/time_test.go | 10 ++-- time_capability_test.go | 7 +++ time_test.go | 51 ++++++++++++++----- 11 files changed, 161 insertions(+), 39 deletions(-) rename internal/rows/arrowbased/{time_bench_test.go => scan_bench_test.go} (64%) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2f0f9ca6..2bf9181c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -102,6 +102,9 @@ jobs: - name: Test-Race run: make test-race + - name: Vet live e2e tests (compile only) + run: CGO_ENABLED=0 go vet -tags e2e . + - name: Build run: make linux @@ -256,3 +259,8 @@ jobs: # the separate nightly-e2e workflow. - name: Build kernel lib + run tagged tests run: make test-kernel + + - name: Vet kernel live e2e tests (compile only) + env: + CGO_LDFLAGS: -L${{ github.workspace }}/internal/backend/kernel/lib/linux_amd64 -l:libdatabricks_sql_kernel.a + run: CGO_ENABLED=1 go vet -tags databricks_kernel,e2e . diff --git a/CHANGELOG.md b/CHANGELOG.md index d782a1af..62ac379f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162). - **Behavior change:** Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results. Tools such as ORM migrators can now act on these qualifiers (databricks/databricks-sql-go#500). -- Support Arrow TIME results on Thrift and SEA, preserving precision as UTC `time.Time` values on `0001-01-01`, with `TIME` column metadata and clock-only nested JSON values. +- **Behavior change:** Support Arrow TIME results on Thrift and SEA as UTC `time.Time` values on `0001-01-01`, preserving precision and using clock-only nested JSON. TIME metadata now reports `TIME`/`time.Time` with no length, including empty and NULL-only results; previously Thrift reported `STRING`/`string` with unbounded length, and SEA reported an empty name with `*interface{}` (databricks/databricks-sql-go#505). - **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication. - Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439). - **Behavior change:** On the Thrift backend, `ExecuteStatement` is now retried (per `WithRetries`) when the connection to the server or proxy can't be opened, for example connection refused or a connect timeout, because the statement was never sent. Failures after the connection exists are still not retried, so a statement never runs twice. With default retry settings, a refused connection now fails after about 15 seconds instead of immediately (databricks/databricks-sql-go#423). diff --git a/README.md b/README.md index 38c6fc50..6ced160f 100644 --- a/README.md +++ b/README.md @@ -374,9 +374,14 @@ SQL TIME values in Arrow results scan into `time.Time` (or `any`) on the referen precision. The date and location are carriers only; session time zones do not shift the clock. Scanning into `string` uses `database/sql`'s RFC3339Nano format (for example, `0001-01-01T09:15:00.123456Z`). NULL scans into `any` as `nil`; -use `sql.NullTime` for nullable typed values. Column metadata reports `TIME` and -`time.Time`. Nested TIME values in ARRAY / MAP / STRUCT JSON use clock-only strings -such as `"09:15:00.123456"`. +use `sql.NullTime` for nullable typed values. Non-NULL midnight equals `time.Time{}` +and has `IsZero() == true`, so detect NULL with `sql.NullTime.Valid`, not `IsZero()`; +JSON fields tagged `omitzero` omit midnight. Do not call `.In(location)` on this +date-less value: it shifts the clock and may introduce historical timezone offsets. +Column metadata reports `TIME` and `time.Time` with no length, even for empty or +NULL-only results. Nested TIME values in ARRAY / MAP / STRUCT JSON use clock-only +strings with trailing fractional zeros omitted, such as `"09:15:00.5"` rather than +`"09:15:00.500000"`; their text can differ from server-rendered TIME(p) values. On Thrift, TIME column metadata requires a server-supplied Arrow schema; without it, the legacy Thrift column metadata is retained. diff --git a/internal/arrowscan/arrowscan.go b/internal/arrowscan/arrowscan.go index 1a00f33e..ba44029f 100644 --- a/internal/arrowscan/arrowscan.go +++ b/internal/arrowscan/arrowscan.go @@ -12,7 +12,8 @@ // - map → {"k0":v0,"k1":v1,...} (keys stringified) // - struct → {"field0":v0,...} // - nested NULL → null -// - time.Time → quoted .String() (matches the Thrift marshalScalar special-case) +// - DATE/TIMESTAMP → quoted time.Time.String() (matches Thrift) +// - TIME → quoted clock-only text, with trailing fractional zeros omitted // - nested decimal → exact scale-applied JSON number literal (never a lossy // float64), matching Thrift's marshalScalar → ValueString // - float32 → native float32 (not widened to float64), so JSON renders @@ -35,7 +36,7 @@ import ( // ScanCell extracts one cell as a driver.Value. Scalars map to their Go value: // bool, all int/uint widths, float (native float32/float64), string, binary, -// date, timestamp, and top-level decimal (as an exact fixed-point string, +// date, time, timestamp, and top-level decimal (as an exact fixed-point string, // matching the Thrift path — a float64 would lose precision beyond ~17 digits). // Nested types (List/Map/Struct, and VARIANT which // arrives nested) render to a JSON string byte-identical to the Thrift path. @@ -631,10 +632,8 @@ func writeScalarJSON(b *strings.Builder, v any) error { return nil } -// scalarForJSON returns the Go value used for a nested leaf that is not itself a -// container — today only a map key (values are written directly by writeJSON). -// It reuses ScanCell's scalar arm, so a decimal key renders via the exact-string -// path (writeJSONKey then quotes it), never a lossy float64. +// scalarForJSON converts nested scalar values and map keys. TIME uses clock-only +// text; decimals use ScanCell's exact-string path, never a lossy float64. func scalarForJSON(col arrow.Array, row int, loc *time.Location) (any, error) { if col.IsNull(row) { return nil, nil diff --git a/internal/arrowscan/time_test.go b/internal/arrowscan/time_test.go index a61abf41..c4492ee3 100644 --- a/internal/arrowscan/time_test.go +++ b/internal/arrowscan/time_test.go @@ -1,6 +1,7 @@ package arrowscan import ( + "reflect" "testing" "time" @@ -37,3 +38,23 @@ func TestScanTimeUnits(t *testing.T) { }) } } + +func TestTimeNullOnlyMetadata(t *testing.T) { + for _, dataType := range []arrow.DataType{arrow.FixedWidthTypes.Time32s, arrow.FixedWidthTypes.Time32ms, arrow.FixedWidthTypes.Time64us, arrow.FixedWidthTypes.Time64ns} { + t.Run(dataType.String(), func(t *testing.T) { + builder := array.NewBuilder(memory.DefaultAllocator, dataType) + defer builder.Release() + builder.AppendNull() + values := builder.NewArray() + defer values.Release() + got, err := ScanCell(values, 0, nil) + require.NoError(t, err) + require.Nil(t, got) + metadata := ColumnTypeInfoFor(dataType) + require.Equal(t, "TIME", metadata.DatabaseTypeName) + require.Equal(t, reflect.TypeOf(time.Time{}), metadata.ScanType) + require.Zero(t, metadata.Length) + require.False(t, metadata.HasLength) + }) + } +} diff --git a/internal/rows/arrowbased/columnValues.go b/internal/rows/arrowbased/columnValues.go index 512fabf2..9a629b34 100644 --- a/internal/rows/arrowbased/columnValues.go +++ b/internal/rows/arrowbased/columnValues.go @@ -616,7 +616,8 @@ func (tvc *decimal128Container) SetValueArray(colData arrow.ArrayData) error { } // marshalScalar appends a non-complex nested value (a struct field, list -// element, or map value/key) to dst as its JSON fragment. When the holder is a +// element, or map value/key) to dst as its JSON fragment. TIME uses clock-only +// text; other temporal values use time.Time.String(). When the holder is a // native decimal128, it emits the exact scale-applied decimal string as a JSON // number literal rather than marshaling the lossy float64 returned by // holder.Value — otherwise a DECIMAL(5,2) 19.99 would render as diff --git a/internal/rows/arrowbased/time_bench_test.go b/internal/rows/arrowbased/scan_bench_test.go similarity index 64% rename from internal/rows/arrowbased/time_bench_test.go rename to internal/rows/arrowbased/scan_bench_test.go index fb4d652c..3a297ccc 100644 --- a/internal/rows/arrowbased/time_bench_test.go +++ b/internal/rows/arrowbased/scan_bench_test.go @@ -11,23 +11,39 @@ import ( "github.com/databricks/databricks-sql-go/internal/arrowscan" ) +type scanBenchmarkCase struct { + name string + dataType arrow.DataType + value string +} + func BenchmarkScanExistingColumns(b *testing.B) { - for _, test := range []struct { - name string - dataType arrow.DataType - value string - }{ + benchmarkScanColumns(b, []scanBenchmarkCase{ {"int64", arrow.PrimitiveTypes.Int64, "123456789"}, {"string", arrow.BinaryTypes.String, "existing column"}, {"timestamp", arrow.FixedWidthTypes.Timestamp_us, "2026-11-14 09:15:00.123456Z"}, - } { + }) +} + +func BenchmarkScanTimeColumns(benchmark *testing.B) { + benchmarkScanColumns(benchmark, []scanBenchmarkCase{ + {"time32s", arrow.FixedWidthTypes.Time32s, "09:15:00"}, + {"time32ms", arrow.FixedWidthTypes.Time32ms, "09:15:00.123"}, + {"time64us", arrow.FixedWidthTypes.Time64us, "09:15:00.123456"}, + {"time64ns", arrow.FixedWidthTypes.Time64ns, "09:15:00.123456789"}, + }) +} + +func benchmarkScanColumns(benchmark *testing.B, cases []scanBenchmarkCase) { + benchmark.Helper() + for _, test := range cases { builder := array.NewBuilder(memory.DefaultAllocator, test.dataType) if err := builder.AppendValueFromString(test.value); err != nil { - b.Fatal(err) + benchmark.Fatal(err) } values := builder.NewArray() builder.Release() - b.Run(test.name+"/thrift", func(b *testing.B) { + benchmark.Run(test.name+"/thrift", func(b *testing.B) { holder, err := (&arrowValueContainerMaker{}).makeColumnValueContainer(test.dataType, time.UTC, func(timestamp arrow.Timestamp) time.Time { return timestamp.ToTime(arrow.Microsecond) }, nil) @@ -46,7 +62,7 @@ func BenchmarkScanExistingColumns(b *testing.B) { } } }) - b.Run(test.name+"/kernel", func(b *testing.B) { + benchmark.Run(test.name+"/kernel", func(b *testing.B) { b.ReportAllocs() b.ResetTimer() for iteration := 0; iteration < b.N; iteration++ { diff --git a/internal/rows/arrowbased/time_test.go b/internal/rows/arrowbased/time_test.go index fc84299a..c982c14e 100644 --- a/internal/rows/arrowbased/time_test.go +++ b/internal/rows/arrowbased/time_test.go @@ -111,3 +111,45 @@ func TestTimeColumns(t *testing.T) { }) } } + +func TestNestedTimeFractionalZeros(t *testing.T) { + for _, test := range []struct { + dataType arrow.DataType + clock string + expected string + }{ + {arrow.FixedWidthTypes.Time32s, "10:30:00", "10:30:00"}, + {arrow.FixedWidthTypes.Time32ms, "09:15:30.500", "09:15:30.5"}, + {arrow.FixedWidthTypes.Time64us, "06:30:45.887000", "06:30:45.887"}, + {arrow.FixedWidthTypes.Time64ns, "09:15:30.500000000", "09:15:30.5"}, + } { + t.Run(test.dataType.String(), func(t *testing.T) { + pool := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer pool.AssertSize(t, 0) + for _, nested := range []struct { + dataType arrow.DataType + input string + expected string + }{ + {arrow.ListOf(test.dataType), fmt.Sprintf(`[[%q]]`, test.clock), fmt.Sprintf(`[%q]`, test.expected)}, + {arrow.StructOf(arrow.Field{Name: "clock", Type: test.dataType}), fmt.Sprintf(`[{"clock":%q}]`, test.clock), fmt.Sprintf(`{"clock":%q}`, test.expected)}, + } { + builder := array.NewBuilder(pool, nested.dataType) + defer builder.Release() + require.NoError(t, builder.UnmarshalJSON([]byte(nested.input))) + values := builder.NewArray() + defer values.Release() + holder, err := (&arrowValueContainerMaker{}).makeColumnValueContainer(nested.dataType, time.UTC, nil, nil) + require.NoError(t, err) + defer holder.Release() + require.NoError(t, holder.SetValueArray(values.Data())) + thrift, err := holder.Value(0) + require.NoError(t, err) + require.Equal(t, nested.expected, thrift) + kernel, err := arrowscan.ScanCell(values, 0, nil) + require.NoError(t, err) + require.Equal(t, nested.expected, kernel) + } + }) + } +} diff --git a/internal/rows/time_test.go b/internal/rows/time_test.go index f0560c84..d3eaafc4 100644 --- a/internal/rows/time_test.go +++ b/internal/rows/time_test.go @@ -3,14 +3,15 @@ package rows import ( "bytes" "context" + "reflect" "sync" "testing" + "time" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/ipc" "github.com/stretchr/testify/require" - "github.com/databricks/databricks-sql-go/internal/arrowscan" "github.com/databricks/databricks-sql-go/internal/cli_service" "github.com/databricks/databricks-sql-go/internal/client" "github.com/databricks/databricks-sql-go/internal/config" @@ -32,7 +33,6 @@ func TestTimeColumnMetadata(t *testing.T) { }}}, resultSetMetadata: &cli_service.TGetResultSetMetadataResp{ArrowSchema: schemaBytes.Bytes(), ResultFormat: &format}, } - want := arrowscan.ColumnTypeInfoFor(dataType) var readers sync.WaitGroup for range 8 { readers.Go(func() { @@ -63,11 +63,7 @@ func TestTimeColumnMetadata(t *testing.T) { require.Equal(t, "STRING", fallbackRows.ColumnTypeDatabaseTypeName(0), fallback) require.Equal(t, scanTypeString, fallbackRows.ColumnTypeScanType(0), fallback) } - require.Equal(t, scanTypeDateTime, rowSet.ColumnTypeScanType(0)) - require.Equal(t, want.ScanType, rowSet.ColumnTypeScanType(0)) - require.Equal(t, want.DatabaseTypeName, rowSet.ColumnTypeDatabaseTypeName(0)) - rowSet.RowScanner = nil - require.Equal(t, "TIME", rowSet.ColumnTypeDatabaseTypeName(0)) + require.Equal(t, reflect.TypeOf(time.Time{}), rowSet.ColumnTypeScanType(0)) } }) } diff --git a/time_capability_test.go b/time_capability_test.go index 615cab70..efdb067c 100644 --- a/time_capability_test.go +++ b/time_capability_test.go @@ -20,6 +20,8 @@ func serverTimeUnavailable(err error) bool { } message := strings.ToUpper(execution.Error()) return strings.Contains(message, "[UNSUPPORTED_TYPED_LITERAL]") && strings.Contains(message, `OF THE TYPE "TIME"`) || + strings.Contains(message, "[UNSUPPORTED_DATATYPE]") && strings.Contains(message, `UNSUPPORTED DATA TYPE "TIME"`) || + strings.Contains(message, "[UNSUPPORTED_TIME_TYPE]") && strings.Contains(message, "THE DATA TYPE TIME IS NOT SUPPORTED") || strings.Contains(message, "TIME TYPE IS DISABLED") || strings.Contains(message, "TIME TYPE IS NOT ENABLED") } @@ -32,9 +34,14 @@ func TestServerTimeUnavailable(t *testing.T) { want bool }{ {"TIME literal", "0A000", unsupported, true}, + {"TIME datatype", "0A000", `[UNSUPPORTED_DATATYPE] Unsupported data type "TIME".`, true}, + {"TIME construction", "0A000", "[UNSUPPORTED_TIME_TYPE] The data type TIME is not supported.", true}, {"TIME disabled", "0A000", "The TIME type is disabled.", true}, {"TIME not enabled", "0A000", "The TIME type is not enabled.", true}, {"other literal", "0A000", `[UNSUPPORTED_TYPED_LITERAL] Literals of the type "DATE" are not supported. Supported types are "TIME".`, false}, + {"other datatype", "0A000", `[UNSUPPORTED_DATATYPE] Unsupported data type "DATE". SELECT TIME '09:15:00'`, false}, + {"other construction", "0A000", "[UNSUPPORTED_TIME_TYPE] The data type DATE is not supported. SELECT TIME '09:15:00'", false}, + {"unavailable config", "42K0I", "[CONFIG_NOT_AVAILABLE.WITHOUT_SUGGESTION] spark.sql.timeType.enabled is not available", false}, {"other feature", "0A000", "Some other feature is disabled. SELECT TIME '09:15:00'", false}, {"unauthenticated", "28000", "HTTP 401: " + unsupported, false}, {"forbidden", "42501", "HTTP 403: " + unsupported, false}, diff --git a/time_test.go b/time_test.go index e72706d2..05ed417a 100644 --- a/time_test.go +++ b/time_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "encoding/json" "net/url" "reflect" "strconv" @@ -24,12 +25,15 @@ func TestTimeScanDestinations(t *testing.T) { dataType arrow.DataType clock string nanos int + empty bool }{ - {arrow.FixedWidthTypes.Time32s, "09:15:00", 0}, - {arrow.FixedWidthTypes.Time32ms, "09:15:00.123", 123000000}, - {arrow.FixedWidthTypes.Time64us, "09:15:00.123456", 123456000}, - {arrow.FixedWidthTypes.Time64ns, "09:15:00.123456", 123456000}, - {arrow.FixedWidthTypes.Time64ns, "09:15:00.123456789", 123456789}, + {arrow.FixedWidthTypes.Time32s, "09:15:00", 0, false}, + {arrow.FixedWidthTypes.Time32ms, "09:15:00.123", 123000000, false}, + {arrow.FixedWidthTypes.Time64us, "09:15:00.123456", 123456000, false}, + {arrow.FixedWidthTypes.Time64ns, "09:15:00.123456", 123456000, false}, + {arrow.FixedWidthTypes.Time64ns, "09:15:00.123456789", 123456789, false}, + {arrow.FixedWidthTypes.Time64ns, "00:00:00", 0, false}, + {arrow.FixedWidthTypes.Time64ns, "empty", 0, true}, } { t.Run(test.dataType.String()+"/"+test.clock, func(t *testing.T) { state := &callState{} @@ -44,6 +48,7 @@ func TestTimeScanDestinations(t *testing.T) { {Name: "text", Type: test.dataType}, {Name: "missing", Type: test.dataType, Nullable: true}, {Name: "sibling", Type: arrow.BinaryTypes.String}, {Name: "missing_any", Type: test.dataType, Nullable: true}, + {Name: "nullable", Type: test.dataType, Nullable: true}, } for index, field := range fields { metadata.Schema.Columns = append(metadata.Schema.Columns, &cli_service.TColumnDesc{ @@ -54,12 +59,15 @@ func TestTimeScanDestinations(t *testing.T) { schema := arrow.NewSchema(fields, nil) builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) defer builder.Release() - for index := range 3 { - require.NoError(t, builder.Field(index).AppendValueFromString(test.clock)) + if !test.empty { + for index := range 3 { + require.NoError(t, builder.Field(index).AppendValueFromString(test.clock)) + } + builder.Field(3).AppendNull() + require.NoError(t, builder.Field(4).AppendValueFromString("still readable")) + builder.Field(5).AppendNull() + require.NoError(t, builder.Field(6).AppendValueFromString(test.clock)) } - builder.Field(3).AppendNull() - require.NoError(t, builder.Field(4).AppendValueFromString("still readable")) - builder.Field(5).AppendNull() record := builder.NewRecordBatch() defer record.Release() var schemaBytes, stream bytes.Buffer @@ -74,6 +82,9 @@ func TestTimeScanDestinations(t *testing.T) { state.executeStatementResp.DirectResults.ResultSet.Results = &cli_service.TRowSet{ ArrowBatches: []*cli_service.TSparkArrowBatch{{Batch: stream.Bytes()[len(metadata.ArrowSchema):], RowCount: 1}}, } + if test.empty { + state.executeStatementResp.DirectResults.ResultSet.Results.ArrowBatches = []*cli_service.TSparkArrowBatch{} + } server := getServer(state) defer server.Close() address, err := url.Parse(server.URL) @@ -96,18 +107,34 @@ func TestTimeScanDestinations(t *testing.T) { require.False(t, hasLength) } require.Equal(t, "STRING", columns[4].DatabaseTypeName()) + if test.empty { + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + return + } require.True(t, rows.Next(), "%v", rows.Err()) var typed time.Time var dynamic any var missingAny any = "not null" var text, sibling string - var missing sql.NullTime - require.NoError(t, rows.Scan(&typed, &dynamic, &text, &missing, &sibling, &missingAny)) + var missing, nullable sql.NullTime + require.NoError(t, rows.Scan(&typed, &dynamic, &text, &missing, &sibling, &missingAny, &nullable)) want := time.Date(1, time.January, 1, 9, 15, 0, test.nanos, time.UTC) + if test.clock == "00:00:00" { + want = time.Time{} + require.True(t, typed.IsZero()) + encoded, err := json.Marshal(struct { + Clock time.Time `json:"clock,omitzero"` + }{Clock: typed}) + require.NoError(t, err) + require.JSONEq(t, `{}`, string(encoded)) + } require.Equal(t, want, typed) require.Equal(t, want, dynamic) require.Equal(t, want.Format(time.RFC3339Nano), text) require.False(t, missing.Valid) + require.True(t, nullable.Valid) + require.Equal(t, want, nullable.Time) require.Nil(t, missingAny) require.Equal(t, "still readable", sibling) columns, err = rows.ColumnTypes() From c266846b1332283b0202c178079c82a03ad122d4 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sat, 14 Nov 2026 08:49:55 +0000 Subject: [PATCH 3/5] Finalize TIME mapping decision and live verification Document the maintainer-approved year-1 UTC convention and midnight trade-off. Verify precision, NULLs, real Arrow schemas and large results on Thrift and SEA; fail opt-in live tests when TIME is unavailable. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .github/workflows/nightly-e2e.yml | 2 +- README.md | 11 +- doc.go | 4 + internal/arrowscan/time.go | 2 + kernel_time_e2e_test.go | 2 + time_arrow_e2e_test.go | 179 ++++++++++++++++++++++++++++++ time_capability_test.go | 59 ---------- time_e2e_test.go | 17 ++- 8 files changed, 209 insertions(+), 67 deletions(-) create mode 100644 time_arrow_e2e_test.go delete mode 100644 time_capability_test.go diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index e9177905..441f47f9 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -90,7 +90,7 @@ jobs: DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} run: | go test -tags e2e -count=1 -v -timeout 15m \ - -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads|TestE2EDecimalMetadata|TestE2ETimeResults' . + -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads|TestE2EDecimalMetadata|TestE2ETime' . kernel-e2e: name: E2E (kernel backend) diff --git a/README.md b/README.md index 6ced160f..41debb60 100644 --- a/README.md +++ b/README.md @@ -369,12 +369,16 @@ STRUCT and VARIANT (as JSON) are supported. BINARY is returned as `sql.RawBytes` Both backends return GEOMETRY / GEOGRAPHY as WKT / EWKT by default. Set `enableGeoSpatialSupport=true` on the kernel path to opt into EWKB. +| SQL type | Go value | NULL detection | +|---|---|---| +| `TIME` | `time.Time` on `0001-01-01` UTC | `sql.NullTime.Valid` or a nil `*time.Time`, never `IsZero()`; midnight is the zero time and JSON `omitzero` omits it. | + SQL TIME values in Arrow results scan into `time.Time` (or `any`) on the reference date `0001-01-01` in UTC, preserving the Arrow second/millisecond/microsecond/nanosecond precision. The date and location are carriers only; session time zones do not shift the clock. Scanning into `string` uses `database/sql`'s RFC3339Nano format (for example, `0001-01-01T09:15:00.123456Z`). NULL scans into `any` as `nil`; -use `sql.NullTime` for nullable typed values. Non-NULL midnight equals `time.Time{}` +use `sql.NullTime` or `*time.Time` for nullable typed values. Non-NULL midnight equals `time.Time{}` and has `IsZero() == true`, so detect NULL with `sql.NullTime.Valid`, not `IsZero()`; JSON fields tagged `omitzero` omit midnight. Do not call `.In(location)` on this date-less value: it shifts the clock and may introduce historical timezone offsets. @@ -387,9 +391,8 @@ without it, the legacy Thrift column metadata is retained. Live TIME tests require the `e2e` build tag: `go test -tags e2e -run '^TestE2ETimeResults$' .` or `go test -tags databricks_kernel,e2e -run '^TestKernelE2ETimeResults$' .`. -They require the `DATABRICKS_PECOTESTING_*` credentials and skip only when the server -reports that TIME is disabled; a workspace with the TIME beta enabled is needed -to verify live result conversion. +They require the `DATABRICKS_PECOTESTING_*` credentials and a workspace with the TIME +beta enabled; missing credentials or unavailable TIME support fail the tests. On Thrift, TIMESTAMP_NTZ handling requires native timestamp fields in the server-supplied Arrow schema. Without that schema, NTZ still reports `TIMESTAMP` diff --git a/doc.go b/doc.go index b0cc32a0..429d8416 100644 --- a/doc.go +++ b/doc.go @@ -536,6 +536,10 @@ STRING --> string DATE --> time.Time +TIME --> time.Time (0001-01-01 UTC). Detect NULL with sql.NullTime.Valid or a nil +*time.Time, never IsZero(): non-NULL midnight is the zero time and JSON omitzero +omits it. + TIMESTAMP --> time.Time DECIMAL(p,s) --> sql.RawBytes diff --git a/internal/arrowscan/time.go b/internal/arrowscan/time.go index e630a2f2..933f9316 100644 --- a/internal/arrowscan/time.go +++ b/internal/arrowscan/time.go @@ -11,6 +11,8 @@ func timeOfDayValue(value int64, unit arrow.TimeUnit) driver.Value { return timeOfDay(value, unit) } +// timeOfDay carries TIME on 0001-01-01 UTC. Midnight equals time.Time{} and +// JSON omitzero omits it; detect NULL with sql.NullTime or *time.Time, not IsZero. func timeOfDay(value int64, unit arrow.TimeUnit) time.Time { return time.Time{}.Add(time.Duration(value) * unit.Multiplier()) } diff --git a/kernel_time_e2e_test.go b/kernel_time_e2e_test.go index be4de0be..cc076b8d 100644 --- a/kernel_time_e2e_test.go +++ b/kernel_time_e2e_test.go @@ -12,4 +12,6 @@ func TestKernelE2ETimeResults(t *testing.T) { db := timeTestDB(t, options...) defer db.Close() //nolint:errcheck assertTimeResults(t, db) + t.Run("inline Arrow", func(t *testing.T) { assertTimeArrowResults(t, db, 3) }) + t.Run("CloudFetch", func(t *testing.T) { assertTimeArrowResults(t, db, 250000) }) } diff --git a/time_arrow_e2e_test.go b/time_arrow_e2e_test.go new file mode 100644 index 00000000..11e8fc3c --- /dev/null +++ b/time_arrow_e2e_test.go @@ -0,0 +1,179 @@ +//go:build e2e + +package dbsql + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "net/http" + "sync" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/stretchr/testify/require" + + "github.com/databricks/databricks-sql-go/internal/backend" + thriftbackend "github.com/databricks/databricks-sql-go/internal/backend/thrift" + "github.com/databricks/databricks-sql-go/internal/cli_service" + "github.com/databricks/databricks-sql-go/internal/client" + "github.com/databricks/databricks-sql-go/internal/config" + dbsqlrows "github.com/databricks/databricks-sql-go/rows" +) + +func TestE2ETimeArrowResults(t *testing.T) { + for _, test := range []struct { + name string + rows int + format cli_service.TSparkRowSetType + }{ + {"inline", 3, cli_service.TSparkRowSetType_ARROW_BASED_SET}, + {"CloudFetch", 250000, cli_service.TSparkRowSetType_URL_BASED_SET}, + } { + t.Run(test.name, func(t *testing.T) { + host, path, token := readPecoTestingCreds(t) + require.NotEmpty(t, host) + require.NotEmpty(t, path) + require.NotEmpty(t, token) + connectorValue, err := NewConnector(WithServerHostname(host), WithHTTPPath(path), WithAccessToken(token), + WithSessionParams(map[string]string{"timezone": "America/Los_Angeles"})) + require.NoError(t, err) + capture := &timeWireClient{} + connectorValue.(*connector).thriftBackendFactory = func(ctx context.Context, cfg *config.Config, httpClient *http.Client) (backend.Backend, error) { + service, err := client.InitThriftClient(cfg, httpClient) + if err != nil { + return nil, err + } + capture.TCLIService = service + return thriftbackend.NewForTest(capture, nil, cfg), nil + } + db := sql.OpenDB(connectorValue) + db.SetMaxOpenConns(1) + defer db.Close() //nolint:errcheck + assertTimeArrowResults(t, db, test.rows) + capture.mu.Lock() + defer capture.mu.Unlock() + require.NotEmpty(t, capture.metadata) + for _, metadata := range capture.metadata { + require.Equal(t, test.format, metadata.GetResultFormat()) + require.NotEmpty(t, metadata.ArrowSchema) + reader, err := ipc.NewReader(bytes.NewReader(metadata.ArrowSchema)) + require.NoError(t, err) + require.Equal(t, arrow.FixedWidthTypes.Time64ns, reader.Schema().Field(1).Type) + reader.Release() + typeID := metadata.Schema.Columns[1].TypeDesc.Types[0].PrimitiveEntry.Type + t.Logf("TIME wire TTypeId=%s (%d), Arrow=time64[ns], result format=%s", typeID, typeID, metadata.GetResultFormat()) + require.Equal(t, cli_service.TTypeId_STRING_TYPE, typeID) + } + require.Equal(t, test.format == cli_service.TSparkRowSetType_URL_BASED_SET, capture.links) + }) + } +} + +type timeWireClient struct { + cli_service.TCLIService + mu sync.Mutex + metadata []*cli_service.TGetResultSetMetadataResp + links bool +} + +func (capture *timeWireClient) observe(metadata *cli_service.TGetResultSetMetadataResp, result *cli_service.TFetchResultsResp) { + capture.mu.Lock() + defer capture.mu.Unlock() + if metadata != nil { + capture.metadata = append(capture.metadata, metadata) + } + if result != nil { + if result.ResultSetMetadata != nil { + capture.metadata = append(capture.metadata, result.ResultSetMetadata) + } + if result.Results != nil && len(result.Results.ResultLinks) > 0 { + capture.links = true + } + } +} + +func (capture *timeWireClient) ExecuteStatement(ctx context.Context, req *cli_service.TExecuteStatementReq) (*cli_service.TExecuteStatementResp, error) { + response, err := capture.TCLIService.ExecuteStatement(ctx, req) + if response != nil && response.DirectResults != nil { + capture.observe(response.DirectResults.ResultSetMetadata, response.DirectResults.ResultSet) + } + return response, err +} + +func (capture *timeWireClient) GetResultSetMetadata(ctx context.Context, req *cli_service.TGetResultSetMetadataReq) (*cli_service.TGetResultSetMetadataResp, error) { + response, err := capture.TCLIService.GetResultSetMetadata(ctx, req) + capture.observe(response, nil) + return response, err +} + +func (capture *timeWireClient) FetchResults(ctx context.Context, req *cli_service.TFetchResultsReq) (*cli_service.TFetchResultsResp, error) { + response, err := capture.TCLIService.FetchResults(ctx, req) + capture.observe(nil, response) + return response, err +} + +func assertTimeArrowResults(t *testing.T, db *sql.DB, wantRows int) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + query := "SELECT id, CASE WHEN id % 3 = 0 THEN CAST(NULL AS TIME) WHEN id % 3 = 1 THEN TIME '00:00:00' ELSE TIME '23:59:59.123456' END AS clock, array(TIME '09:15:00.123456', CAST(NULL AS TIME)) AS clocks, named_struct('clock', TIME '09:15:00.123456', 'missing', CAST(NULL AS TIME)) AS structure, sha2(CAST(id AS STRING), 512) AS payload FROM range(?)" + rows, err := db.QueryContext(ctx, query, int64(wantRows)) + require.NoError(t, err) + defer rows.Close() //nolint:errcheck + columns, err := rows.ColumnTypes() + require.NoError(t, err) + require.Equal(t, "TIME", columns[1].DatabaseTypeName()) + count := 0 + for rows.Next() { + var id int64 + var clock sql.NullTime + var clocks, structure, payload string + require.NoError(t, rows.Scan(&id, &clock, &clocks, &structure, &payload)) + require.Equal(t, id%3 != 0, clock.Valid) + if clock.Valid { + want := time.Time{} + if id%3 == 2 { + want = time.Date(1, time.January, 1, 23, 59, 59, 123456000, time.UTC) + } + require.Equal(t, want, clock.Time) + } + if count == 0 { + require.JSONEq(t, `["09:15:00.123456",null]`, clocks) + require.JSONEq(t, `{"clock":"09:15:00.123456","missing":null}`, structure) + } + require.Len(t, payload, 128) + count++ + } + require.NoError(t, rows.Err()) + require.Equal(t, wantRows, count) + require.NoError(t, rows.Close()) + + connection, err := db.Conn(ctx) + require.NoError(t, err) + defer connection.Close() //nolint:errcheck + require.NoError(t, connection.Raw(func(raw any) error { + result, err := raw.(driver.QueryerContext).QueryContext(ctx, query, []driver.NamedValue{{Ordinal: 1, Value: int64(wantRows)}}) + require.NoError(t, err) + defer result.Close() //nolint:errcheck + batches, err := result.(dbsqlrows.Rows).GetArrowBatches(ctx) + require.NoError(t, err) + defer batches.Close() + schema, err := batches.Schema() + require.NoError(t, err) + require.Equal(t, arrow.FixedWidthTypes.Time64ns, schema.Field(1).Type) + var batchRows int64 + for batches.HasNext() { + batch, err := batches.Next() + require.NoError(t, err) + batchRows += batch.NumRows() + batch.Release() + } + require.Equal(t, int64(wantRows), batchRows) + t.Logf("verified time64[ns] schema and %d rows through database/sql and Arrow batches", wantRows) + return nil + })) +} diff --git a/time_capability_test.go b/time_capability_test.go deleted file mode 100644 index efdb067c..00000000 --- a/time_capability_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package dbsql - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/require" - - dbsqlerr "github.com/databricks/databricks-sql-go/errors" - dbsqlerrint "github.com/databricks/databricks-sql-go/internal/errors" -) - -func serverTimeUnavailable(err error) bool { - var execution dbsqlerr.DBExecutionError - if !errors.As(err, &execution) || execution.SqlState() != "0A000" { - return false - } - message := strings.ToUpper(execution.Error()) - return strings.Contains(message, "[UNSUPPORTED_TYPED_LITERAL]") && strings.Contains(message, `OF THE TYPE "TIME"`) || - strings.Contains(message, "[UNSUPPORTED_DATATYPE]") && strings.Contains(message, `UNSUPPORTED DATA TYPE "TIME"`) || - strings.Contains(message, "[UNSUPPORTED_TIME_TYPE]") && strings.Contains(message, "THE DATA TYPE TIME IS NOT SUPPORTED") || - strings.Contains(message, "TIME TYPE IS DISABLED") || strings.Contains(message, "TIME TYPE IS NOT ENABLED") -} - -func TestServerTimeUnavailable(t *testing.T) { - const unsupported = `[UNSUPPORTED_TYPED_LITERAL] Literals of the type "TIME" are not supported.` - for _, test := range []struct { - name string - state string - message string - want bool - }{ - {"TIME literal", "0A000", unsupported, true}, - {"TIME datatype", "0A000", `[UNSUPPORTED_DATATYPE] Unsupported data type "TIME".`, true}, - {"TIME construction", "0A000", "[UNSUPPORTED_TIME_TYPE] The data type TIME is not supported.", true}, - {"TIME disabled", "0A000", "The TIME type is disabled.", true}, - {"TIME not enabled", "0A000", "The TIME type is not enabled.", true}, - {"other literal", "0A000", `[UNSUPPORTED_TYPED_LITERAL] Literals of the type "DATE" are not supported. Supported types are "TIME".`, false}, - {"other datatype", "0A000", `[UNSUPPORTED_DATATYPE] Unsupported data type "DATE". SELECT TIME '09:15:00'`, false}, - {"other construction", "0A000", "[UNSUPPORTED_TIME_TYPE] The data type DATE is not supported. SELECT TIME '09:15:00'", false}, - {"unavailable config", "42K0I", "[CONFIG_NOT_AVAILABLE.WITHOUT_SUGGESTION] spark.sql.timeType.enabled is not available", false}, - {"other feature", "0A000", "Some other feature is disabled. SELECT TIME '09:15:00'", false}, - {"unauthenticated", "28000", "HTTP 401: " + unsupported, false}, - {"forbidden", "42501", "HTTP 403: " + unsupported, false}, - {"missing state", "", unsupported, false}, - {"scanner failure", "0A000", "databricks: arrow row scanner unhandled type time64[ns]", false}, - } { - t.Run(test.name, func(t *testing.T) { - err := dbsqlerrint.NewExecutionErrorWithState(context.Background(), test.message, nil, "", test.state) - require.Equal(t, test.want, serverTimeUnavailable(err)) - require.Equal(t, test.want, serverTimeUnavailable(fmt.Errorf("query failed: %w", *err))) - }) - } - require.False(t, serverTimeUnavailable(nil)) - require.False(t, serverTimeUnavailable(errors.New(unsupported))) -} diff --git a/time_e2e_test.go b/time_e2e_test.go index d2a3fbd1..c90bc92f 100644 --- a/time_e2e_test.go +++ b/time_e2e_test.go @@ -5,6 +5,7 @@ package dbsql import ( "context" "database/sql" + "fmt" "reflect" "testing" "time" @@ -35,9 +36,6 @@ func assertTimeResults(t *testing.T, db *sql.DB) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() rows, err := db.QueryContext(ctx, "SELECT clock AS typed, clock AS dynamic, clock AS text, CAST(NULL AS TIME) AS missing FROM (SELECT TIME '09:15:00.123456' AS clock)") - if serverTimeUnavailable(err) { - t.Skip("live TIME verification pending: this workspace has not enabled the TIME beta") - } require.NoError(t, err) defer rows.Close() //nolint:errcheck columns, err := rows.ColumnTypes() @@ -68,4 +66,17 @@ func assertTimeResults(t *testing.T, db *sql.DB) { require.JSONEq(t, `["09:15:00.123456",null]`, list) require.JSONEq(t, `{"clock":"09:15:00.123456","missing":null}`, structure) require.JSONEq(t, `{"09:15:00.123456":"09:15:00.123456"}`, mapping) + + for precision, resolution := range []time.Duration{time.Second, 100 * time.Millisecond, 10 * time.Millisecond, time.Millisecond, 100 * time.Microsecond, 10 * time.Microsecond, time.Microsecond} { + t.Run(fmt.Sprintf("TIME(%d)", precision), func(t *testing.T) { + var clock, midnight, missing *time.Time + query := fmt.Sprintf("SELECT CAST(TIME '09:15:00.123456' AS TIME(%d)), CAST(TIME '00:00:00' AS TIME(%d)), CAST(NULL AS TIME(%d))", precision, precision, precision) + require.NoError(t, db.QueryRowContext(ctx, query).Scan(&clock, &midnight, &missing)) + require.NotNil(t, clock) + require.Equal(t, want.Truncate(resolution), *clock) + require.NotNil(t, midnight) + require.Equal(t, time.Time{}, *midnight) + require.Nil(t, missing) + }) + } } From 57fd9cbc9fe173ae8757cec8695ac62134b1d92c Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sat, 14 Nov 2026 09:24:46 +0000 Subject: [PATCH 4/5] Fix TIME metadata for empty kernel results Honor logical TIME names when empty-result manifests use UTF-8 fields. Cover empty and NULL-only metadata on both live backends, and scope TIME beta nightly coverage to the verified DBSQL warehouse. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .github/workflows/nightly-e2e.yml | 4 +- internal/arrowscan/coltype.go | 15 +++++++- internal/arrowscan/coltype_bench_test.go | 29 ++++++++++++++ internal/arrowscan/time_test.go | 49 ++++++++++++++++++++++++ time_e2e_test.go | 42 ++++++++++++++++++++ 5 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 internal/arrowscan/coltype_bench_test.go diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index 441f47f9..da16592f 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -90,7 +90,7 @@ jobs: DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} run: | go test -tags e2e -count=1 -v -timeout 15m \ - -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads|TestE2EDecimalMetadata|TestE2ETime' . + -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads|TestE2EDecimalMetadata|^TestE2ETime(Results|ArrowResults)$' . kernel-e2e: name: E2E (kernel backend) @@ -267,4 +267,4 @@ jobs: run: | CGO_ENABLED=1 go test -tags databricks_kernel,e2e -count=1 -v -timeout 30m \ -run 'TestKernelE2E' \ - -skip 'TestKernelE2EM2M' . + -skip 'TestKernelE2EM2M|^TestKernelE2ETimeResults$' . diff --git a/internal/arrowscan/coltype.go b/internal/arrowscan/coltype.go index 5dc4382d..109a101c 100644 --- a/internal/arrowscan/coltype.go +++ b/internal/arrowscan/coltype.go @@ -150,12 +150,25 @@ func IsBinaryGeospatialField(field arrow.Field) bool { return geospatial && field.Type.ID() == arrow.STRUCT } -// ColumnTypeInfoForField preserves logical timestamp and geospatial types that +// ColumnTypeInfoForField preserves logical time, timestamp and geospatial types that // the kernel's physical Arrow representation alone cannot distinguish. func ColumnTypeInfoForField(field arrow.Field) ColumnTypeInfo { if field.Type != nil && field.Type.ID() == arrow.TIMESTAMP { return timestampColumnTypeInfo(field) } + if field.Type != nil && (field.Type.ID() == arrow.STRING || field.Type.ID() == arrow.LARGE_STRING) { + for _, key := range [...]string{databricksTypeNameMetadataKey, sparkSQLNameMetadataKey} { + name, ok := field.Metadata.GetValue(key) + if !ok || strings.TrimSpace(name) == "" { + continue + } + switch strings.ToUpper(strings.TrimSpace(name)) { + case "TIME", "TIME(0)", "TIME(1)", "TIME(2)", "TIME(3)", "TIME(4)", "TIME(5)", "TIME(6)": + return ColumnTypeInfoFor(arrow.FixedWidthTypes.Time64ns) + } + break + } + } name, geospatial := LogicalGeospatialType(field) if !geospatial { return ColumnTypeInfoFor(field.Type) diff --git a/internal/arrowscan/coltype_bench_test.go b/internal/arrowscan/coltype_bench_test.go new file mode 100644 index 00000000..a8b6c0af --- /dev/null +++ b/internal/arrowscan/coltype_bench_test.go @@ -0,0 +1,29 @@ +package arrowscan + +import ( + "testing" + + "github.com/apache/arrow-go/v18/arrow" +) + +func BenchmarkColumnTypeInfoForField(benchmark *testing.B) { + for _, field := range []arrow.Field{ + {Name: "string", Type: arrow.BinaryTypes.String}, + {Name: "int64", Type: arrow.PrimitiveTypes.Int64}, + {Name: "timestamp", Type: arrow.FixedWidthTypes.Timestamp_us}, + {Name: "TIME manifest", Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{ + databricksTypeNameMetadataKey: "TIME", sparkSQLNameMetadataKey: "TIME(6)", + })}, + } { + benchmark.Run(field.Name, func(benchmark *testing.B) { + benchmark.ReportAllocs() + var info ColumnTypeInfo + for iteration := 0; iteration < benchmark.N; iteration++ { + info = ColumnTypeInfoForField(field) + } + if info.ScanType == nil { + benchmark.Fatal("missing scan type") + } + }) + } +} diff --git a/internal/arrowscan/time_test.go b/internal/arrowscan/time_test.go index c4492ee3..e12c2393 100644 --- a/internal/arrowscan/time_test.go +++ b/internal/arrowscan/time_test.go @@ -1,6 +1,7 @@ package arrowscan import ( + "fmt" "reflect" "testing" "time" @@ -58,3 +59,51 @@ func TestTimeNullOnlyMetadata(t *testing.T) { }) } } + +func TestTimeManifestFieldMetadata(t *testing.T) { + type metadataCase struct { + name string + kernelName string + sparkName string + wantTime bool + } + cases := []metadataCase{ + {"manifest", "TIME", "TIME(6)", true}, + {"kernel only", "TIME", "", true}, + {"case and whitespace", " time ", "", true}, + {"blank kernel name", " ", "TIME", true}, + {"ordinary string", "STRING", "STRING", false}, + {"no metadata", "", "", false}, + {"timestamp", "", "TIMESTAMP", false}, + {"similar name", "", "TIMEZONE", false}, + {"unsupported precision", "", "TIME(7)", false}, + {"kernel name takes precedence", "STRING", "TIME(6)", false}, + } + for precision := range 7 { + name := fmt.Sprintf("TIME(%d)", precision) + cases = append(cases, metadataCase{name, "", name, true}) + } + for _, dataType := range []arrow.DataType{arrow.BinaryTypes.String, arrow.BinaryTypes.LargeString} { + for _, test := range cases { + t.Run(dataType.String()+"/"+test.name, func(t *testing.T) { + metadata := make(map[string]string) + if test.kernelName != "" { + metadata[databricksTypeNameMetadataKey] = test.kernelName + } + if test.sparkName != "" { + metadata[sparkSQLNameMetadataKey] = test.sparkName + } + field := arrow.Field{Name: "clock", Type: dataType, Nullable: true, Metadata: arrow.MetadataFrom(metadata)} + want := ColumnTypeInfoFor(dataType) + if test.wantTime { + want = ColumnTypeInfoFor(arrow.FixedWidthTypes.Time64ns) + } + got := ColumnTypeInfoForField(field) + require.Equal(t, want.DatabaseTypeName, got.DatabaseTypeName) + require.Equal(t, want.ScanType, got.ScanType) + require.Equal(t, want.Length, got.Length) + require.Equal(t, want.HasLength, got.HasLength) + }) + } + } +} diff --git a/time_e2e_test.go b/time_e2e_test.go index c90bc92f..0ff2e0eb 100644 --- a/time_e2e_test.go +++ b/time_e2e_test.go @@ -79,4 +79,46 @@ func assertTimeResults(t *testing.T, db *sql.DB) { require.Nil(t, missing) }) } + assertTimeEmptyResults(t, db) +} + +func assertTimeEmptyResults(t *testing.T, db *sql.DB) { + t.Helper() + for _, test := range []struct { + name string + query string + rows int + }{ + {"empty range", "SELECT TIME '09:15:00' AS clock, CAST(NULL AS TIME) AS missing FROM range(0)", 0}, + {"empty filter", "SELECT TIME '09:15:00' AS clock, CAST(NULL AS TIME) AS missing FROM range(10) WHERE id > 100", 0}, + {"NULL only", "SELECT CAST(NULL AS TIME) AS clock, CAST(NULL AS TIME) AS missing FROM range(3)", 3}, + } { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + rows, err := db.QueryContext(ctx, test.query) + require.NoError(t, err) + defer rows.Close() //nolint:errcheck + columns, err := rows.ColumnTypes() + require.NoError(t, err) + require.Len(t, columns, 2) + for _, column := range columns { + require.Equal(t, "TIME", column.DatabaseTypeName()) + require.Equal(t, reflect.TypeOf(time.Time{}), column.ScanType()) + length, hasLength := column.Length() + require.Zero(t, length) + require.False(t, hasLength) + } + count := 0 + for rows.Next() { + var clock, missing sql.NullTime + require.NoError(t, rows.Scan(&clock, &missing)) + require.False(t, clock.Valid) + require.False(t, missing.Valid) + count++ + } + require.NoError(t, rows.Err()) + require.Equal(t, test.rows, count) + }) + } } From 20f84d30e678fa01a9d2954431cb0547aa6cf8e1 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sat, 14 Nov 2026 09:45:42 +0000 Subject: [PATCH 5/5] Document TIME parameter binding and encoding assumptions Document the supported TIME wire encoding assumption and timezone-safe parameter binding. Exercise string round trips on both live backends. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- README.md | 4 ++++ doc.go | 2 ++ time_e2e_test.go | 7 +++++++ 3 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 41debb60..cff17a51 100644 --- a/README.md +++ b/README.md @@ -382,12 +382,16 @@ use `sql.NullTime` or `*time.Time` for nullable typed values. Non-NULL midnight and has `IsZero() == true`, so detect NULL with `sql.NullTime.Valid`, not `IsZero()`; JSON fields tagged `omitzero` omit midnight. Do not call `.In(location)` on this date-less value: it shifts the clock and may introduce historical timezone offsets. +To bind TIME in `CAST(? AS TIME)`, pass `value.Format("15:04:05.999999")`, not the +`time.Time`: `time.Time` binds as `TIMESTAMP` and can shift the clock in non-UTC sessions. Column metadata reports `TIME` and `time.Time` with no length, even for empty or NULL-only results. Nested TIME values in ARRAY / MAP / STRUCT JSON use clock-only strings with trailing fractional zeros omitted, such as `"09:15:00.5"` rather than `"09:15:00.500000"`; their text can differ from server-rendered TIME(p) values. On Thrift, TIME column metadata requires a server-supplied Arrow schema; without it, the legacy Thrift column metadata is retained. +On SEA, UTF-8 fields tagged TIME describe empty manifest schemas; non-empty TIME +is assumed to arrive as Arrow time32/time64, not UTF-8. Live TIME tests require the `e2e` build tag: `go test -tags e2e -run '^TestE2ETimeResults$' .` or `go test -tags databricks_kernel,e2e -run '^TestKernelE2ETimeResults$' .`. diff --git a/doc.go b/doc.go index 429d8416..aec4b5e1 100644 --- a/doc.go +++ b/doc.go @@ -539,6 +539,8 @@ DATE --> time.Time TIME --> time.Time (0001-01-01 UTC). Detect NULL with sql.NullTime.Valid or a nil *time.Time, never IsZero(): non-NULL midnight is the zero time and JSON omitzero omits it. +To bind TIME, pass value.Format("15:04:05.999999") to CAST(? AS TIME), not the +time.Time: time.Time parameters bind as TIMESTAMP and can shift in non-UTC sessions. TIMESTAMP --> time.Time diff --git a/time_e2e_test.go b/time_e2e_test.go index 0ff2e0eb..2556f92b 100644 --- a/time_e2e_test.go +++ b/time_e2e_test.go @@ -60,6 +60,13 @@ func assertTimeResults(t *testing.T, db *sql.DB) { require.False(t, rows.Next()) require.NoError(t, rows.Err()) + t.Run("TIME parameter as string", func(testState *testing.T) { + var rebound time.Time + err := db.QueryRowContext(ctx, "SELECT CAST(? AS TIME)", typed.Format("15:04:05.999999")).Scan(&rebound) + require.NoError(testState, err) + require.Equal(testState, want, rebound) + }) + var list, structure, mapping string err = db.QueryRowContext(ctx, "SELECT array(TIME '09:15:00.123456', CAST(NULL AS TIME)), named_struct('clock', TIME '09:15:00.123456', 'missing', CAST(NULL AS TIME)), map(TIME '09:15:00.123456', TIME '09:15:00.123456')").Scan(&list, &structure, &mapping) require.NoError(t, err)