diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d92df40..e2581e63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased (v2.0.0 prototype) +- Render non-finite FLOAT / DOUBLE values inside ARRAY / MAP / STRUCT results as the JSON strings `"NaN"`, `"Infinity"`, and `"-Infinity"` on both backends instead of failing row iteration. SQL NULL remains `null`; top-level floats remain numeric (databricks/databricks-sql-go#573). - Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162). - **Behavior change:** Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results. Tools such as ORM migrators can now act on these qualifiers (databricks/databricks-sql-go#500). - **Behavior change:** Support Arrow TIME results on Thrift and SEA as UTC `time.Time` values on `0001-01-01`, preserving precision and using clock-only nested JSON. TIME metadata now reports `TIME`/`time.Time` with no length, including empty and NULL-only results; previously Thrift reported `STRING`/`string` with unbounded length, and SEA reported an empty name with `*interface{}` (databricks/databricks-sql-go#505). diff --git a/README.md b/README.md index c6b98b85..d34f9bfb 100644 --- a/README.md +++ b/README.md @@ -376,6 +376,14 @@ STRUCT and VARIANT (as JSON) are supported. BINARY is returned as `sql.RawBytes` Both backends return GEOMETRY / GEOGRAPHY as WKT / EWKT by default. Set `enableGeoSpatialSupport=true` on the kernel path to opt into EWKB. +In ARRAY / MAP / STRUCT JSON, non-finite FLOAT and DOUBLE values render as the +strings `"NaN"`, `"Infinity"`, and `"-Infinity"`, distinct from SQL NULL (`null`). +Finite values remain JSON numbers. Use `any` or a custom decoder for fields that +can contain special values; `encoding/json` does not unmarshal these strings into +plain `float32` or `float64` fields. Top-level FLOAT / DOUBLE values remain numeric. +Non-finite floating-point MAP keys retain the Go spellings `"NaN"`, `"+Inf"`, and +`"-Inf"` already used by the kernel backend. + | SQL type | Go value | NULL detection | |---|---|---| | `TIME` | `time.Time` on `0001-01-01` UTC | `sql.NullTime.Valid` or a nil `*time.Time`, never `IsZero()`; midnight is the zero time and JSON `omitzero` omits it. | diff --git a/doc.go b/doc.go index bc5f182a..989de155 100644 --- a/doc.go +++ b/doc.go @@ -558,7 +558,11 @@ INTERVAL (year-month) --> string INTERVAL (day-time) --> string For ARRAY, STRUCT, and MAP types, sql.Scan can cast sql.RawBytes to JSON string, which can be unmarshalled to Golang -arrays, maps, and structs. For example: +arrays, maps, and structs. Nested non-finite FLOAT and DOUBLE values are represented as the JSON strings +"NaN", "Infinity", and "-Infinity", distinct from SQL NULL (null). Finite values remain JSON numbers. +Use any or a custom decoder for fields that can contain special values; encoding/json cannot unmarshal +these strings into plain float fields. Top-level FLOAT and DOUBLE values remain numeric. Non-finite +floating-point MAP keys retain the kernel backend's Go spellings "NaN", "+Inf", and "-Inf". For example: type structVal struct { StringField string `json:"string_field"` diff --git a/internal/arrowscan/arrowscan.go b/internal/arrowscan/arrowscan.go index ba44029f..bce4282e 100644 --- a/internal/arrowscan/arrowscan.go +++ b/internal/arrowscan/arrowscan.go @@ -587,6 +587,7 @@ func writeStructJSON(b *strings.Builder, s *array.Struct, row int, loc *time.Loc // base64 "YWJj", NOT fmt "%v" [97 98 99]), then quote-wrap the result if it is // not already a JSON string (numbers/bools become "7"/"true" keys). func writeJSONKey(b *strings.Builder, v any) { + v = jsonfmt.MapKey(v) // A time.Time key renders as its quoted .String() (same special-case as // writeScalarJSON / the Thrift marshalScalar), not RFC3339. if t, ok := v.(time.Time); ok { @@ -610,8 +611,7 @@ func writeJSONKey(b *strings.Builder, v any) { } // writeScalarJSON writes a scalar leaf, mirroring the Thrift marshalScalar: a -// time.Time becomes a quoted .String(); everything else renders as json.Marshal -// would. +// time.Time becomes a quoted .String(); everything else uses jsonfmt.Append. func writeScalarJSON(b *strings.Builder, v any) error { if v == nil { b.WriteString("null") diff --git a/internal/jsonfmt/jsonfmt.go b/internal/jsonfmt/jsonfmt.go index 5b24c282..978e88a3 100644 --- a/internal/jsonfmt/jsonfmt.go +++ b/internal/jsonfmt/jsonfmt.go @@ -1,12 +1,12 @@ -// Package jsonfmt renders scalar values as JSON, byte-identical to -// encoding/json.Marshal, for the nested-type renderers shared by the Thrift -// (arrowbased) and kernel (arrowscan) result paths. +// Package jsonfmt renders scalar values as JSON for the nested-type renderers +// shared by the Thrift (arrowbased) and kernel (arrowscan) result paths. +// It matches encoding/json.Marshal except that non-finite floats become the +// JSON strings "NaN", "Infinity", and "-Infinity" rather than errors. // // Those renderers marshal one scalar per nested element, so per-call overhead // dominates. Go 1.27 routes encoding/json through the json/v2 engine by default, // which made that per-call cost markedly higher; formatting the common scalar -// types directly avoids it. Anything not handled here falls back to json.Marshal, -// so output never diverges from it. +// types directly avoids it. Anything not handled here falls back to json.Marshal. package jsonfmt import ( @@ -16,8 +16,8 @@ import ( "unicode/utf8" ) -// Append appends the JSON encoding of v to dst, exactly as json.Marshal(v) -// would render it, and returns the extended slice. +// Append appends the JSON encoding of v to dst and returns the extended slice. +// Unlike json.Marshal, it quotes non-finite floats instead of rejecting them. func Append(dst []byte, v any) ([]byte, error) { switch t := v.(type) { case nil: @@ -43,14 +43,8 @@ func Append(dst []byte, v any) ([]byte, error) { case uint64: return strconv.AppendUint(dst, t, 10), nil case float32: - if !isFinite(float64(t)) { - break - } return appendFloat(dst, float64(t), 32), nil case float64: - if !isFinite(t) { - break - } return appendFloat(dst, t, 64), nil case string: if needsEscape(t) { @@ -60,8 +54,7 @@ func Append(dst []byte, v any) ([]byte, error) { dst = append(dst, t...) return append(dst, '"'), nil } - // NaN/Inf (json.Marshal's error), strings needing escapes, []byte (base64), - // and any other type. + // Strings needing escapes, []byte (base64), and any other type. b, err := json.Marshal(v) if err != nil { return dst, err @@ -72,13 +65,42 @@ func Append(dst []byte, v any) ([]byte, error) { return append(dst, b...), nil } -func isFinite(f float64) bool { - return !math.IsInf(f, 0) && !math.IsNaN(f) +// MapKey returns the scalar to encode for a MAP key. Non-finite float keys keep +// the Go spellings used by the kernel renderer before values supported them. +func MapKey(v any) any { + var f float64 + switch value := v.(type) { + case float32: + f = float64(value) + case float64: + f = value + default: + return v + } + if math.IsNaN(f) { + return "NaN" + } + if math.IsInf(f, 0) { + if f < 0 { + return "-Inf" + } + return "+Inf" + } + return v } -// appendFloat mirrors encoding/json's ES6-style float formatting: plain decimal -// unless the magnitude is below 1e-6 or at least 1e21, with exponents unpadded. +// appendFloat quotes non-finite values to keep valid SQL floats distinct from +// SQL NULL in JSON. Finite values retain encoding/json's ES6-style formatting. func appendFloat(dst []byte, f float64, bits int) []byte { + if math.IsNaN(f) { + return append(dst, `"NaN"`...) + } + if math.IsInf(f, 0) { + if f < 0 { + return append(dst, `"-Infinity"`...) + } + return append(dst, `"Infinity"`...) + } abs := math.Abs(f) format := byte('f') // Compare at the value's own width so float32 cutoffs match exactly. diff --git a/internal/jsonfmt/jsonfmt_test.go b/internal/jsonfmt/jsonfmt_test.go index 028ecc1c..5f3ea38e 100644 --- a/internal/jsonfmt/jsonfmt_test.go +++ b/internal/jsonfmt/jsonfmt_test.go @@ -33,7 +33,6 @@ func TestMarshalMatchesEncodingJSON(t *testing.T) { math.MaxFloat64, math.SmallestNonzeroFloat64, -math.MaxFloat64, float32(0), float32(math.Copysign(0, -1)), float32(3.14), float32(0.1), float32(1e21), float32(1e-6), float32(1e-7), float32(math.MaxFloat32), float32(math.SmallestNonzeroFloat32), - math.NaN(), math.Inf(1), math.Inf(-1), float32(math.NaN()), float32(math.Inf(-1)), "", "alpha", "with space", "a\"b", `back\slash`, "", "a&b", "tab\there", "nl\n", "\x00", "\x7f", "héllo", "日本語", "

", "\xff\xfe", "emoji 😀", []byte("abc"), []byte(nil), @@ -89,13 +88,59 @@ func TestMarshalRandomUnicodeStrings(t *testing.T) { func TestMarshalRandomFloats(t *testing.T) { r := rand.New(rand.NewSource(1)) //nolint:gosec // G404: deterministic test inputs, not security for i := 0; i < 200000; i++ { - assertParity(t, math.Float64frombits(r.Uint64())) - assertParity(t, math.Float32frombits(r.Uint32())) + f64 := math.Float64frombits(r.Uint64()) + if !math.IsNaN(f64) && !math.IsInf(f64, 0) { + assertParity(t, f64) + } + f32 := math.Float32frombits(r.Uint32()) + if !math.IsNaN(float64(f32)) && !math.IsInf(float64(f32), 0) { + assertParity(t, f32) + } assertParity(t, r.NormFloat64()*math.Pow(10, float64(r.Intn(50)-25))) assertParity(t, float32(r.NormFloat64()*math.Pow(10, float64(r.Intn(30)-15)))) } } +func TestAppendNonFiniteFloats(t *testing.T) { + cases := []struct { + name string + value any + want string + }{ + {"float32_nan", float32(math.NaN()), `"NaN"`}, + {"float32_infinity", float32(math.Inf(1)), `"Infinity"`}, + {"float32_negative_infinity", float32(math.Inf(-1)), `"-Infinity"`}, + {"float64_nan", math.NaN(), `"NaN"`}, + {"float64_infinity", math.Inf(1), `"Infinity"`}, + {"float64_negative_infinity", math.Inf(-1), `"-Infinity"`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + for _, prefix := range []string{"", "[1.5,"} { + got, err := Append([]byte(prefix), tc.value) + if err != nil || string(got) != prefix+tc.want { + t.Fatalf("Append(%q, %v) = %q, %v; want %q", prefix, tc.value, got, err, prefix+tc.want) + } + } + }) + } +} + +func TestMapKeyPreservesEncoding(t *testing.T) { + values := []any{nil, true, int32(7), int64(-9), "NaN", "+Inf", "key", float32(1.5), + float32(math.Copysign(0, -1)), 1.5, math.Copysign(0, -1), math.MaxFloat64} + for _, value := range values { + want, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + got, err := Append(nil, MapKey(value)) + if err != nil || string(got) != string(want) { + t.Errorf("MapKey(%v): got %q, %v; want %s", value, got, err, want) + } + } +} + func TestAppendKeepsPrefix(t *testing.T) { got, err := Append([]byte("["), int32(7)) if err != nil || string(got) != "[7" { @@ -112,6 +157,7 @@ func TestAppendKeepsPrefix(t *testing.T) { func TestAppendFastPathDoesNotAllocate(t *testing.T) { buf := make([]byte, 0, 64) for _, v := range []any{int64(-9000), int32(42), uint64(7), 3.14, 0.30000000000000004, 1e-7, float32(0.5), true, + math.NaN(), math.Inf(1), math.Inf(-1), float32(math.NaN()), float32(math.Inf(1)), float32(math.Inf(-1)), "alpha", "café 日本語 😀"} { if n := testing.AllocsPerRun(100, func() { buf, _ = Append(buf[:0], v) }); n != 0 { t.Errorf("Append(%#v) allocated %v times per run, want 0", v, n) @@ -121,7 +167,7 @@ func TestAppendFastPathDoesNotAllocate(t *testing.T) { // jsonfmt appends into a reused buffer, as both renderers do. func BenchmarkAppendScalars(b *testing.B) { - // The last three take json.Marshal's fallback. + // The escaped string and []byte take json.Marshal's fallback. values := []any{int32(42), int64(-9000), 3.14, 0.30000000000000004, float32(0.5), true, "alpha0", "café crème", `a"b`, []byte("abc"), math.NaN()} b.Run("jsonfmt", func(b *testing.B) { diff --git a/internal/rows/arrowbased/columnValues.go b/internal/rows/arrowbased/columnValues.go index 9a629b34..0351fbef 100644 --- a/internal/rows/arrowbased/columnValues.go +++ b/internal/rows/arrowbased/columnValues.go @@ -293,7 +293,7 @@ func (mvc *mapValueContainer) Value(i int) (any, error) { return nil, err } - key, err := marshalScalar(keyScratch[:0], mvc.keys, int(i+s), k) + key, err := marshalScalar(keyScratch[:0], mvc.keys, int(i+s), jsonfmt.MapKey(k)) if err != nil { return nil, err } diff --git a/internal/rows/arrowbased/nonfinite_test.go b/internal/rows/arrowbased/nonfinite_test.go new file mode 100644 index 00000000..d3d513a9 --- /dev/null +++ b/internal/rows/arrowbased/nonfinite_test.go @@ -0,0 +1,170 @@ +package arrowbased + +import ( + "encoding/json" + "math" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/databricks/databricks-sql-go/internal/arrowscan" +) + +func TestNonFiniteFloatRendering(t *testing.T) { + for _, dt := range []arrow.DataType{arrow.PrimitiveTypes.Float32, arrow.PrimitiveTypes.Float64} { + t.Run(dt.Name(), func(t *testing.T) { + pool := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer pool.AssertSize(t, 0) + values := []float64{1.5, math.NaN(), math.Inf(1), math.Inf(-1)} + names := []string{"finite", "nan", "positive", "negative"} + appendValue := func(b array.Builder, value float64) { + switch b := b.(type) { + case *array.Float32Builder: + b.Append(float32(value)) + case *array.Float64Builder: + b.Append(value) + default: + t.Fatalf("unexpected float builder %T", b) + } + } + appendListValues := func(b array.Builder) { + for _, value := range values { + appendValue(b, value) + } + b.AppendNull() + } + t.Run("top_level", func(t *testing.T) { + b := array.NewBuilder(pool, dt) + defer b.Release() + appendListValues(b) + arr := b.NewArray() + defer arr.Release() + holder, err := (&arrowValueContainerMaker{}).makeColumnValueContainer(dt, time.UTC, nil, nil) + if err != nil { + t.Fatal(err) + } + defer holder.Release() + if err := holder.SetValueArray(arr.Data()); err != nil { + t.Fatal(err) + } + for row, want := range values { + thrift, thriftErr := holder.Value(row) + kernel, kernelErr := arrowscan.ScanCell(arr, row, time.UTC) + if thriftErr != nil || kernelErr != nil { + t.Fatalf("row %d: thrift error %v, kernel error %v", row, thriftErr, kernelErr) + } + for _, value := range []any{thrift, kernel} { + var got float64 + switch value := value.(type) { + case float32: + if dt.ID() != arrow.FLOAT32 { + t.Fatalf("unexpected float32 for %s", dt) + } + got = float64(value) + case float64: + if dt.ID() != arrow.FLOAT64 { + t.Fatalf("unexpected float64 for %s", dt) + } + got = value + default: + t.Fatalf("top-level value has non-numeric type %T", value) + } + if got != want && !(math.IsNaN(got) && math.IsNaN(want)) { + t.Errorf("row %d: got %v, want %v", row, got, want) + } + } + } + kernelNull, err := arrowscan.ScanCell(arr, len(values), time.UTC) + if !holder.IsNull(len(values)) || kernelNull != nil || err != nil { + t.Fatalf("top-level NULL changed: kernel %v, %v", kernelNull, err) + } + }) + const listJSON = `[1.5,"NaN","Infinity","-Infinity",null]` + cases := []struct { + name string + build func() arrow.Array + want string + }{ + {"array", func() arrow.Array { + b := array.NewListBuilder(pool, dt) + defer b.Release() + b.Append(true) + appendListValues(b.ValueBuilder()) + return b.NewArray() + }, listJSON}, + {"map_values", func() arrow.Array { + b := array.NewMapBuilder(pool, arrow.BinaryTypes.String, dt, false) + defer b.Release() + b.Append(true) + b.KeyBuilder().(*array.StringBuilder).AppendValues(append(names, "null"), nil) + appendListValues(b.ItemBuilder()) + return b.NewArray() + }, `{"finite":1.5,"nan":"NaN","positive":"Infinity","negative":"-Infinity","null":null}`}, + {"map_keys", func() arrow.Array { + b := array.NewMapBuilder(pool, dt, arrow.BinaryTypes.String, false) + defer b.Release() + b.Append(true) + for _, value := range values { + appendValue(b.KeyBuilder(), value) + } + b.ItemBuilder().(*array.StringBuilder).AppendValues(names, nil) + return b.NewArray() + }, `{"1.5":"finite","NaN":"nan","+Inf":"positive","-Inf":"negative"}`}, + {"struct", func() arrow.Array { + fields := make([]arrow.Field, 0, len(names)+1) + for _, name := range append(names, "null") { + fields = append(fields, arrow.Field{Name: name, Type: dt, Nullable: true}) + } + b := array.NewStructBuilder(pool, arrow.StructOf(fields...)) + defer b.Release() + b.Append(true) + for i, value := range values { + appendValue(b.FieldBuilder(i), value) + } + b.FieldBuilder(len(values)).AppendNull() + return b.NewArray() + }, `{"finite":1.5,"nan":"NaN","positive":"Infinity","negative":"-Infinity","null":null}`}, + {"nested", func() arrow.Array { + b := array.NewStructBuilder(pool, arrow.StructOf(arrow.Field{ + Name: "values", Type: arrow.ListOf(dt), Nullable: true, + })) + defer b.Release() + b.Append(true) + list := b.FieldBuilder(0).(*array.ListBuilder) + list.Append(true) + appendListValues(list.ValueBuilder()) + return b.NewArray() + }, `{"values":` + listJSON + `}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + arr := tc.build() + defer arr.Release() + holder, err := (&arrowValueContainerMaker{}).makeColumnValueContainer(arr.DataType(), time.UTC, nil, nil) + if err != nil { + t.Fatal(err) + } + defer holder.Release() + if err := holder.SetValueArray(arr.Data()); err != nil { + t.Fatal(err) + } + thrift, thriftErr := holder.Value(0) + kernel, kernelErr := arrowscan.ScanCell(arr, 0, time.UTC) + for _, result := range []struct { + backend string + value any + err error + }{{"thrift", thrift, thriftErr}, {"kernel", kernel, kernelErr}} { + if result.err != nil || result.value != tc.want { + t.Errorf("%s: got %v, %v; want %s", result.backend, result.value, result.err, tc.want) + } else if !json.Valid([]byte(result.value.(string))) { + t.Errorf("%s: invalid JSON: %s", result.backend, result.value) + } + } + }) + } + }) + } +}