From b40427531c2fd8cce52c45ee08217e06101e52d4 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sat, 14 Nov 2026 03:27:42 +0000 Subject: [PATCH 1/2] fix: report DECIMAL precision and scale in result metadata Expose the optional precision/scale interface on Thrift and SEA rows using their existing declared schema. Cover populated and empty results, decimal boundaries, manifest qualifiers, and non-decimal columns. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- CHANGELOG.md | 1 + driver_e2e_test.go | 75 +++++++++++ .../backend/kernel/decimal_metadata_test.go | 113 ++++++++++++++++ internal/backend/kernel/rows.go | 33 +++++ internal/rows/decimal_metadata_test.go | 122 ++++++++++++++++++ internal/rows/rows.go | 19 +++ kernel_e2e_test.go | 18 +++ 7 files changed, 381 insertions(+) create mode 100644 internal/backend/kernel/decimal_metadata_test.go create mode 100644 internal/rows/decimal_metadata_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b4f0466d..86a56d53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased (v2.0.0 prototype) - Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162). +- Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results (databricks/databricks-sql-go#500). - **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication. - Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439). - **Behavior change:** On the Thrift backend, `ExecuteStatement` is now retried (per `WithRetries`) when the connection to the server or proxy can't be opened, for example connection refused or a connect timeout, because the statement was never sent. Failures after the connection exists are still not retried, so a statement never runs twice. With default retry settings, a refused connection now fails after about 15 seconds instead of immediately (databricks/databricks-sql-go#423). diff --git a/driver_e2e_test.go b/driver_e2e_test.go index 17e76c37..5799eb5e 100644 --- a/driver_e2e_test.go +++ b/driver_e2e_test.go @@ -33,6 +33,81 @@ import ( "github.com/stretchr/testify/require" ) +func TestDecimalMetadataThriftE2E(t *testing.T) { + host, httpPath, token := pecoTestingCreds(t) + for _, native := range []bool{false, true} { + t.Run(fmt.Sprintf("native=%t", native), func(t *testing.T) { + connector, err := NewConnector( + WithServerHostname(host), WithHTTPPath(httpPath), WithAccessToken(token), + WithArrowNativeDecimal(native), + ) + require.NoError(t, err) + db := sql.OpenDB(connector) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + testDecimalResultMetadata(t, db) + }) + } +} + +func testDecimalResultMetadata(t *testing.T, db *sql.DB) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + const projection = "SELECT CAST(3.5 AS DECIMAL(10, 2)) AS amount, " + + "CAST(1 AS DECIMAL(38, 0)) AS whole, CAST(0.1 AS DECIMAL(38, 38)) AS all_fraction" + for _, empty := range []bool{false, true} { + t.Run(fmt.Sprintf("empty=%t", empty), func(t *testing.T) { + query := projection + if empty { + query += " WHERE 1 = 0" + } + result, err := db.QueryContext(ctx, query) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, result.Close()) }) + columns, err := result.ColumnTypes() + require.NoError(t, err) + require.Len(t, columns, 3) + for index, pair := range [][2]int64{{10, 2}, {38, 0}, {38, 38}} { + precision, scale, ok := columns[index].DecimalSize() + require.True(t, ok, "%s must report DECIMAL qualifiers", columns[index].Name()) + require.Equal(t, pair[0], precision) + require.Equal(t, pair[1], scale) + } + count := 0 + for result.Next() { + var amount float64 + var whole, fraction any + require.NoError(t, result.Scan(&amount, &whole, &fraction)) + require.Equal(t, 3.5, amount) + count++ + } + require.NoError(t, result.Err()) + if empty { + require.Zero(t, count) + } else { + require.Equal(t, 1, count) + } + require.NoError(t, result.Close()) + }) + } + result, err := db.QueryContext(ctx, "SELECT CAST(1 AS DEC(10, 2)), CAST(1 AS NUMERIC(38, 0)), 1, 'text', 1.5D") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, result.Close()) }) + columns, err := result.ColumnTypes() + require.NoError(t, err) + require.Len(t, columns, 5) + for index, column := range columns { + precision, scale, ok := column.DecimalSize() + if index < 2 { + require.True(t, ok) + require.Equal(t, [][2]int64{{10, 2}, {38, 0}}[index], [2]int64{precision, scale}) + } else { + require.False(t, ok, "%s is not DECIMAL", column.DatabaseTypeName()) + } + } + require.NoError(t, result.Close()) +} + func TestWorkflowExample(t *testing.T) { // these are global variables that get changed depending on the the expected response diff --git a/internal/backend/kernel/decimal_metadata_test.go b/internal/backend/kernel/decimal_metadata_test.go new file mode 100644 index 00000000..ad515e9d --- /dev/null +++ b/internal/backend/kernel/decimal_metadata_test.go @@ -0,0 +1,113 @@ +//go:build cgo && databricks_kernel + +package kernel + +import ( + "database/sql/driver" + "fmt" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/stretchr/testify/require" +) + +func TestDecimalMetadataKernelSchema(t *testing.T) { + for _, rowCount := range []int{0, 1} { + for _, text := range []bool{false, true} { + t.Run(fmt.Sprintf("rows=%d/text=%t", rowCount, text), func(t *testing.T) { + pairs := [][2]int32{{10, 2}, {38, 0}, {38, 38}} + fields := make([]arrow.Field, len(pairs)) + for index, pair := range pairs { + fields[index] = arrow.Field{Type: &arrow.Decimal128Type{Precision: pair[0], Scale: pair[1]}} + if text { + fields[index].Type = arrow.BinaryTypes.String + fields[index].Metadata = arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_precision": fmt.Sprint(pair[0]), "databricks.type_scale": fmt.Sprint(pair[1]), + }) + } + } + schema := arrow.NewSchema(fields, nil) + resultRows := &kernelRows{schema: schema} + if rowCount > 0 { + builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) + defer builder.Release() + for _, field := range builder.Fields() { + field.AppendNull() + } + resultRows.cur = builder.NewRecordBatch() + defer resultRows.cur.Release() + } + decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale) + require.True(t, supported, "kernel rows must expose declared DECIMAL qualifiers") + for index, pair := range pairs { + precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index) + require.True(t, ok) + require.Equal(t, int64(pair[0]), precision) + require.Equal(t, int64(pair[1]), scale) + } + }) + } + } +} + +func TestDecimalMetadataKernelUnavailable(t *testing.T) { + for name, field := range map[string]arrow.Field{ + "integer": {Type: arrow.PrimitiveTypes.Int64}, + "string": {Type: arrow.BinaryTypes.String}, + "float": {Type: arrow.PrimitiveTypes.Float64}, + "nested": {Type: arrow.ListOf(&arrow.Decimal128Type{Precision: 10, Scale: 2})}, + "missing_scale": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_precision": "10", + })}, + "malformed_precision": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_precision": "bad", "databricks.type_scale": "2", + })}, + "malformed_scale": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_precision": "10", "databricks.type_scale": "bad", + })}, + "missing_precision": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_scale": "0", + })}, + "non_decimal_qualifiers": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "STRING", "databricks.type_precision": "10", "databricks.type_scale": "2", + })}, + } { + t.Run(name, func(t *testing.T) { + resultRows := &kernelRows{schema: arrow.NewSchema([]arrow.Field{field}, nil)} + decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale) + require.True(t, supported) + for _, index := range []int{-1, 0, 1} { + precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index) + require.Equal(t, int64(0), precision) + require.Equal(t, int64(0), scale) + require.False(t, ok) + } + }) + } +} + +func TestDecimalMetadataKernelManifestPriority(t *testing.T) { + for _, precisionText := range []string{"38", "invalid"} { + t.Run(precisionText, func(t *testing.T) { + field := arrow.Field{ + Type: &arrow.Decimal128Type{Precision: 10, Scale: 2}, + Metadata: arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_precision": precisionText, "databricks.type_scale": "0", + }), + } + resultRows := &kernelRows{schema: arrow.NewSchema([]arrow.Field{field}, nil)} + decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale) + require.True(t, supported) + precision, scale, ok := decimalRows.ColumnTypePrecisionScale(0) + require.True(t, ok) + if precisionText == "invalid" { + require.Equal(t, int64(10), precision) + } else { + require.Equal(t, int64(38), precision) + } + require.Equal(t, int64(0), scale) + }) + } +} diff --git a/internal/backend/kernel/rows.go b/internal/backend/kernel/rows.go index aa662f55..b8906d18 100644 --- a/internal/backend/kernel/rows.go +++ b/internal/backend/kernel/rows.go @@ -18,6 +18,8 @@ import ( "fmt" "io" "reflect" + "strconv" + "strings" "time" "unsafe" @@ -40,6 +42,7 @@ var ( _ driver.RowsColumnTypeDatabaseTypeName = (*kernelRows)(nil) _ driver.RowsColumnTypeNullable = (*kernelRows)(nil) _ driver.RowsColumnTypeLength = (*kernelRows)(nil) + _ driver.RowsColumnTypePrecisionScale = (*kernelRows)(nil) // Public batch API, matching the Thrift path (rows.rows). GetArrowBatches is // real; GetArrowIPCStreams rejects (kernel C ABI exports C Data, not IPC bytes). _ dbrows.Rows = (*kernelRows)(nil) @@ -178,6 +181,36 @@ func (r *kernelRows) ColumnTypeNullable(index int) (nullable, ok bool) { return false, false } +func (r *kernelRows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { + if r.schema == nil || index < 0 || index >= r.schema.NumFields() { + return 0, 0, false + } + field := r.schema.Field(index) + decimalType, native := field.Type.(arrow.DecimalType) + name, _ := field.Metadata.GetValue("databricks.type_name") + if !native && !strings.EqualFold(strings.TrimSpace(name), "DECIMAL") { + return 0, 0, false + } + hasPrecision, hasScale := native, native + if native { + precision, scale = int64(decimalType.GetPrecision()), int64(decimalType.GetScale()) + } + if text, present := field.Metadata.GetValue("databricks.type_precision"); present { + if value, err := strconv.ParseInt(text, 10, 32); err == nil && value > 0 { + precision, hasPrecision = value, true + } + } + if text, present := field.Metadata.GetValue("databricks.type_scale"); present { + if value, err := strconv.ParseInt(text, 10, 32); err == nil { + scale, hasScale = value, true + } + } + if !hasPrecision || !hasScale { + return 0, 0, false + } + return precision, scale, true +} + // ColumnTypeLength returns a variable-length column's length (math.MaxInt64, // unbounded) for string/binary/nested/interval types and (0, false) for // fixed-width types, matching the Thrift path. An out-of-range index returns diff --git a/internal/rows/decimal_metadata_test.go b/internal/rows/decimal_metadata_test.go new file mode 100644 index 00000000..2c80c5d7 --- /dev/null +++ b/internal/rows/decimal_metadata_test.go @@ -0,0 +1,122 @@ +package rows + +import ( + "context" + "database/sql/driver" + "fmt" + "io" + "testing" + + "github.com/databricks/databricks-sql-go/internal/cli_service" + "github.com/databricks/databricks-sql-go/internal/client" + "github.com/databricks/databricks-sql-go/internal/config" + "github.com/stretchr/testify/require" +) + +func decimalMetadataColumn(typeID cli_service.TTypeId, precision, scale *int32) *cli_service.TColumnDesc { + return &cli_service.TColumnDesc{TypeDesc: &cli_service.TTypeDesc{Types: []*cli_service.TTypeEntry{{ + PrimitiveEntry: &cli_service.TPrimitiveTypeEntry{ + Type: typeID, + TypeQualifiers: &cli_service.TTypeQualifiers{Qualifiers: map[string]*cli_service.TTypeQualifierValue{ + "precision": {I32Value: precision}, "scale": {I32Value: scale}, + }}, + }, + }}}} +} + +func TestDecimalMetadataThriftResults(t *testing.T) { + for _, rowCount := range []int{0, 1} { + for _, direct := range []bool{false, true} { + t.Run(fmt.Sprintf("rows=%d/direct=%t", rowCount, direct), func(t *testing.T) { + pairs := [][2]int32{{10, 2}, {38, 0}, {38, 38}} + schema := &cli_service.TTableSchema{} + result := &cli_service.TRowSet{} + for _, pair := range pairs { + precision, scale := pair[0], pair[1] + schema.Columns = append(schema.Columns, decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, &precision, &scale)) + values := []string{} + if rowCount > 0 { + values = append(values, "1") + } + result.Columns = append(result.Columns, &cli_service.TColumn{StringVal: &cli_service.TStringColumn{Values: values, Nulls: []byte{0}}}) + } + metadata := &cli_service.TGetResultSetMetadataResp{Schema: schema} + hasMore := false + fetch := &cli_service.TFetchResultsResp{Results: result, HasMoreRows: &hasMore} + metadataCalls := 0 + testClient := &client.TestClient{ + FnCloseOperation: func(context.Context, *cli_service.TCloseOperationReq) (*cli_service.TCloseOperationResp, error) { + return &cli_service.TCloseOperationResp{}, nil + }, + FnGetResultSetMetadata: func(context.Context, *cli_service.TGetResultSetMetadataReq) (*cli_service.TGetResultSetMetadataResp, error) { + metadataCalls++ + return metadata, nil + }, + FnFetchResults: func(context.Context, *cli_service.TFetchResultsReq) (*cli_service.TFetchResultsResp, error) { + return fetch, nil + }, + } + var directResults *cli_service.TSparkDirectResults + if direct { + directResults = &cli_service.TSparkDirectResults{ResultSetMetadata: metadata, ResultSet: fetch} + } + resultRows, err := NewRows(context.Background(), nil, testClient, config.WithDefaults(), directResults, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, resultRows.Close()) }) + decimalRows, supported := resultRows.(driver.RowsColumnTypePrecisionScale) + require.True(t, supported, "rows must expose declared DECIMAL qualifiers") + check := func() { + for index, pair := range pairs { + precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index) + require.True(t, ok) + require.Equal(t, int64(pair[0]), precision) + require.Equal(t, int64(pair[1]), scale) + } + } + check() + values := make([]driver.Value, len(pairs)) + for range rowCount { + require.NoError(t, resultRows.Next(values)) + } + require.ErrorIs(t, resultRows.Next(values), io.EOF) + check() + if direct { + require.Zero(t, metadataCalls) + } else { + require.Equal(t, 1, metadataCalls) + } + }) + } + } +} + +func TestDecimalMetadataThriftUnavailable(t *testing.T) { + precision, scale := int32(10), int32(2) + noQualifiers := decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, nil, nil) + noQualifiers.TypeDesc.Types[0].PrimitiveEntry.TypeQualifiers = nil + emptyQualifiers := decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, nil, nil) + emptyQualifiers.TypeDesc.Types[0].PrimitiveEntry.TypeQualifiers.Qualifiers = nil + for name, column := range map[string]*cli_service.TColumnDesc{ + "integer": decimalMetadataColumn(cli_service.TTypeId_INT_TYPE, &precision, &scale), + "string": decimalMetadataColumn(cli_service.TTypeId_STRING_TYPE, &precision, &scale), + "float": decimalMetadataColumn(cli_service.TTypeId_DOUBLE_TYPE, &precision, &scale), + "nested": decimalMetadataColumn(cli_service.TTypeId_ARRAY_TYPE, &precision, &scale), + "missing_precision": decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, nil, &scale), + "missing_scale": decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, &precision, nil), + "missing_both": decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, nil, nil), + "no_qualifiers": noQualifiers, + "empty_qualifiers": emptyQualifiers, + } { + t.Run(name, func(t *testing.T) { + resultRows := &rows{client: &client.TestClient{}, schema: &cli_service.TTableSchema{Columns: []*cli_service.TColumnDesc{column}}} + decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale) + require.True(t, supported) + for _, index := range []int{-1, 0, 1} { + precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index) + require.Equal(t, int64(0), precision) + require.Equal(t, int64(0), scale) + require.False(t, ok) + } + }) + } +} diff --git a/internal/rows/rows.go b/internal/rows/rows.go index 7cf7df50..cffc6367 100644 --- a/internal/rows/rows.go +++ b/internal/rows/rows.go @@ -33,6 +33,7 @@ import ( // RowsColumnTypeDatabaseTypeName // RowsColumnTypeNullable // RowsColumnTypeLength +// RowsColumnTypePrecisionScale type rows struct { // The RowScanner is responsible for handling the different // formats in which the query results can be returned @@ -102,6 +103,7 @@ var _ driver.RowsColumnTypeScanType = (*rows)(nil) var _ driver.RowsColumnTypeDatabaseTypeName = (*rows)(nil) var _ driver.RowsColumnTypeNullable = (*rows)(nil) var _ driver.RowsColumnTypeLength = (*rows)(nil) +var _ driver.RowsColumnTypePrecisionScale = (*rows)(nil) var _ dbsqlrows.Rows = (*rows)(nil) // TelemetryCallbacks bundles the optional telemetry hooks passed into NewRows. @@ -498,6 +500,23 @@ func (r *rows) ColumnTypeNullable(index int) (nullable, ok bool) { return false, false } +func (r *rows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { + column, err := r.getColumnMetadataByIndex(index) + if err != nil || rowscanner.GetDBTypeID(column) != cli_service.TTypeId_DECIMAL_TYPE { + return 0, 0, false + } + qualifiers := rowscanner.GetDBTypeQualifiers(column) + if qualifiers == nil { + return 0, 0, false + } + precisionValue := qualifiers.Qualifiers[cli_service.PRECISION] + scaleValue := qualifiers.Qualifiers[cli_service.SCALE] + if precisionValue == nil || !precisionValue.IsSetI32Value() || scaleValue == nil || !scaleValue.IsSetI32Value() { + return 0, 0, false + } + return int64(precisionValue.GetI32Value()), int64(scaleValue.GetI32Value()), true +} + func (r *rows) ColumnTypeLength(index int) (length int64, ok bool) { columnInfo, err := r.getColumnMetadataByIndex(index) if err != nil { diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index 074d6314..a82f7746 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -29,6 +29,24 @@ func kernelTestDB(t *testing.T) *sql.DB { return kernelTestDBWith(t) } +func TestDecimalMetadataKernelE2E(t *testing.T) { + for _, native := range []bool{false, true} { + name := "arrow_native_decimal=false" + if native { + name = "arrow_native_decimal=true" + } + t.Run(name, func(t *testing.T) { + db := kernelTestDBWith(t, WithArrowNativeDecimal(native)) + t.Cleanup(func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }) + testDecimalResultMetadata(t, db) + }) + } +} + // TestKernelE2ESelect1 is the smallest end-to-end proof: PAT session over the // kernel, execute, scan one scalar row. func TestKernelE2ESelect1(t *testing.T) { From acf15cfa6ebe2e34478dc98a333219dcd53d3bdf Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Sat, 14 Nov 2026 05:55:01 +0000 Subject: [PATCH 2/2] fix: align decimal metadata with kernel schema and nightly coverage Limit SEA qualifiers to the native Decimal128 schema, keep metadata consistent, and exercise the effective decimal scan options. Include both live tests and DecimalSize parity in nightly coverage. Signed-off-by: Vu Anh Phung Co-authored-by: Isaac --- .github/workflows/nightly-e2e.yml | 6 +- CHANGELOG.md | 2 +- driver_e2e_test.go | 2 +- .../backend/kernel/decimal_metadata_test.go | 91 +++++++++---------- internal/backend/kernel/rows.go | 31 +------ kernel_e2e_test.go | 12 +-- kernel_parity_test.go | 9 +- 7 files changed, 65 insertions(+), 88 deletions(-) diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index ed170188..41b8e1db 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -78,7 +78,7 @@ jobs: # in the default pure-Go build (CGO_ENABLED=0). -run targets the high-value # real-warehouse scenarios: large multi-page CloudFetch drain-past-caller- # deadline, exact row count, and a non-positive download-thread count - # (issue #439). -count=1 defeats the test cache so a scheduled + # (issue #439), and DECIMAL metadata (issue #500). -count=1 defeats the test cache so a scheduled # run always re-exercises the warehouse. NOT -short, so # TestE2ECloudFetchExactRowCount (guarded by -short) actually runs. - name: Run Thrift E2E @@ -90,7 +90,7 @@ jobs: DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} run: | go test -count=1 -v -timeout 15m \ - -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads' . + -run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads|TestE2EDecimalMetadata' . kernel-e2e: name: E2E (kernel backend) @@ -244,7 +244,7 @@ jobs: DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }} run: | CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \ - -run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \ + -run 'TestKernelE2E|TestKernelThriftParity|TestKernelThriftColumnTypeParity|TestKernelParamsVsThrift' \ -skip 'TestKernelE2EM2M' . # Non-blocking Reyden reference-engine leg: reruns the TestKernelE2E* funcs against diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a56d53..77a60de2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased (v2.0.0 prototype) - Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162). -- Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results (databricks/databricks-sql-go#500). +- **Behavior change:** Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results. Tools such as ORM migrators can now act on these qualifiers (databricks/databricks-sql-go#500). - **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication. - Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439). - **Behavior change:** On the Thrift backend, `ExecuteStatement` is now retried (per `WithRetries`) when the connection to the server or proxy can't be opened, for example connection refused or a connect timeout, because the statement was never sent. Failures after the connection exists are still not retried, so a statement never runs twice. With default retry settings, a refused connection now fails after about 15 seconds instead of immediately (databricks/databricks-sql-go#423). diff --git a/driver_e2e_test.go b/driver_e2e_test.go index 5799eb5e..010a8e49 100644 --- a/driver_e2e_test.go +++ b/driver_e2e_test.go @@ -33,7 +33,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestDecimalMetadataThriftE2E(t *testing.T) { +func TestE2EDecimalMetadata(t *testing.T) { host, httpPath, token := pecoTestingCreds(t) for _, native := range []bool{false, true} { t.Run(fmt.Sprintf("native=%t", native), func(t *testing.T) { diff --git a/internal/backend/kernel/decimal_metadata_test.go b/internal/backend/kernel/decimal_metadata_test.go index ad515e9d..6391d3c1 100644 --- a/internal/backend/kernel/decimal_metadata_test.go +++ b/internal/backend/kernel/decimal_metadata_test.go @@ -8,56 +8,57 @@ import ( "testing" "github.com/apache/arrow-go/v18/arrow" - "github.com/apache/arrow-go/v18/arrow/array" - "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/databricks/databricks-sql-go/internal/arrowscan" "github.com/stretchr/testify/require" ) func TestDecimalMetadataKernelSchema(t *testing.T) { - for _, rowCount := range []int{0, 1} { - for _, text := range []bool{false, true} { - t.Run(fmt.Sprintf("rows=%d/text=%t", rowCount, text), func(t *testing.T) { - pairs := [][2]int32{{10, 2}, {38, 0}, {38, 38}} - fields := make([]arrow.Field, len(pairs)) - for index, pair := range pairs { - fields[index] = arrow.Field{Type: &arrow.Decimal128Type{Precision: pair[0], Scale: pair[1]}} - if text { - fields[index].Type = arrow.BinaryTypes.String - fields[index].Metadata = arrow.MetadataFrom(map[string]string{ - "databricks.type_name": "DECIMAL", "databricks.type_precision": fmt.Sprint(pair[0]), "databricks.type_scale": fmt.Sprint(pair[1]), - }) - } + for _, metadata := range []bool{false, true} { + t.Run(fmt.Sprintf("metadata=%t", metadata), func(t *testing.T) { + pairs := [][2]int32{{10, 2}, {38, 0}, {38, 38}} + fields := make([]arrow.Field, len(pairs)) + columnTypes := make([]arrowscan.ColumnTypeInfo, len(pairs)) + for index, pair := range pairs { + fields[index] = arrow.Field{Type: &arrow.Decimal128Type{Precision: pair[0], Scale: pair[1]}} + if metadata { + fields[index].Metadata = arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_precision": fmt.Sprint(pair[0]), "databricks.type_scale": fmt.Sprint(pair[1]), + }) } - schema := arrow.NewSchema(fields, nil) - resultRows := &kernelRows{schema: schema} - if rowCount > 0 { - builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) - defer builder.Release() - for _, field := range builder.Fields() { - field.AppendNull() - } - resultRows.cur = builder.NewRecordBatch() - defer resultRows.cur.Release() - } - decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale) - require.True(t, supported, "kernel rows must expose declared DECIMAL qualifiers") - for index, pair := range pairs { - precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index) - require.True(t, ok) - require.Equal(t, int64(pair[0]), precision) - require.Equal(t, int64(pair[1]), scale) - } - }) - } + columnTypes[index] = arrowscan.ColumnTypeInfoForField(fields[index]) + } + schema := arrow.NewSchema(fields, nil) + resultRows := &kernelRows{schema: schema, colTypes: columnTypes} + decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale) + require.True(t, supported, "kernel rows must expose declared DECIMAL qualifiers") + for index, pair := range pairs { + precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index) + require.True(t, ok) + require.Equal(t, int64(pair[0]), precision) + require.Equal(t, int64(pair[1]), scale) + require.Equal(t, "DECIMAL", resultRows.ColumnTypeDatabaseTypeName(index)) + _, hasLength := resultRows.ColumnTypeLength(index) + require.False(t, hasLength) + } + }) } } func TestDecimalMetadataKernelUnavailable(t *testing.T) { for name, field := range map[string]arrow.Field{ - "integer": {Type: arrow.PrimitiveTypes.Int64}, - "string": {Type: arrow.BinaryTypes.String}, - "float": {Type: arrow.PrimitiveTypes.Float64}, - "nested": {Type: arrow.ListOf(&arrow.Decimal128Type{Precision: 10, Scale: 2})}, + "integer": {Type: arrow.PrimitiveTypes.Int64}, + "string": {Type: arrow.BinaryTypes.String}, + "float": {Type: arrow.PrimitiveTypes.Float64}, + "nested": {Type: arrow.ListOf(&arrow.Decimal128Type{Precision: 10, Scale: 2})}, + "decimal32": {Type: &arrow.Decimal32Type{Precision: 7, Scale: 2}}, + "decimal64": {Type: &arrow.Decimal64Type{Precision: 12, Scale: 2}}, + "decimal256": {Type: &arrow.Decimal256Type{Precision: 40, Scale: 2}}, + "string_manifest_decimal": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_precision": "10", "databricks.type_scale": "2", + })}, + "integer_manifest_decimal": {Type: arrow.PrimitiveTypes.Int64, Metadata: arrow.MetadataFrom(map[string]string{ + "databricks.type_name": "DECIMAL", "databricks.type_precision": "10", "databricks.type_scale": "2", + })}, "missing_scale": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{ "databricks.type_name": "DECIMAL", "databricks.type_precision": "10", })}, @@ -88,7 +89,7 @@ func TestDecimalMetadataKernelUnavailable(t *testing.T) { } } -func TestDecimalMetadataKernelManifestPriority(t *testing.T) { +func TestDecimalMetadataKernelUsesNativeType(t *testing.T) { for _, precisionText := range []string{"38", "invalid"} { t.Run(precisionText, func(t *testing.T) { field := arrow.Field{ @@ -102,12 +103,8 @@ func TestDecimalMetadataKernelManifestPriority(t *testing.T) { require.True(t, supported) precision, scale, ok := decimalRows.ColumnTypePrecisionScale(0) require.True(t, ok) - if precisionText == "invalid" { - require.Equal(t, int64(10), precision) - } else { - require.Equal(t, int64(38), precision) - } - require.Equal(t, int64(0), scale) + require.Equal(t, int64(10), precision) + require.Equal(t, int64(2), scale) }) } } diff --git a/internal/backend/kernel/rows.go b/internal/backend/kernel/rows.go index b8906d18..59973c09 100644 --- a/internal/backend/kernel/rows.go +++ b/internal/backend/kernel/rows.go @@ -18,8 +18,6 @@ import ( "fmt" "io" "reflect" - "strconv" - "strings" "time" "unsafe" @@ -181,34 +179,15 @@ func (r *kernelRows) ColumnTypeNullable(index int) (nullable, ok bool) { return false, false } -func (r *kernelRows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { - if r.schema == nil || index < 0 || index >= r.schema.NumFields() { +func (resultRows *kernelRows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { + if resultRows.schema == nil || index < 0 || index >= resultRows.schema.NumFields() { return 0, 0, false } - field := r.schema.Field(index) - decimalType, native := field.Type.(arrow.DecimalType) - name, _ := field.Metadata.GetValue("databricks.type_name") - if !native && !strings.EqualFold(strings.TrimSpace(name), "DECIMAL") { + decimalType, supported := resultRows.schema.Field(index).Type.(*arrow.Decimal128Type) + if !supported { return 0, 0, false } - hasPrecision, hasScale := native, native - if native { - precision, scale = int64(decimalType.GetPrecision()), int64(decimalType.GetScale()) - } - if text, present := field.Metadata.GetValue("databricks.type_precision"); present { - if value, err := strconv.ParseInt(text, 10, 32); err == nil && value > 0 { - precision, hasPrecision = value, true - } - } - if text, present := field.Metadata.GetValue("databricks.type_scale"); present { - if value, err := strconv.ParseInt(text, 10, 32); err == nil { - scale, hasScale = value, true - } - } - if !hasPrecision || !hasScale { - return 0, 0, false - } - return precision, scale, true + return int64(decimalType.Precision), int64(decimalType.Scale), true } // ColumnTypeLength returns a variable-length column's length (math.MaxInt64, diff --git a/kernel_e2e_test.go b/kernel_e2e_test.go index a82f7746..d2d39575 100644 --- a/kernel_e2e_test.go +++ b/kernel_e2e_test.go @@ -29,14 +29,14 @@ func kernelTestDB(t *testing.T) *sql.DB { return kernelTestDBWith(t) } -func TestDecimalMetadataKernelE2E(t *testing.T) { - for _, native := range []bool{false, true} { - name := "arrow_native_decimal=false" - if native { - name = "arrow_native_decimal=true" +func TestKernelE2EDecimalMetadata(t *testing.T) { + for _, asFloat := range []bool{false, true} { + name := "decimal_as_float=false" + if asFloat { + name = "decimal_as_float=true" } t.Run(name, func(t *testing.T) { - db := kernelTestDBWith(t, WithArrowNativeDecimal(native)) + db := kernelTestDBWith(t, WithKernelDecimalAsFloat(asFloat)) t.Cleanup(func() { if err := db.Close(); err != nil { t.Error(err) diff --git a/kernel_parity_test.go b/kernel_parity_test.go index cdc6fbef..2531977e 100644 --- a/kernel_parity_test.go +++ b/kernel_parity_test.go @@ -95,7 +95,7 @@ func TestKernelE2EInterval(t *testing.T) { // TestKernelThriftColumnTypeParity is the live guard for PECOBLR-3692: the kernel // backend must report the SAME sql.ColumnType metadata (DatabaseTypeName, -// ScanType, Nullable, Length) as the Thrift backend for every type. The +// ScanType, Nullable, Length, DecimalSize) as the Thrift backend for every type. The // value-parity suites above compare scanned VALUES and so are blind to this — the // gap was a Rows.ColumnType* omission (kernelRows implemented only driver.Rows), // which surfaces as "" / interface{} metadata, not a wrong value. Named @@ -139,7 +139,7 @@ func TestKernelThriftColumnTypeParity(t *testing.T) { } // columnTypeStrings renders each column's sql.ColumnType metadata to a stable -// string (name, DatabaseTypeName, ScanType, Nullable, Length), so the kernel and +// string (name, DatabaseTypeName, ScanType, Nullable, Length, DecimalSize), so the kernel and // Thrift backends can be compared field-for-field. A nil ScanType renders as // "" so the pre-fix interface{} fallback would differ visibly. func columnTypeStrings(t *testing.T, db *sql.DB, query string) []string { @@ -161,8 +161,9 @@ func columnTypeStrings(t *testing.T, db *sql.DB, query string) []string { } nl, nlok := ct.Nullable() ln, lnok := ct.Length() - out[i] = fmt.Sprintf("name=%s db=%s scan=%s nullable=%v/%v len=%d/%v", - ct.Name(), ct.DatabaseTypeName(), scan, nl, nlok, ln, lnok) + precision, scale, hasDecimal := ct.DecimalSize() + out[i] = fmt.Sprintf("name=%s db=%s scan=%s nullable=%v/%v len=%d/%v decimal=%d,%d/%v", + ct.Name(), ct.DatabaseTypeName(), scan, nl, nlok, ln, lnok, precision, scale, hasDecimal) } return out }