Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/nightly-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ jobs:
# in the default pure-Go build (CGO_ENABLED=0). -run targets the high-value
# real-warehouse scenarios: large multi-page CloudFetch drain-past-caller-
# deadline, exact row count, and a non-positive download-thread count
# (issue #439). -count=1 defeats the test cache so a scheduled
# (issue #439), and DECIMAL metadata (issue #500). -count=1 defeats the test cache so a scheduled
# run always re-exercises the warehouse. NOT -short, so
# TestE2ECloudFetchExactRowCount (guarded by -short) actually runs.
- name: Run Thrift E2E
Expand All @@ -90,7 +90,7 @@ jobs:
DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
run: |
go test -count=1 -v -timeout 15m \
-run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads' .
-run 'TestE2EArrowBatchesSurviveQueryContextCancellation|TestE2ECloudFetchExactRowCount|TestE2ECloudFetchNonPositiveMaxDownloadThreads|TestE2EDecimalMetadata' .

kernel-e2e:
name: E2E (kernel backend)
Expand Down Expand Up @@ -244,7 +244,7 @@ jobs:
DATABRICKS_PECOTESTING_TOKEN_PERSONAL: ${{ secrets.DATABRICKS_PECOTESTING_TOKEN_PERSONAL }}
run: |
CGO_ENABLED=1 go test -tags databricks_kernel -count=1 -v -timeout 30m \
-run 'TestKernelE2E|TestKernelThriftParity|TestKernelParamsVsThrift' \
-run 'TestKernelE2E|TestKernelThriftParity|TestKernelThriftColumnTypeParity|TestKernelParamsVsThrift' \
-skip 'TestKernelE2EM2M' .

# Non-blocking Reyden reference-engine leg: reruns the TestKernelE2E* funcs against
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## Unreleased (v2.0.0 prototype)

- Fix the module-root package conflict when building, linting, or running Gazelle with the `tools` build tag (databricks/databricks-sql-go#162).
- **Behavior change:** Report declared DECIMAL precision and scale through `sql.ColumnType.DecimalSize()` on Thrift and SEA, including zero-row results. Tools such as ORM migrators can now act on these qualifiers (databricks/databricks-sql-go#500).
- **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication.
- Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439).
- **Behavior change:** On the Thrift backend, `ExecuteStatement` is now retried (per `WithRetries`) when the connection to the server or proxy can't be opened, for example connection refused or a connect timeout, because the statement was never sent. Failures after the connection exists are still not retried, so a statement never runs twice. With default retry settings, a refused connection now fails after about 15 seconds instead of immediately (databricks/databricks-sql-go#423).
Expand Down
75 changes: 75 additions & 0 deletions driver_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,81 @@ import (
"github.com/stretchr/testify/require"
)

func TestE2EDecimalMetadata(t *testing.T) {
host, httpPath, token := pecoTestingCreds(t)
for _, native := range []bool{false, true} {
t.Run(fmt.Sprintf("native=%t", native), func(t *testing.T) {
connector, err := NewConnector(
WithServerHostname(host), WithHTTPPath(httpPath), WithAccessToken(token),
WithArrowNativeDecimal(native),
)
require.NoError(t, err)
db := sql.OpenDB(connector)
t.Cleanup(func() { require.NoError(t, db.Close()) })
testDecimalResultMetadata(t, db)
})
}
}

func testDecimalResultMetadata(t *testing.T, db *sql.DB) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
const projection = "SELECT CAST(3.5 AS DECIMAL(10, 2)) AS amount, " +
"CAST(1 AS DECIMAL(38, 0)) AS whole, CAST(0.1 AS DECIMAL(38, 38)) AS all_fraction"
for _, empty := range []bool{false, true} {
t.Run(fmt.Sprintf("empty=%t", empty), func(t *testing.T) {
query := projection
if empty {
query += " WHERE 1 = 0"
}
result, err := db.QueryContext(ctx, query)
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, result.Close()) })
columns, err := result.ColumnTypes()
require.NoError(t, err)
require.Len(t, columns, 3)
for index, pair := range [][2]int64{{10, 2}, {38, 0}, {38, 38}} {
precision, scale, ok := columns[index].DecimalSize()
require.True(t, ok, "%s must report DECIMAL qualifiers", columns[index].Name())
require.Equal(t, pair[0], precision)
require.Equal(t, pair[1], scale)
}
count := 0
for result.Next() {
var amount float64
var whole, fraction any
require.NoError(t, result.Scan(&amount, &whole, &fraction))
require.Equal(t, 3.5, amount)
count++
}
require.NoError(t, result.Err())
if empty {
require.Zero(t, count)
} else {
require.Equal(t, 1, count)
}
require.NoError(t, result.Close())
})
}
result, err := db.QueryContext(ctx, "SELECT CAST(1 AS DEC(10, 2)), CAST(1 AS NUMERIC(38, 0)), 1, 'text', 1.5D")
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, result.Close()) })
columns, err := result.ColumnTypes()
require.NoError(t, err)
require.Len(t, columns, 5)
for index, column := range columns {
precision, scale, ok := column.DecimalSize()
if index < 2 {
require.True(t, ok)
require.Equal(t, [][2]int64{{10, 2}, {38, 0}}[index], [2]int64{precision, scale})
} else {
require.False(t, ok, "%s is not DECIMAL", column.DatabaseTypeName())
}
}
require.NoError(t, result.Close())
}

func TestWorkflowExample(t *testing.T) {

// these are global variables that get changed depending on the the expected response
Expand Down
110 changes: 110 additions & 0 deletions internal/backend/kernel/decimal_metadata_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
//go:build cgo && databricks_kernel

package kernel

import (
"database/sql/driver"
"fmt"
"testing"

"github.com/apache/arrow-go/v18/arrow"
"github.com/databricks/databricks-sql-go/internal/arrowscan"
"github.com/stretchr/testify/require"
)

func TestDecimalMetadataKernelSchema(t *testing.T) {
for _, metadata := range []bool{false, true} {
t.Run(fmt.Sprintf("metadata=%t", metadata), func(t *testing.T) {
pairs := [][2]int32{{10, 2}, {38, 0}, {38, 38}}
fields := make([]arrow.Field, len(pairs))
columnTypes := make([]arrowscan.ColumnTypeInfo, len(pairs))
for index, pair := range pairs {
fields[index] = arrow.Field{Type: &arrow.Decimal128Type{Precision: pair[0], Scale: pair[1]}}
if metadata {
fields[index].Metadata = arrow.MetadataFrom(map[string]string{
"databricks.type_name": "DECIMAL", "databricks.type_precision": fmt.Sprint(pair[0]), "databricks.type_scale": fmt.Sprint(pair[1]),
})
}
columnTypes[index] = arrowscan.ColumnTypeInfoForField(fields[index])
}
schema := arrow.NewSchema(fields, nil)
resultRows := &kernelRows{schema: schema, colTypes: columnTypes}
decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale)
require.True(t, supported, "kernel rows must expose declared DECIMAL qualifiers")
for index, pair := range pairs {
precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index)
require.True(t, ok)
require.Equal(t, int64(pair[0]), precision)
require.Equal(t, int64(pair[1]), scale)
require.Equal(t, "DECIMAL", resultRows.ColumnTypeDatabaseTypeName(index))
_, hasLength := resultRows.ColumnTypeLength(index)
require.False(t, hasLength)
}
})
}
}

func TestDecimalMetadataKernelUnavailable(t *testing.T) {
for name, field := range map[string]arrow.Field{
"integer": {Type: arrow.PrimitiveTypes.Int64},
"string": {Type: arrow.BinaryTypes.String},
"float": {Type: arrow.PrimitiveTypes.Float64},
"nested": {Type: arrow.ListOf(&arrow.Decimal128Type{Precision: 10, Scale: 2})},
"decimal32": {Type: &arrow.Decimal32Type{Precision: 7, Scale: 2}},
"decimal64": {Type: &arrow.Decimal64Type{Precision: 12, Scale: 2}},
"decimal256": {Type: &arrow.Decimal256Type{Precision: 40, Scale: 2}},
"string_manifest_decimal": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{
"databricks.type_name": "DECIMAL", "databricks.type_precision": "10", "databricks.type_scale": "2",
})},
"integer_manifest_decimal": {Type: arrow.PrimitiveTypes.Int64, Metadata: arrow.MetadataFrom(map[string]string{
"databricks.type_name": "DECIMAL", "databricks.type_precision": "10", "databricks.type_scale": "2",
})},
"missing_scale": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{
"databricks.type_name": "DECIMAL", "databricks.type_precision": "10",
})},
"malformed_precision": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{
"databricks.type_name": "DECIMAL", "databricks.type_precision": "bad", "databricks.type_scale": "2",
})},
"malformed_scale": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{
"databricks.type_name": "DECIMAL", "databricks.type_precision": "10", "databricks.type_scale": "bad",
})},
"missing_precision": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{
"databricks.type_name": "DECIMAL", "databricks.type_scale": "0",
})},
"non_decimal_qualifiers": {Type: arrow.BinaryTypes.String, Metadata: arrow.MetadataFrom(map[string]string{
"databricks.type_name": "STRING", "databricks.type_precision": "10", "databricks.type_scale": "2",
})},
} {
t.Run(name, func(t *testing.T) {
resultRows := &kernelRows{schema: arrow.NewSchema([]arrow.Field{field}, nil)}
decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale)
require.True(t, supported)
for _, index := range []int{-1, 0, 1} {
precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index)
require.Equal(t, int64(0), precision)
require.Equal(t, int64(0), scale)
require.False(t, ok)
}
})
}
}

func TestDecimalMetadataKernelUsesNativeType(t *testing.T) {
for _, precisionText := range []string{"38", "invalid"} {
t.Run(precisionText, func(t *testing.T) {
field := arrow.Field{
Type: &arrow.Decimal128Type{Precision: 10, Scale: 2},
Metadata: arrow.MetadataFrom(map[string]string{
"databricks.type_name": "DECIMAL", "databricks.type_precision": precisionText, "databricks.type_scale": "0",
}),
}
resultRows := &kernelRows{schema: arrow.NewSchema([]arrow.Field{field}, nil)}
decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale)
require.True(t, supported)
precision, scale, ok := decimalRows.ColumnTypePrecisionScale(0)
require.True(t, ok)
require.Equal(t, int64(10), precision)
require.Equal(t, int64(2), scale)
})
}
}
12 changes: 12 additions & 0 deletions internal/backend/kernel/rows.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ var (
_ driver.RowsColumnTypeDatabaseTypeName = (*kernelRows)(nil)
_ driver.RowsColumnTypeNullable = (*kernelRows)(nil)
_ driver.RowsColumnTypeLength = (*kernelRows)(nil)
_ driver.RowsColumnTypePrecisionScale = (*kernelRows)(nil)
// Public batch API, matching the Thrift path (rows.rows). GetArrowBatches is
// real; GetArrowIPCStreams rejects (kernel C ABI exports C Data, not IPC bytes).
_ dbrows.Rows = (*kernelRows)(nil)
Expand Down Expand Up @@ -178,6 +179,17 @@ func (r *kernelRows) ColumnTypeNullable(index int) (nullable, ok bool) {
return false, false
}

func (resultRows *kernelRows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) {
if resultRows.schema == nil || index < 0 || index >= resultRows.schema.NumFields() {
return 0, 0, false
}
decimalType, supported := resultRows.schema.Field(index).Type.(*arrow.Decimal128Type)
if !supported {
return 0, 0, false
}
return int64(decimalType.Precision), int64(decimalType.Scale), true
}

// ColumnTypeLength returns a variable-length column's length (math.MaxInt64,
// unbounded) for string/binary/nested/interval types and (0, false) for
// fixed-width types, matching the Thrift path. An out-of-range index returns
Expand Down
122 changes: 122 additions & 0 deletions internal/rows/decimal_metadata_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package rows

import (
"context"
"database/sql/driver"
"fmt"
"io"
"testing"

"github.com/databricks/databricks-sql-go/internal/cli_service"
"github.com/databricks/databricks-sql-go/internal/client"
"github.com/databricks/databricks-sql-go/internal/config"
"github.com/stretchr/testify/require"
)

func decimalMetadataColumn(typeID cli_service.TTypeId, precision, scale *int32) *cli_service.TColumnDesc {
return &cli_service.TColumnDesc{TypeDesc: &cli_service.TTypeDesc{Types: []*cli_service.TTypeEntry{{
PrimitiveEntry: &cli_service.TPrimitiveTypeEntry{
Type: typeID,
TypeQualifiers: &cli_service.TTypeQualifiers{Qualifiers: map[string]*cli_service.TTypeQualifierValue{
"precision": {I32Value: precision}, "scale": {I32Value: scale},
}},
},
}}}}
}

func TestDecimalMetadataThriftResults(t *testing.T) {
for _, rowCount := range []int{0, 1} {
for _, direct := range []bool{false, true} {
t.Run(fmt.Sprintf("rows=%d/direct=%t", rowCount, direct), func(t *testing.T) {
pairs := [][2]int32{{10, 2}, {38, 0}, {38, 38}}
schema := &cli_service.TTableSchema{}
result := &cli_service.TRowSet{}
for _, pair := range pairs {
precision, scale := pair[0], pair[1]
schema.Columns = append(schema.Columns, decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, &precision, &scale))
values := []string{}
if rowCount > 0 {
values = append(values, "1")
}
result.Columns = append(result.Columns, &cli_service.TColumn{StringVal: &cli_service.TStringColumn{Values: values, Nulls: []byte{0}}})
}
metadata := &cli_service.TGetResultSetMetadataResp{Schema: schema}
hasMore := false
fetch := &cli_service.TFetchResultsResp{Results: result, HasMoreRows: &hasMore}
metadataCalls := 0
testClient := &client.TestClient{
FnCloseOperation: func(context.Context, *cli_service.TCloseOperationReq) (*cli_service.TCloseOperationResp, error) {
return &cli_service.TCloseOperationResp{}, nil
},
FnGetResultSetMetadata: func(context.Context, *cli_service.TGetResultSetMetadataReq) (*cli_service.TGetResultSetMetadataResp, error) {
metadataCalls++
return metadata, nil
},
FnFetchResults: func(context.Context, *cli_service.TFetchResultsReq) (*cli_service.TFetchResultsResp, error) {
return fetch, nil
},
}
var directResults *cli_service.TSparkDirectResults
if direct {
directResults = &cli_service.TSparkDirectResults{ResultSetMetadata: metadata, ResultSet: fetch}
}
resultRows, err := NewRows(context.Background(), nil, testClient, config.WithDefaults(), directResults, nil)
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, resultRows.Close()) })
decimalRows, supported := resultRows.(driver.RowsColumnTypePrecisionScale)
require.True(t, supported, "rows must expose declared DECIMAL qualifiers")
check := func() {
for index, pair := range pairs {
precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index)
require.True(t, ok)
require.Equal(t, int64(pair[0]), precision)
require.Equal(t, int64(pair[1]), scale)
}
}
check()
values := make([]driver.Value, len(pairs))
for range rowCount {
require.NoError(t, resultRows.Next(values))
}
require.ErrorIs(t, resultRows.Next(values), io.EOF)
check()
if direct {
require.Zero(t, metadataCalls)
} else {
require.Equal(t, 1, metadataCalls)
}
})
}
}
}

func TestDecimalMetadataThriftUnavailable(t *testing.T) {
precision, scale := int32(10), int32(2)
noQualifiers := decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, nil, nil)
noQualifiers.TypeDesc.Types[0].PrimitiveEntry.TypeQualifiers = nil
emptyQualifiers := decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, nil, nil)
emptyQualifiers.TypeDesc.Types[0].PrimitiveEntry.TypeQualifiers.Qualifiers = nil
for name, column := range map[string]*cli_service.TColumnDesc{
"integer": decimalMetadataColumn(cli_service.TTypeId_INT_TYPE, &precision, &scale),
"string": decimalMetadataColumn(cli_service.TTypeId_STRING_TYPE, &precision, &scale),
"float": decimalMetadataColumn(cli_service.TTypeId_DOUBLE_TYPE, &precision, &scale),
"nested": decimalMetadataColumn(cli_service.TTypeId_ARRAY_TYPE, &precision, &scale),
"missing_precision": decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, nil, &scale),
"missing_scale": decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, &precision, nil),
"missing_both": decimalMetadataColumn(cli_service.TTypeId_DECIMAL_TYPE, nil, nil),
"no_qualifiers": noQualifiers,
"empty_qualifiers": emptyQualifiers,
} {
t.Run(name, func(t *testing.T) {
resultRows := &rows{client: &client.TestClient{}, schema: &cli_service.TTableSchema{Columns: []*cli_service.TColumnDesc{column}}}
decimalRows, supported := any(resultRows).(driver.RowsColumnTypePrecisionScale)
require.True(t, supported)
for _, index := range []int{-1, 0, 1} {
precision, scale, ok := decimalRows.ColumnTypePrecisionScale(index)
require.Equal(t, int64(0), precision)
require.Equal(t, int64(0), scale)
require.False(t, ok)
}
})
}
}
Loading
Loading