From 5b07a0f8a302aee4b07fb30006cb0a3b1e48521f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Fri, 13 Nov 2026 23:01:41 +0000 Subject: [PATCH 01/19] Complete Volume transfers through the native backend Signed-off-by: Aakash Saravanan --- CHANGELOG.md | 2 + connection.go | 36 +++++----- connection_test.go | 35 ++++++++++ doc.go | 10 ++- internal/backend/kernel/backend.go | 11 --- internal/backend/kernel/cgo.go | 5 ++ internal/backend/kernel/errors_classify.go | 3 + .../kernel/include/databricks_kernel.h | 20 +++++- internal/backend/kernel/kernel_test.go | 27 -------- internal/backend/kernel/operation.go | 69 +++++++++++++++++-- internal/backend/kernel/staging.go | 67 ------------------ internal/backend/kernel/staging_test.go | 41 ----------- 12 files changed, 153 insertions(+), 173 deletions(-) delete mode 100644 internal/backend/kernel/staging.go delete mode 100644 internal/backend/kernel/staging_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 319e2922..c8bb944b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased (v2.0.0 prototype) +- Support Volume PUT/GET/REMOVE through kernel ExecContext with staging roots, including resolved-symlink authorization and context cancellation. QueryContext retains raw instructions. + - **Breaking change:** Upgrade Apache Arrow Go from v12 to v18.8.0 (`github.com/apache/arrow-go/v18`). The public `rows.ArrowBatchIterator` now uses v18 `arrow.RecordBatch` and `*arrow.Schema` types; callers using Arrow batches must migrate their Arrow imports and dependent code. This requires a major driver release (v2.0.0), including a `/v2` driver module path before publication. - Fix CloudFetch reads failing with `row number 0 is not contained in any arrow batch: EOF` when `WithMaxDownloadThreads` / `maxDownloadThreads` is 0 or negative. Such values now log a warning and use the default of 10 (databricks/databricks-sql-go#439). - Honor `WithCloudFetch(false)` / `useCloudFetch=false` on the kernel backend to fetch complete results without cloud-storage downloads. The option overrides conflicting session configuration and remains effective after automatic backend recovery. Servers without paged inline support return a configuration error; the driver does not fall back to storage downloads. diff --git a/connection.go b/connection.go index 19de7f76..d93e7e2d 100644 --- a/connection.go +++ b/connection.go @@ -764,25 +764,29 @@ func (c *conn) execStagingOperation( return nil } - if len(driverctx.StagingPathsFromContext(ctx)) != 0 { - // Telemetry callback for staging operation row fetching (chunk timing not tracked for staging ops). - telemetryUpdate := func(chunkCount int, bytesDownloaded int64, chunkIndex int, chunkLatencyMs int64, totalChunksPresent int32) { - if c.telemetry != nil { - c.telemetry.AddTag(ctx, telemetry.TagChunkCount, chunkCount) - c.telemetry.AddTag(ctx, telemetry.TagBytesDownloaded, bytesDownloaded) - } + if len(driverctx.StagingPathsFromContext(ctx)) == 0 { + return dbsqlerrint.NewDriverError(ctx, "staging ctx must be provided.", nil) + } + if native, ok := op.(interface{ CompleteStaging(context.Context) error }); ok { + if err := native.CompleteStaging(ctx); err != nil { + return dbsqlerrint.NewDriverError(ctx, "error performing staging operation", err) } - row, err = op.Results(ctx, &rows.TelemetryCallbacks{ - OnChunkFetched: telemetryUpdate, - }) - if err != nil { - return dbsqlerrint.NewDriverError(ctx, "error reading row.", err) + return nil + } + // Telemetry callback for staging operation row fetching (chunk timing not tracked for staging ops). + telemetryUpdate := func(chunkCount int, bytesDownloaded int64, chunkIndex int, chunkLatencyMs int64, totalChunksPresent int32) { + if c.telemetry != nil { + c.telemetry.AddTag(ctx, telemetry.TagChunkCount, chunkCount) + c.telemetry.AddTag(ctx, telemetry.TagBytesDownloaded, bytesDownloaded) } - defer row.Close() //nolint:errcheck - - } else { - return dbsqlerrint.NewDriverError(ctx, "staging ctx must be provided.", nil) } + row, err = op.Results(ctx, &rows.TelemetryCallbacks{ + OnChunkFetched: telemetryUpdate, + }) + if err != nil { + return dbsqlerrint.NewDriverError(ctx, "error reading row.", err) + } + defer row.Close() //nolint:errcheck var sqlRow []driver.Value colNames := row.Columns() diff --git a/connection_test.go b/connection_test.go index 59be9150..e8d4e02b 100644 --- a/connection_test.go +++ b/connection_test.go @@ -17,7 +17,9 @@ import ( "github.com/apache/thrift/lib/go/thrift" "github.com/pkg/errors" + "github.com/databricks/databricks-sql-go/driverctx" dbsqlerr "github.com/databricks/databricks-sql-go/errors" + "github.com/databricks/databricks-sql-go/internal/backend" thriftbackend "github.com/databricks/databricks-sql-go/internal/backend/thrift" "github.com/databricks/databricks-sql-go/internal/cli_service" "github.com/databricks/databricks-sql-go/internal/client" @@ -974,3 +976,36 @@ func TestConn_handleStagingRetry(t *testing.T) { assert.Less(t, elapsed, 2*time.Second, "context cancel should abort PUT retry backoff promptly") }) } + +type nativeStagingStub struct { + backend.Operation + marked bool + completed bool + err error +} + +func (o *nativeStagingStub) StatementID() string { return "volume-test" } +func (o *nativeStagingStub) IsStaging(context.Context) (bool, error) { return o.marked, nil } +func (o *nativeStagingStub) CompleteStaging(context.Context) error { o.completed = true; return o.err } + +func TestConnNativeStagingDispatch(t *testing.T) { + for _, tc := range []struct { + name string + marked, roots, wantCall, wantError bool + }{ + {"ordinary query", false, true, false, false}, + {"missing roots", true, false, false, true}, + {"native completion", true, true, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + if tc.roots { + ctx = driverctx.NewContextWithStagingInfo(ctx, []string{t.TempDir()}) + } + op := &nativeStagingStub{marked: tc.marked} + err := (&conn{}).execStagingOperation(op, ctx) + assert.Equal(t, tc.wantError, err != nil) + assert.Equal(t, tc.wantCall, op.completed) + }) + } +} diff --git a/doc.go b/doc.go index b0cc32a0..5181b881 100644 --- a/doc.go +++ b/doc.go @@ -227,8 +227,7 @@ RetryWaitMin / RetryWaitMax / RetryMax, including the disable form, forwarded to kernel's HTTP retry config); and the TLS, proxy, and session-conf (query tags, statement timeout, time zone) options. Federated token providers use one token snapshot during setup. Nothing is silently ignored: WithTimeout and token-provider / external / -static authenticators are rejected at connect; staging (PUT/GET/REMOVE on a -Unity Catalog volume) is rejected at execute. WithMaxRows is accepted but inert (the +static authenticators are rejected at connect. WithMaxRows is accepted but inert (the kernel manages fetching below the C ABI). Kernel HTTP requests inherit the driver's ClientTimeout (900s by default). If @@ -368,6 +367,13 @@ After doing so, you can execute staging operations using this context using the _, err1 := db.ExecContext(ctx, `PUT 'staging/file.csv' INTO '/Volumes/main/staging_test/e2etests/file.csv' OVERWRITE`) +With the kernel backend, ExecContext completes the transfer and reports zero +rows affected. QueryContext returns raw instruction rows without file effects. +Both the requested local path and its resolved symlink target must remain +within the configured staging roots. Symlinks within those roots are permitted; +escapes are rejected, which is stricter than Thrift's requested-path check. +The staging context is required for REMOVE as well as PUT and GET. + # Errors There are three error types exposed via dbsql/errors diff --git a/internal/backend/kernel/backend.go b/internal/backend/kernel/backend.go index d8e20d08..bdb8c54f 100644 --- a/internal/backend/kernel/backend.go +++ b/internal/backend/kernel/backend.go @@ -72,7 +72,6 @@ import ( "sync/atomic" "time" - dbsqlerr "github.com/databricks/databricks-sql-go/errors" "github.com/databricks/databricks-sql-go/internal/backend" ) @@ -908,15 +907,5 @@ func (k *KernelBackend) SessionID() string { return k.sessionID } // parameters (req.Params) are bound onto the statement in execute (see // bindParams). func (k *KernelBackend) Execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) { - // Staging (Unity Catalog volume PUT/GET/REMOVE) needs a local file transfer the - // kernel path can't perform, and the kernel C ABI surfaces no IsStagingOperation - // signal to drive conn.execStagingOperation. Reject it here rather than let - // IsStaging return false and report success with no file moved (silent data loss). - // (Bound parameters, by contrast, ARE now supported — bound in execute via the - // kernel's raw-param C ABI.) - if isStagingStatement(req.Query) { - return &kernelOp{}, fmt.Errorf("databricks: staging operations (PUT/GET/REMOVE on a volume) are %w; "+ - "use the default (Thrift) backend", dbsqlerr.ErrNotSupportedByKernel) - } return k.execute(ctx, req) } diff --git a/internal/backend/kernel/cgo.go b/internal/backend/kernel/cgo.go index a8a79f6d..9e31b65b 100644 --- a/internal/backend/kernel/cgo.go +++ b/internal/backend/kernel/cgo.go @@ -38,7 +38,9 @@ import "C" import ( "context" "fmt" + "os" "runtime" + "syscall" "unsafe" "github.com/databricks/databricks-sql-go/driverctx" @@ -172,6 +174,9 @@ func lastError(code C.KernelStatusCode) *KernelError { HTTPStatus: uint16(e.http_status), Retryable: bool(e.retryable), } + if code := C.kernel_get_last_error_local_os_error(); code != 0 { + ke.cause = &os.PathError{Op: "file", Path: C.GoString(C.kernel_get_last_error_local_path()), Err: syscall.Errno(code)} + } if e.sql_state != nil { ke.SQLState = C.GoString(e.sql_state) } diff --git a/internal/backend/kernel/errors_classify.go b/internal/backend/kernel/errors_classify.go index 73b7c39c..7d4fdd7b 100644 --- a/internal/backend/kernel/errors_classify.go +++ b/internal/backend/kernel/errors_classify.go @@ -49,8 +49,11 @@ type KernelError struct { HTTPStatus uint16 Retryable bool QueryID string + cause error } +func (e *KernelError) Unwrap() error { return e.cause } + func (e *KernelError) Error() string { // Append the server query id when present — it is the one correlation handle // to server-side query history, and on the execute-error path (no exec handle) diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index 965c09b6..fe88926b 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -190,6 +190,11 @@ typedef struct KernelError { * last kernel call cleared the buffer. */ bool kernel_get_last_error(KernelError* out); +/* These accessors preserve the last error. Strings are borrowed until the next + * error-clearing kernel call; absent strings are NULL, absent OS code is zero. */ +int32_t kernel_get_last_error_local_os_error(void); +const char* kernel_get_last_error_local_kind(void); +const char* kernel_get_last_error_local_path(void); /* ─── Opaque handles ────────────────────────────────────────────────── * @@ -595,12 +600,12 @@ KernelStatusCode kernel_session_config_set_u2m_token_cache_config(KernelSessionC * * Recognised keys (case-insensitive): * - "cloudfetch_enabled" = "true"|"false" — false serves every result - * inline (SEA disposition = INLINE), mirroring JDBC EnableQueryResultDownload=0. + * through paged inline delivery without cloud-storage downloads. * - "cloudfetch_link_prefetch_window" = positive integer. * - "cloudfetch_max_chunks_in_memory" = positive integer (bounds peak RSS; * values above the kernel ceiling of 256 are clamped with a warning). * - "inline_max_chunks_in_memory" = positive integer (inline-Arrow prefetch - * window, used when cloudfetch_enabled = false). + * window for paged inline results). * - "complex_types_as_json" = "true"|"false" — render ARRAY / MAP / STRUCT / * VARIANT / GEOMETRY / GEOGRAPHY as JSON text (PySQL * _use_arrow_native_complex_types=False, JDBC complexDatatypeSupport=false). @@ -966,6 +971,17 @@ KernelStatusCode kernel_executed_async_statement_close( */ KernelStatusCode kernel_result_stream_get_schema(kernel_result_stream_t* stream, struct ArrowSchema* out); +/* Volume completion preserves raw rows. Roots are a JSON array of strings; + * out_status is 0=ordinary result, 1=input bytes required, 2=completed. + * The detached result-stream canceller can interrupt completion. */ +KernelStatusCode kernel_executed_statement_is_volume_operation( + const kernel_executed_statement_t* exec, bool* out); +KernelStatusCode kernel_result_stream_is_volume_operation( + const kernel_result_stream_t* stream, bool* out); +KernelStatusCode kernel_result_stream_complete_volume_operation( + kernel_result_stream_t* stream, const char* allowed_paths_json, + bool remove_not_found_is_success, int32_t* out_status); + KernelStatusCode kernel_result_stream_next_batch(kernel_result_stream_t* stream, struct ArrowArray* out_array, struct ArrowSchema* out_schema); diff --git a/internal/backend/kernel/kernel_test.go b/internal/backend/kernel/kernel_test.go index cd379412..3485fb78 100644 --- a/internal/backend/kernel/kernel_test.go +++ b/internal/backend/kernel/kernel_test.go @@ -531,33 +531,6 @@ func TestExecuteHandleLessOpContract(t *testing.T) { } } -// TestExecuteRejectsStaging drives Execute with a staging statement (not just the -// isStagingStatement detector in isolation) to pin the detector→Execute wiring: a -// refactor that dropped or reordered the check would silently reopen the -// silent-no-op data-loss path. Mirrors TestExecuteRejectsParams. -func TestExecuteRejectsStaging(t *testing.T) { - k := &KernelBackend{} - op, err := k.Execute(context.Background(), backend.ExecRequest{ - Query: "PUT '/tmp/f' INTO '/Volumes/main/s/e/f.csv'", - }) - if err == nil { - t.Fatal("expected an error for a staging statement, got nil") - } - if !errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) { - t.Errorf("staging rejection should wrap ErrNotSupportedByKernel, got %v", err) - } - if op == nil { - t.Fatal("Execute must return a non-nil Operation per the Backend contract") - } - closed, closeErr := op.Close(context.Background()) - if closeErr != nil { - t.Errorf("Close error = %v, want nil", closeErr) - } - if closed { - t.Error("Close on a handle-less op must report closed=false (no CLOSE_STATEMENT)") - } -} - // ExecutionError must satisfy the same public contract as the Thrift path so the // errors.Is → errors.As → SqlState()/QueryId() recipe documented in doc.go works // on the kernel backend too (it previously returned a bare *KernelError that diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index e2f355f0..fa8e130f 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -11,6 +11,7 @@ import "C" import ( "context" "database/sql/driver" + "encoding/json" "errors" "fmt" "sync" @@ -35,8 +36,7 @@ import ( // client-timeout path, let an in-flight cancel finish in the background so // it cannot extend the client deadline // -// Binds any query parameters (bindParams) before executing; staging statements -// are rejected up front by Execute, so none reach here. +// Binds any query parameters (bindParams) before executing. func (k *KernelBackend) execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) { // Log the SQL length, not the text: query bodies can carry PII/secrets in // WHERE/INSERT/SET, and this goes to stderr. Matches the driver's own @@ -384,11 +384,66 @@ func (o *kernelOp) Results(ctx context.Context, callbacks *dbsqlrows.TelemetryCa return newKernelRows(ctx, o, stream, callbacks) } -// IsStaging reports whether this is a staging (PUT/GET/REMOVE) operation. Always -// false on the kernel path: staging statements are rejected up front in -// Execute (isStagingStatement), so no kernelOp is ever produced for one — there -// is nothing for conn.execStagingOperation to act on here. -func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { return false, nil } +func (o *kernelOp) IsStaging(ctx context.Context) (bool, error) { + if o.exec == nil { + return false, nil + } + var marked C.bool + err := call(func() C.KernelStatusCode { + return C.kernel_executed_statement_is_volume_operation(o.exec, &marked) + }) + return bool(marked), err +} + +func (o *kernelOp) CompleteStaging(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + var stream *C.kernel_result_stream_t + if err := call(func() C.KernelStatusCode { + return C.kernel_executed_statement_get_result_stream(o.exec, &stream) + }); err != nil { + return toStatementError(err) + } + defer C.kernel_result_stream_close(stream) + var canceller *C.kernel_result_stream_canceller_t + if err := call(func() C.KernelStatusCode { + return C.kernel_result_stream_canceller_new(stream, &canceller) + }); err != nil { + return toStatementError(err) + } + defer C.kernel_result_stream_canceller_free(canceller) + done, stopped := make(chan struct{}), make(chan struct{}) + go func() { + defer close(stopped) + select { + case <-ctx.Done(): + C.kernel_result_stream_canceller_cancel(canceller) + case <-done: + } + }() + defer func() { close(done); <-stopped }() + encoded, err := json.Marshal(driverctx.StagingPathsFromContext(ctx)) + if err != nil { + return err + } + paths := newCStr(string(encoded)) + defer paths.free() + var outcome C.int32_t + err = call(func() C.KernelStatusCode { + return C.kernel_result_stream_complete_volume_operation(stream, paths.c, true, &outcome) + }) + if err != nil { + if ctx.Err() != nil { + return fmt.Errorf("Volume transfer: %w: %w", ctx.Err(), toStatementError(err)) + } + return toStatementError(err) + } + if outcome != 2 { + return fmt.Errorf("Volume operation requires a local file instruction") + } + return nil +} // Close best-effort closes the executed statement and its statement handle. It is // idempotent (a second call, or a call after Rows.Close already tore the diff --git a/internal/backend/kernel/staging.go b/internal/backend/kernel/staging.go deleted file mode 100644 index c614169c..00000000 --- a/internal/backend/kernel/staging.go +++ /dev/null @@ -1,67 +0,0 @@ -package kernel - -import "strings" - -// This file is intentionally NOT behind the `cgo && databricks_kernel` build tag. -// Staging detection is pure Go (string inspection of the SQL), so keeping it -// untagged lets its test run under CGO_ENABLED=0. - -// isStagingStatement reports whether sql is a Unity Catalog volume staging -// command (PUT / GET / REMOVE). The Thrift backend learns a statement is staging -// from server result metadata (IsStagingOperation); the kernel C ABI exposes no -// such signal, and the kernel path cannot perform the local file transfer these -// commands require. We therefore detect them from the SQL and reject at execute -// time (see Execute) rather than silently returning success with no file moved — -// which is what returning false from IsStaging would do. Case-insensitive; skips -// leading whitespace AND leading SQL comments (`-- ...` and `/* ... */`), so a -// comment-prefixed staging command is still caught rather than slipping through -// as a silent no-op. Errs toward matching only a clear leading keyword — a false -// negative just yields the kernel's normal "unsupported syntax" server error, -// whereas a false positive would reject a legitimate query. -func isStagingStatement(sql string) bool { - s := stripLeadingSQLComments(sql) - for _, kw := range []string{"PUT", "GET", "REMOVE"} { - if len(s) > len(kw) && - strings.EqualFold(s[:len(kw)], kw) && - isSQLTokenBreak(s[len(kw)]) { - return true - } - } - return false -} - -// stripLeadingSQLComments removes leading whitespace and leading SQL comments -// (both `-- ...` line comments and `/* ... */` block comments), repeatedly, so -// the first real token is exposed. An unterminated block comment consumes the -// rest of the string (yielding ""). Comment bodies are not scanned for nested -// markers — SQL block comments do not nest — and this only touches the LEADING -// run, so a comment later in the statement is untouched. -func stripLeadingSQLComments(sql string) string { - s := strings.TrimSpace(sql) - for { - switch { - case strings.HasPrefix(s, "--"): - if i := strings.IndexByte(s, '\n'); i >= 0 { - s = s[i+1:] - } else { - return "" // line comment to end of input - } - case strings.HasPrefix(s, "/*"): - if i := strings.Index(s[2:], "*/"); i >= 0 { - s = s[2+i+2:] - } else { - return "" // unterminated block comment - } - default: - return s - } - s = strings.TrimSpace(s) - } -} - -// isSQLTokenBreak reports whether b ends a leading keyword — a space or a quote -// (staging commands take a quoted path, e.g. PUT '' INTO ''), so -// "PUT '..." matches but an identifier like "PUTS" or "GETTER" does not. -func isSQLTokenBreak(b byte) bool { - return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\'' || b == '"' -} diff --git a/internal/backend/kernel/staging_test.go b/internal/backend/kernel/staging_test.go deleted file mode 100644 index 91d69794..00000000 --- a/internal/backend/kernel/staging_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package kernel - -import "testing" - -func TestIsStagingStatement(t *testing.T) { - staging := []string{ - "PUT '/tmp/f.csv' INTO '/Volumes/main/s/e/f.csv' OVERWRITE", - "GET '/Volumes/main/s/e/f.csv' TO 'local.csv'", - "REMOVE '/Volumes/main/s/e/f.csv'", - " put '/tmp/f' into '/Volumes/x'", // leading space + lowercase - "Get\t'/Volumes/x' to 'y'", // tab after keyword, mixed case - "/* audit */ PUT '/local' INTO '/Volumes/main/s/v'", // leading block comment - "/* a */ /* b */ PUT '/l' INTO '/Volumes/x'", // multiple block comments - "-- upload the file\nPUT '/l' INTO '/Volumes/x'", // leading line comment, then staging - " /* c */\n -- d\n REMOVE '/Volumes/x'", // mixed comments + whitespace - } - for _, sql := range staging { - if !isStagingStatement(sql) { - t.Errorf("expected staging: %q", sql) - } - } - - notStaging := []string{ - "SELECT 1", - "PUTS ON A SHOW", // PUT is a prefix of an identifier, not a token - "GETTER()", // likewise for GET - "REMOVED_AT FROM t", // likewise for REMOVE - "INSERT INTO t VALUES (1)", // contains INTO but is not staging - "", // empty - "-- PUT '/x' INTO '/y'", // whole line commented out; nothing after - "/* PUT '/x' INTO '/y' */", // whole statement inside a block comment - "SELECT 'PUT ' AS col", // keyword only inside a literal - "/* unterminated PUT '/x'", // unterminated block comment consumes the rest - "/* c */ SELECT 1", // real leading comment, but not a staging command - } - for _, sql := range notStaging { - if isStagingStatement(sql) { - t.Errorf("did not expect staging: %q", sql) - } - } -} From 9af910f2b78561b65905ef5710b5f1d1b561600f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 00:07:11 +0000 Subject: [PATCH 02/19] Pin the native source for Volume completion Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index de1b52ca..f85bb468 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -0b5d0edfed32c0745d5abc2468fc3070e2eb2a17 +bdc4d35693ab94ee1991799d12a40503fdf66670 From 3cd1040ac199261bdd2d4dc60ea2ec4c5cc17165 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 00:15:39 +0000 Subject: [PATCH 03/19] Select the Windows-compatible Volume kernel Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index f85bb468..a34a99f0 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -bdc4d35693ab94ee1991799d12a40503fdf66670 +c22bd606abc48599ab780a65a1389d141c3414bf From 26f142a65a798d741aaf2feb32f099c6efb2be25 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 02:53:00 +0000 Subject: [PATCH 04/19] Pin native Volume storage retry and terminal response fixes Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index a34a99f0..8a2f8e59 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -c22bd606abc48599ab780a65a1389d141c3414bf +22f96facc276836bcac7230c8a2c0987f734f2c7 From f0c6f417bf2bb45b0dbb5d39595656daee3c0300 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 03:32:03 +0000 Subject: [PATCH 05/19] Preserve native staging error causes in dispatch checks Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- connection_test.go | 15 +++++++++++---- internal/backend/kernel/operation.go | 4 ++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 8a2f8e59..47d2f1c8 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -22f96facc276836bcac7230c8a2c0987f734f2c7 +08a921377dd79e5a301e007da7290c1984140952 diff --git a/connection_test.go b/connection_test.go index e8d4e02b..3cb96701 100644 --- a/connection_test.go +++ b/connection_test.go @@ -989,23 +989,30 @@ func (o *nativeStagingStub) IsStaging(context.Context) (bool, error) { return o. func (o *nativeStagingStub) CompleteStaging(context.Context) error { o.completed = true; return o.err } func TestConnNativeStagingDispatch(t *testing.T) { + completionErr := errors.New("native completion failed") for _, tc := range []struct { name string marked, roots, wantCall, wantError bool + completionErr error }{ - {"ordinary query", false, true, false, false}, - {"missing roots", true, false, false, true}, - {"native completion", true, true, true, false}, + {"ordinary query", false, true, false, false, nil}, + {"missing roots", true, false, false, true, nil}, + {"native completion", true, true, true, false, nil}, + {"native completion error", true, true, true, true, completionErr}, } { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() if tc.roots { ctx = driverctx.NewContextWithStagingInfo(ctx, []string{t.TempDir()}) } - op := &nativeStagingStub{marked: tc.marked} + op := &nativeStagingStub{marked: tc.marked, err: tc.completionErr} err := (&conn{}).execStagingOperation(op, ctx) assert.Equal(t, tc.wantError, err != nil) assert.Equal(t, tc.wantCall, op.completed) + if tc.completionErr != nil { + assert.ErrorIs(t, err, dbsqlerr.DriverError) + assert.ErrorIs(t, err, tc.completionErr) + } }) } } diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index fa8e130f..3a9db25c 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -435,12 +435,12 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { }) if err != nil { if ctx.Err() != nil { - return fmt.Errorf("Volume transfer: %w: %w", ctx.Err(), toStatementError(err)) + return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) } return toStatementError(err) } if outcome != 2 { - return fmt.Errorf("Volume operation requires a local file instruction") + return errors.New("kernel: volume operation requires a local file instruction") } return nil } From 36298433d83ec5cb94c2d12e8c8ef251373e1c9f Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 05:19:18 +0000 Subject: [PATCH 06/19] Apply session validity policy by Volume failure origin Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/cgo.go | 1 + internal/backend/kernel/errors_classify.go | 1 + .../kernel/include/databricks_kernel.h | 1 + internal/backend/kernel/operation.go | 4 + staging_kernel_test.go | 126 ++++++++++++++++++ 6 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 staging_kernel_test.go diff --git a/KERNEL_REV b/KERNEL_REV index 47d2f1c8..bc57504b 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -08a921377dd79e5a301e007da7290c1984140952 +855847bdca71e07285cfc60332fb2f33f8e47786 diff --git a/internal/backend/kernel/cgo.go b/internal/backend/kernel/cgo.go index 9e31b65b..f6ba5999 100644 --- a/internal/backend/kernel/cgo.go +++ b/internal/backend/kernel/cgo.go @@ -173,6 +173,7 @@ func lastError(code C.KernelStatusCode) *KernelError { VendorCode: int32(e.vendor_code), HTTPStatus: uint16(e.http_status), Retryable: bool(e.retryable), + ErrorCode: C.GoString(C.kernel_get_last_error_code()), } if code := C.kernel_get_last_error_local_os_error(); code != 0 { ke.cause = &os.PathError{Op: "file", Path: C.GoString(C.kernel_get_last_error_local_path()), Err: syscall.Errno(code)} diff --git a/internal/backend/kernel/errors_classify.go b/internal/backend/kernel/errors_classify.go index 7d4fdd7b..7e07d065 100644 --- a/internal/backend/kernel/errors_classify.go +++ b/internal/backend/kernel/errors_classify.go @@ -49,6 +49,7 @@ type KernelError struct { HTTPStatus uint16 Retryable bool QueryID string + ErrorCode string cause error } diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index fe88926b..a88286dc 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -193,6 +193,7 @@ bool kernel_get_last_error(KernelError* out); /* These accessors preserve the last error. Strings are borrowed until the next * error-clearing kernel call; absent strings are NULL, absent OS code is zero. */ int32_t kernel_get_last_error_local_os_error(void); +const char* kernel_get_last_error_code(void); const char* kernel_get_last_error_local_kind(void); const char* kernel_get_last_error_local_path(void); diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index 3a9db25c..cb98619a 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -437,6 +437,10 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { if ctx.Err() != nil { return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) } + var ke *KernelError + if !errors.As(err, &ke) || (ke.ErrorCode != "VOLUME_STORAGE_HTTP_ERROR" && ke.ErrorCode != "VOLUME_STORAGE_NETWORK_ERROR") { + o.backend.evictIfSessionFatal(err) + } return toStatementError(err) } if outcome != 2 { diff --git a/staging_kernel_test.go b/staging_kernel_test.go new file mode 100644 index 00000000..b9193f97 --- /dev/null +++ b/staging_kernel_test.go @@ -0,0 +1,126 @@ +//go:build cgo && databricks_kernel + +package dbsql + +import ( + "bytes" + "context" + "database/sql" + "database/sql/driver" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/databricks/databricks-sql-go/driverctx" + "github.com/databricks/databricks-sql-go/internal/backend/kernel" + "github.com/stretchr/testify/require" +) + +func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { + for _, mode := range []string{"instruction401", "storage401", "storageBody"} { + t.Run(mode, func(t *testing.T) { + directory := t.TempDir() + destination := filepath.Join(directory, "download.bin") + require.NoError(t, os.WriteFile(destination, []byte("old bytes"), 0600)) + var executions, storage atomic.Int64 + var result map[string]any + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/sessions"): + _, _ = io.WriteString(w, `{"session_id":"volume-session"}`) + case r.Method == "POST" && r.URL.Path == "/api/2.0/sql/statements": + executions.Add(1) + _ = json.NewEncoder(w).Encode(result) + case r.URL.Path == "/api/2.0/sql/statements/volume-stmt/result/chunks/1": + w.WriteHeader(401) + _, _ = io.WriteString(w, `{"error_code":"SESSION_EXPIRED","message":"instruction session expired","sql_state":"08003"}`) + case r.URL.Path == "/object": + storage.Add(1) + if mode == "storage401" { + w.WriteHeader(401) + } else { + w.Header().Set("Content-Length", "20") + _, _ = io.WriteString(w, "short") + } + default: + _, _ = io.WriteString(w, `{}`) + } + })) + defer server.Close() + names := []string{"operation", "presignedUrl", "headers", "localFile"} + values := []string{"GET", server.URL + "/object", "{}", destination} + fields, columns := make([]arrow.Field, 4), make([]arrow.Array, 4) + manifestColumns := make([]map[string]any, 4) + for i, name := range names { + fields[i] = arrow.Field{Name: name, Type: arrow.BinaryTypes.String, Nullable: true} + builder := array.NewStringBuilder(memory.DefaultAllocator) + builder.Append(values[i]) + columns[i] = builder.NewStringArray() + builder.Release() + defer columns[i].Release() + manifestColumns[i] = map[string]any{"name": name, "type_name": "STRING", "type_text": "STRING", "position": i} + } + schema := arrow.NewSchema(fields, nil) + record := array.NewRecordBatch(schema, columns, 1) + defer record.Release() + var attachment bytes.Buffer + writer := ipc.NewWriter(&attachment, ipc.WithSchema(schema)) + require.NoError(t, writer.Write(record)) + require.NoError(t, writer.Close()) + chunk := map[string]any{"chunk_index": 0, "row_offset": 0, "row_count": 1, "attachment": base64.StdEncoding.EncodeToString(attachment.Bytes())} + chunks := 1 + if mode == "instruction401" { + chunk["next_chunk_index"], chunks = 1, 2 + } + result = map[string]any{ + "statement_id": "volume-stmt", "status": map[string]any{"state": "SUCCEEDED"}, + "manifest": map[string]any{"format": "ARROW_STREAM", "is_volume_operation": true, "schema": map[string]any{"column_count": 4, "columns": manifestColumns}, "total_row_count": 1, "total_chunk_count": chunks}, + "result": chunk, + } + connector, err := NewConnector(WithServerHostname(server.URL), WithHTTPPath("/sql/1.0/warehouses/test"), + WithAccessToken("test-token"), WithUseKernel(true), WithSkipTLSHostVerify(), WithRetries(-1, time.Millisecond, time.Millisecond)) + require.NoError(t, err) + db := sql.OpenDB(connector) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + connection, err := db.Conn(ctx) + require.NoError(t, err) + defer connection.Close() + _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, []string{directory}), "GET instruction") + require.Error(t, err) + require.NotErrorIs(t, err, driver.ErrBadConn) + var detail *kernel.KernelError + require.ErrorAs(t, err, &detail) + wantCode := map[string]string{"instruction401": "SESSION_EXPIRED", "storage401": "VOLUME_STORAGE_HTTP_ERROR", "storageBody": "VOLUME_STORAGE_NETWORK_ERROR"}[mode] + require.Equal(t, wantCode, detail.ErrorCode) + require.NoError(t, connection.Raw(func(raw any) error { + require.Equal(t, mode != "instruction401", raw.(driver.Validator).IsValid()) + return nil + })) + data, err := os.ReadFile(destination) + require.NoError(t, err) + require.Equal(t, "old bytes", string(data)) + require.Equal(t, int64(1), executions.Load()) + wantStorage := int64(1) + if mode == "instruction401" { + wantStorage = 0 + } + require.Equal(t, wantStorage, storage.Load()) + }) + } +} From 809e8e6bba5b94255fbc5389d1b55c464918a5b8 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 05:39:33 +0000 Subject: [PATCH 07/19] Invalidate fatal instruction sessions before returning cancellation Signed-off-by: Aakash Saravanan --- internal/backend/kernel/operation.go | 6 ++-- staging_kernel_test.go | 51 ++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/internal/backend/kernel/operation.go b/internal/backend/kernel/operation.go index cb98619a..59956ffb 100644 --- a/internal/backend/kernel/operation.go +++ b/internal/backend/kernel/operation.go @@ -434,13 +434,13 @@ func (o *kernelOp) CompleteStaging(ctx context.Context) error { return C.kernel_result_stream_complete_volume_operation(stream, paths.c, true, &outcome) }) if err != nil { - if ctx.Err() != nil { - return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) - } var ke *KernelError if !errors.As(err, &ke) || (ke.ErrorCode != "VOLUME_STORAGE_HTTP_ERROR" && ke.ErrorCode != "VOLUME_STORAGE_NETWORK_ERROR") { o.backend.evictIfSessionFatal(err) } + if ctx.Err() != nil { + return fmt.Errorf("kernel: volume transfer cancelled: %w (kernel error: %w)", ctx.Err(), toStatementError(err)) + } return toStatementError(err) } if outcome != 2 { diff --git a/staging_kernel_test.go b/staging_kernel_test.go index b9193f97..fb769cc5 100644 --- a/staging_kernel_test.go +++ b/staging_kernel_test.go @@ -24,12 +24,28 @@ import ( "github.com/apache/arrow-go/v18/arrow/ipc" "github.com/apache/arrow-go/v18/arrow/memory" "github.com/databricks/databricks-sql-go/driverctx" + "github.com/databricks/databricks-sql-go/internal/backend" "github.com/databricks/databricks-sql-go/internal/backend/kernel" "github.com/stretchr/testify/require" ) +type cancelOnCompletionErrorContext struct { + context.Context + cancel context.CancelFunc + errCalls int +} + +func (c *cancelOnCompletionErrorContext) Err() error { + c.errCalls++ + // Deliver cancellation after native completion returns to make the error race deterministic. + if c.errCalls == 2 { + c.cancel() + } + return c.Context.Err() +} + func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { - for _, mode := range []string{"instruction401", "storage401", "storageBody"} { + for _, mode := range []string{"instruction401", "storage401", "storageBody", "instruction401Cancelled", "storage401Cancelled"} { t.Run(mode, func(t *testing.T) { directory := t.TempDir() destination := filepath.Join(directory, "download.bin") @@ -50,7 +66,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { _, _ = io.WriteString(w, `{"error_code":"SESSION_EXPIRED","message":"instruction session expired","sql_state":"08003"}`) case r.URL.Path == "/object": storage.Add(1) - if mode == "storage401" { + if strings.HasPrefix(mode, "storage401") { w.WriteHeader(401) } else { w.Header().Set("Content-Length", "20") @@ -83,7 +99,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { require.NoError(t, writer.Close()) chunk := map[string]any{"chunk_index": 0, "row_offset": 0, "row_count": 1, "attachment": base64.StdEncoding.EncodeToString(attachment.Bytes())} chunks := 1 - if mode == "instruction401" { + if strings.HasPrefix(mode, "instruction401") { chunk["next_chunk_index"], chunks = 1, 2 } result = map[string]any{ @@ -101,15 +117,36 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { connection, err := db.Conn(ctx) require.NoError(t, err) defer connection.Close() - _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, []string{directory}), "GET instruction") + if strings.HasSuffix(mode, "Cancelled") { + err = connection.Raw(func(raw any) error { + operation, err := raw.(*conn).backend.Execute(ctx, backend.ExecRequest{Query: "GET instruction"}) + if err != nil { + return err + } + defer operation.Close(ctx) + cancelContext, cancelError := context.WithCancel(ctx) + defer cancelError() + stagingContext := driverctx.NewContextWithStagingInfo(&cancelOnCompletionErrorContext{Context: cancelContext, cancel: cancelError}, []string{directory}) + return operation.(interface{ CompleteStaging(context.Context) error }).CompleteStaging(stagingContext) + }) + require.ErrorIs(t, err, context.Canceled) + } else { + _, err = connection.ExecContext(driverctx.NewContextWithStagingInfo(ctx, []string{directory}), "GET instruction") + } require.Error(t, err) require.NotErrorIs(t, err, driver.ErrBadConn) var detail *kernel.KernelError require.ErrorAs(t, err, &detail) - wantCode := map[string]string{"instruction401": "SESSION_EXPIRED", "storage401": "VOLUME_STORAGE_HTTP_ERROR", "storageBody": "VOLUME_STORAGE_NETWORK_ERROR"}[mode] + wantCode := "VOLUME_STORAGE_HTTP_ERROR" + if strings.HasPrefix(mode, "instruction401") { + wantCode = "SESSION_EXPIRED" + } + if mode == "storageBody" { + wantCode = "VOLUME_STORAGE_NETWORK_ERROR" + } require.Equal(t, wantCode, detail.ErrorCode) require.NoError(t, connection.Raw(func(raw any) error { - require.Equal(t, mode != "instruction401", raw.(driver.Validator).IsValid()) + require.Equal(t, !strings.HasPrefix(mode, "instruction401"), raw.(driver.Validator).IsValid()) return nil })) data, err := os.ReadFile(destination) @@ -117,7 +154,7 @@ func TestKernelVolumeFailureOriginControlsSessionValidity(t *testing.T) { require.Equal(t, "old bytes", string(data)) require.Equal(t, int64(1), executions.Load()) wantStorage := int64(1) - if mode == "instruction401" { + if strings.HasPrefix(mode, "instruction401") { wantStorage = 0 } require.Equal(t, wantStorage, storage.Load()) From f62f4692bd8b1b0a8ae968aaada6ee6d87e9d3aa Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sat, 14 Nov 2026 23:54:44 +0000 Subject: [PATCH 08/19] build(kernel): select corrected Volume completion Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index bc57504b..53687ee1 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -855847bdca71e07285cfc60332fb2f33f8e47786 +a543bbc572b627f059461fce351713da8cb59df9 From 35a41a6fa7c4c4ab7741ad0f233afaa810768db2 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 01:48:44 +0000 Subject: [PATCH 09/19] build: select native Volume cancellation and error handling Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 53687ee1..3232e69a 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -a543bbc572b627f059461fce351713da8cb59df9 +566fe7f0a8a2fa246db95045a7000136ee6070d5 From 71a4ad859866ef96894489a83ca9b8ee647fd695 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 05:35:54 +0000 Subject: [PATCH 10/19] build: use native root authorization fix Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- internal/backend/kernel/include/databricks_kernel.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 3232e69a..e0f04c9a 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -566fe7f0a8a2fa246db95045a7000136ee6070d5 +de28034ec75aa6773a5e4e904d77f85511e1aa78 diff --git a/internal/backend/kernel/include/databricks_kernel.h b/internal/backend/kernel/include/databricks_kernel.h index a88286dc..b6e2ded7 100644 --- a/internal/backend/kernel/include/databricks_kernel.h +++ b/internal/backend/kernel/include/databricks_kernel.h @@ -375,6 +375,7 @@ KernelStatusCode kernel_session_config_set_sea_response_compression_enabled( /* Configure the total timeout for one HTTP request, from connection start * through response-body completion. A separate 30s connection timeout also * applies, so connection establishment is bounded by the shorter limit. + * Does not bound Volume storage requests or configure their inactivity limit. * `request_timeout_ms == 0` keeps the kernel default (120s). */ KernelStatusCode kernel_session_config_set_request_timeout(KernelSessionConfig* config, uint64_t request_timeout_ms); From 362f762bb1ffbbb406f335a4cbf1a794e502627c Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 07:56:57 +0000 Subject: [PATCH 11/19] build: update kernel source for replayable uploads Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index e0f04c9a..a85af789 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -de28034ec75aa6773a5e4e904d77f85511e1aa78 +8dd87c0ec0199cae6b3e5bf9cd973e7025409b45 From 855cca1548ebc370115058b0f98a573936b32c33 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 08:21:05 +0000 Subject: [PATCH 12/19] build: update kernel for local stream close Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index a85af789..e3eed1aa 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -8dd87c0ec0199cae6b3e5bf9cd973e7025409b45 +5b2b1da9aef7444f614d99f05cd7512a1afbe075 From 4393f2a4578508e5f3d0f74be0d10efb4936271a Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Sun, 15 Nov 2026 23:44:26 +0000 Subject: [PATCH 13/19] fix(volume): select replayable file uploads and tighten validation Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index e3eed1aa..d7abc8c7 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -5b2b1da9aef7444f614d99f05cd7512a1afbe075 +1347031d3c66e9179cec0a404efebe3685c26802 From 0f849de918a24db3d5ac1c6acb7185f4496c4e91 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 00:49:23 +0000 Subject: [PATCH 14/19] fix(kernel): use permission-preserving Volume downloads Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d7abc8c7..8707bb23 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -1347031d3c66e9179cec0a404efebe3685c26802 +f265b7d78e045d7d28f082084d60e2b0dd3ba367 From 6218f295601d5fbd1136443e54c62ddfe3a47101 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 01:05:07 +0000 Subject: [PATCH 15/19] fix(kernel): remove obsolete staging rejection example Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- examples/kernel/main.go | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/KERNEL_REV b/KERNEL_REV index 8707bb23..2df78f70 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -f265b7d78e045d7d28f082084d60e2b0dd3ba367 +de19f0dd234c5dc0894c6267001630c923901f32 diff --git a/examples/kernel/main.go b/examples/kernel/main.go index 51d99c31..e3f19775 100644 --- a/examples/kernel/main.go +++ b/examples/kernel/main.go @@ -87,14 +87,4 @@ func main() { log.Fatalf("query: %v", err) } fmt.Printf("server version: %s\n", version) - - // Options the kernel can't honor are rejected with an error wrapping - // ErrNotSupportedByKernel, never silently ignored. Staging (PUT/GET/REMOVE on a - // Unity Catalog volume) is one such feature, rejected at execute. - _, err = db.ExecContext(ctx, "PUT '/tmp/local.csv' INTO '/Volumes/main/default/vol/f.csv' OVERWRITE") - if errors.Is(err, dbsqlerr.ErrNotSupportedByKernel) { - fmt.Println("staging is not supported on the kernel backend (use Thrift for staging)") - } else if err != nil { - fmt.Printf("staging: %v\n", err) - } } From 886f7ee42e182e5cb441b0207a200b7183833b90 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 19:29:02 +0000 Subject: [PATCH 16/19] Pin kernel with statement lifecycle cleanup Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index cc9fd909..1b8d987f 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -30c8d8e7ae45b72cf2ff6b6333356a6b37fd533f +728f6cf0931a9e3c321f63534fce240115bdb2f5 From 21ca49b5cbcdc92021ebc3f3f93baf5e8dad1020 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 21:17:22 +0000 Subject: [PATCH 17/19] build: select kernel with async result cleanup Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index 1b8d987f..d6c502ff 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -728f6cf0931a9e3c321f63534fce240115bdb2f5 +029c59cc42727d0294e1a2dc378c78929030a919 From 74d74f9b761f1c4e86e0e770febd55496799190d Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 21:51:22 +0000 Subject: [PATCH 18/19] build: select kernel with Volume instruction error telemetry Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d6c502ff..d5bed17a 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -029c59cc42727d0294e1a2dc378c78929030a919 +ab4f133620fce52f9932508f0b245db630a57c48 From 0f9be07f5d937eb4e272584af2f99ed02873b9b3 Mon Sep 17 00:00:00 2001 From: Aakash Saravanan Date: Mon, 16 Nov 2026 23:57:11 +0000 Subject: [PATCH 19/19] build: pin kernel with complete Volume instruction handling Signed-off-by: Aakash Saravanan --- KERNEL_REV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KERNEL_REV b/KERNEL_REV index d5bed17a..28a4d122 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -ab4f133620fce52f9932508f0b245db630a57c48 +d9fcf834d36dc7f3de1621225db716200151fa0f