diff --git a/docs/provider-transactions.md b/docs/provider-transactions.md new file mode 100644 index 00000000..896ec7d9 --- /dev/null +++ b/docs/provider-transactions.md @@ -0,0 +1,49 @@ +# Provider transactions + +All SQL transactions delegate scheduling, journal/ledger writes, +locking, compensation and recovery to Omni. Plain BEGIN needs no SET. stackql retains SQL translation, +authentication and connection-bound handle storage. + +Defaults: collection `sql`, existing IaC state directory and dynamic lock scope. +Creates get distinct durable intents and retain a conservative resource-address lock. +A single body/toString inverse token supplies identity; ambiguous bindings need explicit targets. + +Optional scope/mode overrides: + +```sql +SET "$.transaction" = '{"collection":"demo","mode":"lazy","keys":["item"],"targets":{"stackql_unstable_demo.service.items":{"key":"item","identity":"id","addressed_by":"id"}}}'; +BEGIN; +-- provider INSERT / SELECT +SELECT * FROM stackql_preview.txn.status; +COMMIT; -- or ROLLBACK +``` + +`mode` accepts `eager` or `lazy`. If omitted, existing session `rollback_type` +configuration chooses execution timing. Reads and RETURNING flush preceding writes. +Rollback discards queued work and compensates executed work under retained locks. +`state` names an optional directory under the same caller/tenant IaC state root. + +Recovery: reconnect, configure the same collection/state with `resume_id`, then BEGIN. +Read status in the resumed session before proceeding; resume advances its version. For an uncertain statement, verify the provider state, +then call `stackql_preview.txn.recover` with `version`, JSON `resources` containing +`key`, `outcome` (`applied`/`absent`), observed resource `version`, and verified identity +for applied creates. `continue = 'true'` explicitly attests statement completion; +false aborts it for subsequent rollback. Recovery sends no provider request. + +Caller authentication and authorization are opt-in through the existing access configuration. +When enabled, ownership and permission checks apply to every transaction operation. +Without it, handles remain session-bound, but durable IDs do not establish user ownership. + +Current limits: only creates with a declared delete +inverse are supported. Unsupported mutations fail before dispatch. Configuration cannot +change inside a transaction; nested transactions are rejected. Consume read cursors before +COMMIT/ROLLBACK. Provider aliases are unsupported; each resource address currently binds +one logical target. Leases default to 300 seconds; execution lifetime defaults to 900 seconds. +Stop or isolate the previous executor before takeover: leases cannot fence provider +requests already in flight. Legacy coordinators, TSM, host inverse execution and the +`--acid` depth configuration are removed. Unsupported local/provider writes fail +before dispatch. No backend SQL ACID isolation is emulated. + +The new adapter requires an Omni release exposing `TransactionSession`. Local validation +uses a build-only modfile; pin the published release before normal CI. Protocol and +invariants: [Omni transactions](https://github.com/stackql-labs/omnisdk/blob/main/docs/developer/transactions.md). diff --git a/go.mod b/go.mod index 08208cf8..737647c5 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.12-alpha01 + github.com/stackql-labs/omnisdk v0.1.12-beta03 github.com/stackql/go-suffix-map v0.0.1-alpha02 github.com/stackql/psql-wire v0.1.2-beta02 github.com/stackql/stackql-parser v0.0.17-alpha05 diff --git a/go.sum b/go.sum index 70803b5e..394aa286 100644 --- a/go.sum +++ b/go.sum @@ -328,8 +328,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.12-alpha01 h1:M5pFE5fdClOQ3StZ+B7Ty9EvR0xbZG8u6Qm0JHgzmSA= -github.com/stackql-labs/omnisdk v0.1.12-alpha01/go.mod h1:G6qZy+6Vbuy9sSfws6erzpbbuThI+cD/uXDQPgQkRBI= +github.com/stackql-labs/omnisdk v0.1.12-beta03 h1:zw5mTcuR7OPYfwDNW4g+wbqA8G//CLNmr6XPSL4Bv+o= +github.com/stackql-labs/omnisdk v0.1.12-beta03/go.mod h1:G6qZy+6Vbuy9sSfws6erzpbbuThI+cD/uXDQPgQkRBI= github.com/stackql/go-suffix-map v0.0.1-alpha02 h1:JI4aiq/OfHW4a/BiNqK31NbyaR94zlUOhGZof3G/OUo= github.com/stackql/go-suffix-map v0.0.1-alpha02/go.mod h1:6BSMz4K/gIYfqdVRb7V8xL165wiPYyCjD9Kx7hLMd9Y= github.com/stackql/psql-wire v0.1.2-beta02 h1:7DfliBpyyCaUHVJ1Jxk37q3zVwVLyh6+FCHeM0/gbAQ= diff --git a/internal/stackql/acid/acid_dto/commit_dto.go b/internal/stackql/acid/acid_dto/commit_dto.go deleted file mode 100644 index 58e14fbc..00000000 --- a/internal/stackql/acid/acid_dto/commit_dto.go +++ /dev/null @@ -1,82 +0,0 @@ -package acid_dto //nolint:revive,stylecheck // meaning is clear - -import ( - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" -) - -var ( - _ CommitCoDomain = &standardCommitCoDomain{} -) - -type CommitCoDomain interface { - GetExecutorOutput() []internaldto.ExecutorOutput - GetError() (error, bool) - GetMessages() []string - GetUndoLog() (binlog.LogEntry, bool) -} - -type standardCommitCoDomain struct { - executorOutputs []internaldto.ExecutorOutput - votingPhaseError error - completionPhaseError error -} - -func NewCommitCoDomain( - executorOutputs []internaldto.ExecutorOutput, - votingPhaseError error, - completionPhaseError error, -) CommitCoDomain { - return &standardCommitCoDomain{ - executorOutputs: executorOutputs, - votingPhaseError: votingPhaseError, - completionPhaseError: completionPhaseError, - } -} - -func (c *standardCommitCoDomain) GetMessages() []string { - var messages []string - if c.votingPhaseError != nil { - messages = append(messages, c.votingPhaseError.Error()) - } - if c.completionPhaseError != nil { - messages = append(messages, c.completionPhaseError.Error()) - } - return messages -} - -func (c *standardCommitCoDomain) GetExecutorOutput() []internaldto.ExecutorOutput { - return c.executorOutputs -} - -func (c *standardCommitCoDomain) GetError() (error, bool) { //nolint:revive // permissable deviation from norm - if c.votingPhaseError != nil { - return c.votingPhaseError, true - } - return c.completionPhaseError, c.completionPhaseError != nil -} - -func (c *standardCommitCoDomain) GetUndoLog() (binlog.LogEntry, bool) { - var undoLogs []binlog.LogEntry - for _, executorOutput := range c.executorOutputs { - undoLog, undoLogExists := executorOutput.GetUndoLog() - if undoLogExists && undoLog != nil { - undoLogs = append(undoLogs, undoLog) - } - } - if len(undoLogs) == 0 { - return nil, false - } - initialUndoLog := undoLogs[len(undoLogs)-1] - rv := initialUndoLog.Clone() - for i := len(undoLogs) - 2; i >= 0; i-- { //nolint:mnd // magic number second from last - currentLog := undoLogs[i] - if currentLog != nil { - for _, s := range currentLog.GetHumanReadable() { - rv.AppendHumanReadable(s) - } - rv.AppendRaw(currentLog.GetRaw()) - } - } - return rv, true -} diff --git a/internal/stackql/acid/tsm/tsm.go b/internal/stackql/acid/tsm/tsm.go deleted file mode 100644 index 3ebd7121..00000000 --- a/internal/stackql/acid/tsm/tsm.go +++ /dev/null @@ -1,12 +0,0 @@ -package tsm - -// Transaction Storage Manager (TSM) -// A monolith containing: -// 1. Log Manager. Write Ahead Logging; WAL. -// 2. Lock Manager. 2 Phase Locking; 2PL... **not** to be confused with 2 Phase Commit. -// 3. Access Methods. In a conventional RDBMS B+-tree index and heap file access primitives, including latches. -// 4. Buffer Pool. It is possible to externalise this componsnes as it does not require shared internal -// awareness with other components (as oppose to the other 3 components). -type TSM interface { - // -} diff --git a/internal/stackql/acid/tsm_physio/access_methods.go b/internal/stackql/acid/tsm_physio/access_methods.go deleted file mode 100644 index fd56c0f2..00000000 --- a/internal/stackql/acid/tsm_physio/access_methods.go +++ /dev/null @@ -1,13 +0,0 @@ -package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature - -var ( - _ AccessMethods = (*accessMethods)(nil) -) - -type AccessMethods interface { - // -} - -type accessMethods struct { - // -} diff --git a/internal/stackql/acid/tsm_physio/best_effort_coordinator.go b/internal/stackql/acid/tsm_physio/best_effort_coordinator.go deleted file mode 100644 index ef236439..00000000 --- a/internal/stackql/acid/tsm_physio/best_effort_coordinator.go +++ /dev/null @@ -1,265 +0,0 @@ -package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature - -import ( - "fmt" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/acid_dto" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/acid/tsm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitivegraph" -) - -var ( - _ Coordinator = &basicBestEffortTransactionCoordinator{} -) - -type basicBestEffortTransactionCoordinator struct { - tsmInstance tsm.TSM - handlerCtx handler.HandlerContext - parent Coordinator - statementSequence []Statement - undoLogs []binlog.LogEntry - redoLogs []binlog.LogEntry - statementGraphs []primitivegraph.PrimitiveGraphHolder - maxTxnDepth int - outputs []internaldto.ExecutorOutput - isExecuted bool - // redoGraphs []primitivegraph.PrimitiveGraph - // undoGraphs []primitivegraph.PrimitiveGraph -} - -func newBasicBestEffortTransactionCoordinator( - tsmInstance tsm.TSM, - handlerCtx handler.HandlerContext, - parent Coordinator, - maxTxnDepth int, -) Coordinator { - return &basicBestEffortTransactionCoordinator{ - tsmInstance: tsmInstance, - handlerCtx: handlerCtx, - parent: parent, - maxTxnDepth: maxTxnDepth, - } -} - -func (m *basicBestEffortTransactionCoordinator) GetQuery() string { - return "" -} - -func (m *basicBestEffortTransactionCoordinator) GetPrimitiveGraphHolder() (primitivegraph.PrimitiveGraphHolder, bool) { - return nil, false -} - -func (m *basicBestEffortTransactionCoordinator) IsReadOnly() bool { - for _, statement := range m.statementSequence { - if !statement.IsReadOnly() { - return false - } - } - return true -} - -func (m *basicBestEffortTransactionCoordinator) GetAST() (sqlparser.Statement, bool) { - return nil, false -} - -func (m *basicBestEffortTransactionCoordinator) GetParent() (Coordinator, bool) { - return m.parent, m.parent != nil -} - -func (m *basicBestEffortTransactionCoordinator) AppendRedoLog(log binlog.LogEntry) { - m.redoLogs = append(m.redoLogs, log) -} - -func (m *basicBestEffortTransactionCoordinator) AppendUndoLog(log binlog.LogEntry) { - m.undoLogs = append(m.undoLogs, log) -} - -func (m *basicBestEffortTransactionCoordinator) IsBegin() bool { - return false -} - -func (m *basicBestEffortTransactionCoordinator) IsCommit() bool { - return false -} - -func (m *basicBestEffortTransactionCoordinator) IsRollback() bool { - return false -} - -func (m *basicBestEffortTransactionCoordinator) SetUndoLog(log binlog.LogEntry) { - m.undoLogs = []binlog.LogEntry{log} -} - -func (m *basicBestEffortTransactionCoordinator) GetRedoLog() (binlog.LogEntry, bool) { - rv := binlog.NewSimpleLogEntry(nil, nil) - if len(m.redoLogs) == 0 { - return nil, false - } - rv.Concatenate(m.redoLogs...) - return rv, true -} - -func (m *basicBestEffortTransactionCoordinator) GetUndoLog() (binlog.LogEntry, bool) { - rv := binlog.NewSimpleLogEntry(nil, nil) - if len(m.undoLogs) == 0 { - return nil, false - } - rv.Concatenate(m.undoLogs...) - return rv, true -} - -func (m *basicBestEffortTransactionCoordinator) Prepare() error { - var err error - for _, stmt := range m.statementSequence { - err = stmt.Prepare() - if err != nil { - return err - } - } - return nil -} - -// This is an approximation of 2PC. -func (m *basicBestEffortTransactionCoordinator) Execute() internaldto.ExecutorOutput { - outputs, err := m.votingPhase(nil) - m.outputs = outputs - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - return internaldto.NewExecutorOutput( - nil, - nil, - nil, - internaldto.NewBackendMessages([]string{"transaction committed"}), - nil, - ) -} - -func (m *basicBestEffortTransactionCoordinator) IsExecuted() bool { - return m.outputs != nil -} - -func (m *basicBestEffortTransactionCoordinator) votingPhase( - commitCtx handler.HandlerContext, -) ([]internaldto.ExecutorOutput, error) { - defer func() { - m.isExecuted = true - }() - var rv []internaldto.ExecutorOutput - for _, stmt := range m.statementSequence { - coDomain := executeQueuedStatement(stmt, commitCtx) - rv = append(rv, coDomain) - err := coDomain.GetError() - undoLog, undoLogExists := coDomain.GetUndoLog() - redoLog, redoLogExists := coDomain.GetRedoLog() - if undoLogExists { - m.undoLogs = append(m.undoLogs, undoLog) - } - if redoLogExists { - m.redoLogs = append(m.redoLogs, redoLog) - } - if err != nil { - return rv, err - } - } - return rv, nil -} - -func (m *basicBestEffortTransactionCoordinator) completionPhase() error { - return nil -} - -func (m *basicBestEffortTransactionCoordinator) Begin() (Coordinator, error) { - if m.maxTxnDepth >= 0 && m.Depth() >= m.maxTxnDepth { - return nil, fmt.Errorf("cannot begin nested transaction of depth = %d", m.Depth()+1) - } - return newBasicBestEffortTransactionCoordinator(m.tsmInstance, m.handlerCtx, m, m.maxTxnDepth), nil -} - -func (m *basicBestEffortTransactionCoordinator) Commit(commitCtx handler.HandlerContext) acid_dto.CommitCoDomain { - rv, err := m.votingPhase(commitCtx) - if err != nil { - return acid_dto.NewCommitCoDomain(rv, err, nil) - } - completionErr := m.completionPhase() - return acid_dto.NewCommitCoDomain(rv, nil, completionErr) -} - -// Rollback is best effort and runs in reverse order. -func (m *basicBestEffortTransactionCoordinator) Rollback() acid_dto.CommitCoDomain { - var coDomains []internaldto.ExecutorOutput - for i := len(m.statementGraphs) - 1; i >= 0; i-- { - stmt := m.statementGraphs[i] - pl := internaldto.NewBasicPrimitiveContext( - nil, - m.handlerCtx.GetOutfile(), - m.handlerCtx.GetOutErrFile(), - ) - inverseGraph := stmt.GetInversePrimitiveGraph() - if inverseGraph == nil { - return acid_dto.NewCommitCoDomain( - nil, - nil, - fmt.Errorf("cannot rollback statement without inverse primitive graph"), - ) - } - optimiseErr := inverseGraph.Optimise() - if optimiseErr != nil { - return acid_dto.NewCommitCoDomain( - nil, - nil, - optimiseErr, - ) - } - coDomain := stmt.GetInversePrimitiveGraph().Execute(pl) - coDomains = append(coDomains, coDomain) - if coDomain.GetError() != nil { - return acid_dto.NewCommitCoDomain( - coDomains, - nil, - coDomain.GetError(), - ) - } - } - return acid_dto.NewCommitCoDomain( - coDomains, - nil, - nil, - ) -} - -func (m *basicBestEffortTransactionCoordinator) Enqueue(stmt Statement) error { - graphHolder, graphHolderExists := stmt.GetPrimitiveGraphHolder() - if !graphHolderExists { - return fmt.Errorf("cannot enqueue statement without primitive graph holder") - } - reversal := graphHolder.GetInversePrimitiveGraph() - if reversal == nil { - return fmt.Errorf("cannot enqueue statement without inverse primitive graph") - } - if reversal.Size() < 1 { - return fmt.Errorf("cannot enqueue statement with empty inverse primitive graph") - } - m.statementGraphs = append(m.statementGraphs, graphHolder) - m.statementSequence = append(m.statementSequence, stmt) - return nil -} - -func (m *basicBestEffortTransactionCoordinator) Depth() int { - return m.depth() -} - -func (m *basicBestEffortTransactionCoordinator) IsRoot() bool { - return m.parent == nil -} - -func (m *basicBestEffortTransactionCoordinator) depth() int { - if m.parent != nil { - return m.parent.Depth() + 1 - } - return 0 -} diff --git a/internal/stackql/acid/tsm_physio/best_effort_orchestrator.go b/internal/stackql/acid/tsm_physio/best_effort_orchestrator.go deleted file mode 100644 index c2429fd0..00000000 --- a/internal/stackql/acid/tsm_physio/best_effort_orchestrator.go +++ /dev/null @@ -1,214 +0,0 @@ -package tsm_physio //nolint:revive,stylecheck // prefer this nomenclature - -import ( - "fmt" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/acid/tsm" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitivegraph" -) - -var ( - _ Orchestrator = &bestEffortOrchestrator{} -) - -// This orchestrator: -// - Supports a simple reversibility semantic. -// - In cases of network partitioning or other failures, -// it will simply spew suggested undo logs. -// -//nolint:unused // TODO: fix this. -type bestEffortOrchestrator struct { - tsmInstance tsm.TSM - txnCoordinator Coordinator - undoLogs []binlog.LogEntry - redoLogs []binlog.LogEntry - redoGraphs []primitivegraph.PrimitiveGraph - undoGraphs []primitivegraph.PrimitiveGraph -} - -func (orc *bestEffortOrchestrator) ProcessQueryOrQueries( - handlerCtx handler.HandlerContext, -) ([]internaldto.ExecutorOutput, bool) { - return orc.processQueryOrQueries(handlerCtx) -} - -func (orc *bestEffortOrchestrator) processQueryOrQueries( - handlerCtx handler.HandlerContext, -) ([]internaldto.ExecutorOutput, bool) { - var retVal []internaldto.ExecutorOutput - cmdString := handlerCtx.GetRawQuery() - splitQueries, _ := sqlparser.SplitStatementToPieces(cmdString) - for _, s := range splitQueries { - response, hasResponse := orc.processQuery(handlerCtx, s) - if hasResponse { - retVal = append(retVal, response...) - } - } - return retVal, len(retVal) > 0 -} - -func (orc *bestEffortOrchestrator) undo(precedingMessages []string) ([]internaldto.ExecutorOutput, bool) { - rollbackREsponse := orc.txnCoordinator.Rollback() - rollbackErr, rollbackErrExists := rollbackREsponse.GetError() - if rollbackErrExists { - return []internaldto.ExecutorOutput{ - internaldto.NewNopEmptyExecutorOutput( - precedingMessages, - ), - internaldto.NewErroneousExecutorOutput(rollbackErr), - }, true - } - return []internaldto.ExecutorOutput{ - internaldto.NewNopEmptyExecutorOutput( - precedingMessages, - ), - internaldto.NewNopEmptyExecutorOutput( - []string{ - "rollback successful", - }, - ), - }, true -} - -//nolint:gocognit,funlen // TODO: review -func (orc *bestEffortOrchestrator) processQuery( - handlerCtx handler.HandlerContext, - query string, -) ([]internaldto.ExecutorOutput, bool) { - if query == "" { - return nil, false - } - clonedCtx := handlerCtx.Clone() - clonedCtx.SetQuery(query) - transactStatement := NewStatement(query, clonedCtx, txn_context.NewTransactionContext(orc.txnCoordinator.Depth())) - prepareErr := transactStatement.Prepare() - if prepareErr != nil { - return []internaldto.ExecutorOutput{ - internaldto.NewErroneousExecutorOutput(prepareErr), - }, true - } - isReadOnly := transactStatement.IsReadOnly() - // TODO: implement eager execution for non-mutating statements - // and lazy execution for mutating statements. - // TODO: implement transaction stack. - if transactStatement.IsBegin() { //nolint:gocritic,nestif // TODO: review - txnCoordinator, beginErr := orc.txnCoordinator.Begin() - if beginErr != nil { - return []internaldto.ExecutorOutput{ - internaldto.NewErroneousExecutorOutput(beginErr), - }, true - } - orc.txnCoordinator = txnCoordinator - return []internaldto.ExecutorOutput{ - internaldto.NewNopEmptyExecutorOutput([]string{"OK"}), - }, true - } else if transactStatement.IsCommit() { - commitCoDomain := orc.txnCoordinator.Commit(handlerCtx) - commitErr, commitErrExists := commitCoDomain.GetError() - if commitErrExists { - return orc.undo([]string{ - commitErr.Error(), - }) - } - retVal := commitCoDomain.GetExecutorOutput() - parent, hasParent := orc.txnCoordinator.GetParent() - if hasParent { - orc.txnCoordinator = parent - retVal = append(retVal, internaldto.NewNopEmptyExecutorOutput([]string{"OK"})) - return retVal, true - } - noParentErr := fmt.Errorf("%s", noParentMessage) - retVal = append(retVal, internaldto.NewErroneousExecutorOutput(noParentErr)) - return retVal, true - } else if transactStatement.IsRollback() { - var retVal []internaldto.ExecutorOutput - rollbackREsponse := orc.txnCoordinator.Rollback() - rollbackErr, rollbackErrExists := rollbackREsponse.GetError() - if rollbackErrExists { - retVal = append(retVal, internaldto.NewErroneousExecutorOutput(rollbackErr)) - retVal = append(retVal, internaldto.NewErroneousExecutorOutput( - fmt.Errorf("Rollback failed"))) - return retVal, true - } - parent, hasParent := orc.txnCoordinator.GetParent() - if hasParent { - orc.txnCoordinator = parent - for _, g := range orc.undoGraphs { - undoOutput := g.Execute(nil) - if undoOutput.GetError() != nil { - retVal = append(retVal, internaldto.NewErroneousExecutorOutput(undoOutput.GetError())) - retVal = append(retVal, internaldto.NewErroneousExecutorOutput( - fmt.Errorf("Rollback failed"))) - return retVal, true - } - } - retVal = append(retVal, internaldto.NewNopEmptyExecutorOutput([]string{"Rollback OK"})) - return retVal, true - } - retVal = append( - retVal, - internaldto.NewErroneousExecutorOutput( - fmt.Errorf("%s", noParentMessage)), - ) - return retVal, true - } - if isReadOnly || orc.txnCoordinator.IsRoot() { - stmtOutput := transactStatement.Execute() - return []internaldto.ExecutorOutput{ - stmtOutput, - }, true - } - - primitiveGraphHolder, primitiveGraphHolderExists := transactStatement.GetPrimitiveGraphHolder() - if !primitiveGraphHolderExists { - // bail - undoMessage := "primitive graph holder does not exist" - if query != "" { - undoMessage = fmt.Sprintf("primitive graph holder does not exist for query: '%s'", query) - } - return orc.undo([]string{ - undoMessage, - }) - } - - enqueueError := orc.txnCoordinator.Enqueue(transactStatement) - - // Before bailing on eager execution error, - // first assemble undo graph. - undoGraphSize := primitiveGraphHolder.GetInversePrimitiveGraph().Size() - if undoGraphSize == 0 { - // bail - undoMessage := "undo graph does not exist" - if query != "" { - undoMessage = fmt.Sprintf("undo graph does not exist for query: '%s'", query) - } - return orc.undo([]string{ - undoMessage, - }) - } - - redoLog, redoLogExists := transactStatement.GetRedoLog() - if redoLogExists { - orc.redoLogs = append(orc.redoLogs, redoLog) - } - - if enqueueError != nil { - // bail - return orc.undo([]string{ - enqueueError.Error(), - }) - } - output := transactStatement.Execute() - if output.GetError() != nil { - // bail - return orc.undo([]string{ - output.GetError().Error(), - }) - } - return []internaldto.ExecutorOutput{output}, true -} diff --git a/internal/stackql/acid/tsm_physio/buffer_pool.go b/internal/stackql/acid/tsm_physio/buffer_pool.go deleted file mode 100644 index 495775ea..00000000 --- a/internal/stackql/acid/tsm_physio/buffer_pool.go +++ /dev/null @@ -1,11 +0,0 @@ -package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature - -var ( - _ BufferPool = (*bufferPool)(nil) -) - -type BufferPool interface { - // -} - -type bufferPool struct{} diff --git a/internal/stackql/acid/tsm_physio/coordinator.go b/internal/stackql/acid/tsm_physio/coordinator.go deleted file mode 100644 index 57622f19..00000000 --- a/internal/stackql/acid/tsm_physio/coordinator.go +++ /dev/null @@ -1,45 +0,0 @@ -package tsm_physio //nolint:revive,stylecheck // prefer this nomenclature - -import ( - "github.com/stackql/stackql/internal/stackql/acid/acid_dto" - "github.com/stackql/stackql/internal/stackql/acid/tsm" - "github.com/stackql/stackql/internal/stackql/constants" - "github.com/stackql/stackql/internal/stackql/handler" -) - -func newCoordinator(tsmInstance tsm.TSM, handlerCtx handler.HandlerContext, maxTxnDepth int) Coordinator { - rollbackType := handlerCtx.GetRollbackType() - switch rollbackType { - case constants.NopRollback: - return newBasicLazyTransactionCoordinator(tsmInstance, nil, maxTxnDepth) - case constants.EagerRollback: - return newBasicBestEffortTransactionCoordinator(tsmInstance, handlerCtx, nil, maxTxnDepth) - default: - return newBasicLazyTransactionCoordinator(tsmInstance, nil, maxTxnDepth) - } -} - -// The transaction coordinator ensures -// that undo and redo logs are kept -// and that 2PC is performed. -type Coordinator interface { - Statement - // Begin a new transaction. - Begin() (Coordinator, error) - // Commit the current transaction. - Commit(handler.HandlerContext) acid_dto.CommitCoDomain - // Rollback the current transaction. - Rollback() acid_dto.CommitCoDomain - // Enqueue a transaction operation. - // This method will return an error - // in the case that the transaction - // context disallows a particular - // operation or type of operation. - Enqueue(Statement) error - // Get the depth of transaction nesting. - Depth() int - // Get the parent transaction manager. - GetParent() (Coordinator, bool) - // - IsRoot() bool -} diff --git a/internal/stackql/acid/tsm_physio/lazy_coordinator.go b/internal/stackql/acid/tsm_physio/lazy_coordinator.go deleted file mode 100644 index 54a9f966..00000000 --- a/internal/stackql/acid/tsm_physio/lazy_coordinator.go +++ /dev/null @@ -1,211 +0,0 @@ -package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature - -import ( - "fmt" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/acid_dto" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/acid/tsm" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitivegraph" -) - -var ( - _ Coordinator = &basicLazyTransactionCoordinator{} -) - -type basicLazyTransactionCoordinator struct { - tsmInstance tsm.TSM - parent Coordinator - statementSequence []Statement - undoLogs []binlog.LogEntry - redoLogs []binlog.LogEntry - maxTxnDepth int - outputs []internaldto.ExecutorOutput - isExecuted bool -} - -func newBasicLazyTransactionCoordinator(tsmInstance tsm.TSM, parent Coordinator, maxTxnDepth int) Coordinator { - return &basicLazyTransactionCoordinator{ - tsmInstance: tsmInstance, - parent: parent, - maxTxnDepth: maxTxnDepth, - } -} - -func (m *basicLazyTransactionCoordinator) GetPrimitiveGraphHolder() (primitivegraph.PrimitiveGraphHolder, bool) { - return nil, false -} - -func (m *basicLazyTransactionCoordinator) IsReadOnly() bool { - for _, statement := range m.statementSequence { - if !statement.IsReadOnly() { - return false - } - } - return true -} - -func (m *basicLazyTransactionCoordinator) GetAST() (sqlparser.Statement, bool) { - return nil, false -} - -func (m *basicLazyTransactionCoordinator) GetParent() (Coordinator, bool) { - return m.parent, m.parent != nil -} - -func (m *basicLazyTransactionCoordinator) AppendRedoLog(log binlog.LogEntry) { - m.redoLogs = append(m.redoLogs, log) -} - -func (m *basicLazyTransactionCoordinator) AppendUndoLog(log binlog.LogEntry) { - m.undoLogs = append(m.undoLogs, log) -} - -func (m *basicLazyTransactionCoordinator) IsBegin() bool { - return false -} - -func (m *basicLazyTransactionCoordinator) IsCommit() bool { - return false -} - -func (m *basicLazyTransactionCoordinator) IsRollback() bool { - return false -} - -func (m *basicLazyTransactionCoordinator) SetUndoLog(log binlog.LogEntry) { - m.undoLogs = []binlog.LogEntry{log} -} - -func (m *basicLazyTransactionCoordinator) GetRedoLog() (binlog.LogEntry, bool) { - rv := binlog.NewSimpleLogEntry(nil, nil) - if len(m.redoLogs) == 0 { - return nil, false - } - rv.Concatenate(m.redoLogs...) - return rv, true -} - -func (m *basicLazyTransactionCoordinator) GetUndoLog() (binlog.LogEntry, bool) { - rv := binlog.NewSimpleLogEntry(nil, nil) - if len(m.undoLogs) == 0 { - return nil, false - } - rv.Concatenate(m.undoLogs...) - return rv, true -} - -func (m *basicLazyTransactionCoordinator) Prepare() error { - var err error - for _, stmt := range m.statementSequence { - err = stmt.Prepare() - if err != nil { - return err - } - } - return nil -} - -// This is an approximation of 2PC. -func (m *basicLazyTransactionCoordinator) Execute() internaldto.ExecutorOutput { - outputs, err := m.votingPhase(nil) - m.outputs = outputs - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - return internaldto.NewExecutorOutput( - nil, - nil, - nil, - internaldto.NewBackendMessages([]string{"transaction committed"}), - nil, - ) -} - -func (m *basicLazyTransactionCoordinator) IsExecuted() bool { - return m.outputs != nil -} - -func (m *basicLazyTransactionCoordinator) votingPhase( - commitCtx handler.HandlerContext, -) ([]internaldto.ExecutorOutput, error) { - defer func() { - m.isExecuted = true - }() - var rv []internaldto.ExecutorOutput - for _, stmt := range m.statementSequence { - coDomain := executeQueuedStatement(stmt, commitCtx) - rv = append(rv, coDomain) - err := coDomain.GetError() - undoLog, undoLogExists := coDomain.GetUndoLog() - redoLog, redoLogExists := coDomain.GetRedoLog() - if undoLogExists { - m.undoLogs = append(m.undoLogs, undoLog) - } - if redoLogExists { - m.redoLogs = append(m.redoLogs, redoLog) - } - if err != nil { - return rv, err - } - } - return rv, nil -} - -func (m *basicLazyTransactionCoordinator) completionPhase() error { - return nil -} - -func (m *basicLazyTransactionCoordinator) GetQuery() string { - return "" -} - -func (m *basicLazyTransactionCoordinator) Begin() (Coordinator, error) { - if m.maxTxnDepth >= 0 && m.Depth() >= m.maxTxnDepth { - return nil, fmt.Errorf("cannot begin nested transaction of depth = %d", m.Depth()+1) - } - return newBasicLazyTransactionCoordinator(m.tsmInstance, m, m.maxTxnDepth), nil -} - -func (m *basicLazyTransactionCoordinator) Commit(commitCtx handler.HandlerContext) acid_dto.CommitCoDomain { - rv, err := m.votingPhase(commitCtx) - if err != nil { - return acid_dto.NewCommitCoDomain(rv, err, nil) - } - completionErr := m.completionPhase() - return acid_dto.NewCommitCoDomain(rv, nil, completionErr) -} - -// Rollback is a no-op for now. -// The redo logs will simply be -// displayed to the user. -func (m *basicLazyTransactionCoordinator) Rollback() acid_dto.CommitCoDomain { - return acid_dto.NewCommitCoDomain( - nil, - nil, - nil, - ) -} - -func (m *basicLazyTransactionCoordinator) Enqueue(stmt Statement) error { - m.statementSequence = append(m.statementSequence, stmt) - return nil -} - -func (m *basicLazyTransactionCoordinator) Depth() int { - return m.depth() -} - -func (m *basicLazyTransactionCoordinator) IsRoot() bool { - return m.parent == nil -} - -func (m *basicLazyTransactionCoordinator) depth() int { - if m.parent != nil { - return m.parent.Depth() + 1 - } - return 0 -} diff --git a/internal/stackql/acid/tsm_physio/lock_manager.go b/internal/stackql/acid/tsm_physio/lock_manager.go deleted file mode 100644 index 7540e583..00000000 --- a/internal/stackql/acid/tsm_physio/lock_manager.go +++ /dev/null @@ -1,11 +0,0 @@ -package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature - -var ( - _ LockManager = (*lockManager)(nil) -) - -type LockManager interface { - // -} - -type lockManager struct{} diff --git a/internal/stackql/acid/tsm_physio/log_manager.go b/internal/stackql/acid/tsm_physio/log_manager.go deleted file mode 100644 index de54110d..00000000 --- a/internal/stackql/acid/tsm_physio/log_manager.go +++ /dev/null @@ -1,33 +0,0 @@ -package tsm_physio //nolint:revive,stylecheck // prefer this nomenclature -import ( - "sync" - - "github.com/stackql/stackql/internal/stackql/handler" -) - -var ( - _ LogManager = (*walManager)(nil) -) - -//nolint:gochecknoglobals // singleton pattern -var ( - walOnce sync.Once - walSingleton LogManager -) - -type LogManager interface { - // -} - -type walManager struct{} - -func getWalManager(_ handler.HandlerContext) (LogManager, error) { - var err error - walOnce.Do(func() { - if err != nil { - return - } - walSingleton = &walManager{} - }) - return walSingleton, err -} diff --git a/internal/stackql/acid/tsm_physio/statement.go b/internal/stackql/acid/tsm_physio/statement.go deleted file mode 100644 index 68c39871..00000000 --- a/internal/stackql/acid/tsm_physio/statement.go +++ /dev/null @@ -1,138 +0,0 @@ -package tsm_physio //nolint:stylecheck // prefer this nomenclature - -import ( - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" - "github.com/stackql/stackql/internal/stackql/primitivegraph" - "github.com/stackql/stackql/internal/stackql/querysubmit" -) - -type Statement interface { - Prepare() error - Execute() internaldto.ExecutorOutput - GetAST() (sqlparser.Statement, bool) - GetPrimitiveGraphHolder() (primitivegraph.PrimitiveGraphHolder, bool) - GetUndoLog() (binlog.LogEntry, bool) - GetRedoLog() (binlog.LogEntry, bool) - IsReadOnly() bool - IsBegin() bool - IsCommit() bool - IsExecuted() bool - IsRollback() bool - GetQuery() string -} - -type basicStatement struct { - query string - handlerCtx handler.HandlerContext - querySubmitter querysubmit.QuerySubmitter - transactionContext txn_context.ITransactionContext - isExecuted bool -} - -func NewStatement( - query string, - handlerCtx handler.HandlerContext, - transactionContext txn_context.ITransactionContext, -) Statement { - return &basicStatement{ - query: query, - handlerCtx: handlerCtx, - querySubmitter: querysubmit.NewQuerySubmitter(), - transactionContext: transactionContext, - } -} - -func (st *basicStatement) GetQuery() string { - return st.query -} - -func (st *basicStatement) IsBegin() bool { - ast, hasAst := st.GetAST() - if hasAst { - _, isBegin := ast.(*sqlparser.Begin) - return isBegin - } - return false -} - -func (st *basicStatement) IsCommit() bool { - ast, hasAst := st.GetAST() - if hasAst { - _, isCommit := ast.(*sqlparser.Commit) - return isCommit - } - return false -} - -func (st *basicStatement) IsRollback() bool { - ast, hasAst := st.GetAST() - if hasAst { - _, isRollback := ast.(*sqlparser.Rollback) - return isRollback - } - return false -} - -func (st *basicStatement) IsReadOnly() bool { - if st.querySubmitter == nil { - return true - } - return st.querySubmitter.IsReadOnly() -} - -func (st *basicStatement) GetUndoLog() (binlog.LogEntry, bool) { - if st.querySubmitter != nil { - st.querySubmitter.GetUndoLog() - } - return nil, false -} - -func (st *basicStatement) GetRedoLog() (binlog.LogEntry, bool) { - if st.querySubmitter != nil { - st.querySubmitter.GetRedoLog() - } - return nil, false -} - -func (st *basicStatement) Prepare() error { - cmdString := st.query - clonedCtx := st.handlerCtx.Clone() - clonedCtx.SetQuery(cmdString) - if st.transactionContext != nil { - st.querySubmitter = st.querySubmitter.WithTransactionContext(st.transactionContext) - } - return st.querySubmitter.PrepareQuery(clonedCtx) -} - -func (st *basicStatement) Execute() internaldto.ExecutorOutput { - st.isExecuted = true - return st.querySubmitter.SubmitQuery() -} - -func (st *basicStatement) IsExecuted() bool { - return st.isExecuted -} - -func (st *basicStatement) GetAST() (sqlparser.Statement, bool) { - return st.querySubmitter.GetStatement() -} - -func (st *basicStatement) GetPrimitiveGraphHolder() (primitivegraph.PrimitiveGraphHolder, bool) { - return st.querySubmitter.GetPrimitiveGraphHolder() -} - -// Deferred work runs under the COMMIT request, rather than the already completed -// request that queued it. Replanning also refreshes caller authorization. -func executeQueuedStatement(stmt Statement, commitCtx handler.HandlerContext) internaldto.ExecutorOutput { - if queued, ok := stmt.(*basicStatement); ok && commitCtx != nil && !queued.IsExecuted() { - queued.handlerCtx.SetExecutionContext(commitCtx.GetExecutionContext()) - if err := queued.Prepare(); err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - } - return stmt.Execute() -} diff --git a/internal/stackql/acid/tsm_physio/tsm_implementation.go b/internal/stackql/acid/tsm_physio/tsm_implementation.go deleted file mode 100644 index 98f06b94..00000000 --- a/internal/stackql/acid/tsm_physio/tsm_implementation.go +++ /dev/null @@ -1,24 +0,0 @@ -package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature - -import ( - "github.com/stackql/stackql/internal/stackql/acid/tsm" - "github.com/stackql/stackql/internal/stackql/handler" -) - -//nolint:unused // TODO: finalise pattern -type tsmImplementation struct { - accessMethods AccessMethods - bufferPool BufferPool - lockManager LockManager - logManager LogManager -} - -func GetTSM(handlerCtx handler.HandlerContext) (tsm.TSM, error) { - walManager, walErr := getWalManager(handlerCtx) - if walErr != nil { - return nil, walErr - } - return &tsmImplementation{ - logManager: walManager, - }, nil -} diff --git a/internal/stackql/acid/tsm_physio/txn_orchestrator.go b/internal/stackql/acid/tsm_physio/txn_orchestrator.go deleted file mode 100644 index 41b06b77..00000000 --- a/internal/stackql/acid/tsm_physio/txn_orchestrator.go +++ /dev/null @@ -1,173 +0,0 @@ -package tsm_physio //nolint:stylecheck,revive // prefer this nomenclature - -import ( - "fmt" - - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/tsm" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" - "github.com/stackql/stackql/internal/stackql/constants" - "github.com/stackql/stackql/internal/stackql/handler" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" -) - -type Orchestrator interface { - ProcessQueryOrQueries( - handlerCtx handler.HandlerContext, - ) ([]internaldto.ExecutorOutput, bool) -} - -func newTxnOrchestrator( - tsmInstance tsm.TSM, - handlerCtx handler.HandlerContext, - txnCoordinator Coordinator) (Orchestrator, error) { - rollbackType := handlerCtx.GetRollbackType() - switch rollbackType { - case constants.NopRollback: - return newStdTxnOrchestrator(tsmInstance, handlerCtx, txnCoordinator) - case constants.EagerRollback: - return newBestEffortTxnOrchestrator(tsmInstance, handlerCtx, txnCoordinator) - default: - return newStdTxnOrchestrator(tsmInstance, handlerCtx, txnCoordinator) - } -} - -func newStdTxnOrchestrator( - tsmInstance tsm.TSM, - _ handler.HandlerContext, - txnCoordinator Coordinator) (Orchestrator, error) { - return &standardOrchestrator{ - tsmInstance: tsmInstance, - txnCoordinator: txnCoordinator, - }, nil -} - -func newBestEffortTxnOrchestrator( - tsmInstance tsm.TSM, - _ handler.HandlerContext, - txnCoordinator Coordinator) (Orchestrator, error) { - return &bestEffortOrchestrator{ - tsmInstance: tsmInstance, - txnCoordinator: txnCoordinator, - }, nil -} - -type standardOrchestrator struct { - tsmInstance tsm.TSM - txnCoordinator Coordinator -} - -func (orc *standardOrchestrator) ProcessQueryOrQueries( - handlerCtx handler.HandlerContext, -) ([]internaldto.ExecutorOutput, bool) { - return orc.processQueryOrQueries(handlerCtx) -} - -func (orc *standardOrchestrator) processQueryOrQueries( - handlerCtx handler.HandlerContext, -) ([]internaldto.ExecutorOutput, bool) { - var retVal []internaldto.ExecutorOutput - cmdString := handlerCtx.GetRawQuery() - splitQueries, _ := sqlparser.SplitStatementToPieces(cmdString) - for _, s := range splitQueries { - response, hasResponse := orc.processQuery(handlerCtx, s) - if hasResponse { - retVal = append(retVal, response...) - } - } - return retVal, len(retVal) > 0 -} - -//nolint:gocognit // TODO: review -func (orc *standardOrchestrator) processQuery( - handlerCtx handler.HandlerContext, - query string, -) ([]internaldto.ExecutorOutput, bool) { - if query == "" { - return nil, false - } - clonedCtx := handlerCtx.Clone() - clonedCtx.SetQuery(query) - transactStatement := NewStatement(query, clonedCtx, txn_context.NewTransactionContext(orc.txnCoordinator.Depth())) - prepareErr := transactStatement.Prepare() - if prepareErr != nil { - return []internaldto.ExecutorOutput{ - internaldto.NewErroneousExecutorOutput(prepareErr), - }, true - } - isReadOnly := transactStatement.IsReadOnly() - // TODO: implement eager execution for non-mutating statements - // and lazy execution for mutating statements. - // TODO: implement transaction stack. - if transactStatement.IsBegin() { //nolint:gocritic,nestif // TODO: review - txnCoordinator, beginErr := orc.txnCoordinator.Begin() - if beginErr != nil { - return []internaldto.ExecutorOutput{ - internaldto.NewErroneousExecutorOutput(beginErr), - }, true - } - orc.txnCoordinator = txnCoordinator - return []internaldto.ExecutorOutput{ - internaldto.NewNopEmptyExecutorOutput([]string{"OK"}), - }, true - } else if transactStatement.IsCommit() { - commitCoDomain := orc.txnCoordinator.Commit(handlerCtx) - commitErr, commitErrExists := commitCoDomain.GetError() - if commitErrExists { - retVal := []internaldto.ExecutorOutput{ - internaldto.NewErroneousExecutorOutput(commitErr), - } - undoLog, undoLogExists := commitCoDomain.GetUndoLog() - if undoLogExists && undoLog != nil { - humanReadable := undoLog.GetHumanReadable() - if len(humanReadable) > 0 { - displayUndoLogs := make([]string, len(humanReadable)) - for i, h := range humanReadable { - displayUndoLogs[i] = fmt.Sprintf("UNDO required: %s", h) - } - retVal = append(retVal, internaldto.NewNopEmptyExecutorOutput(displayUndoLogs)) - } - } - return retVal, true - } - retVal := commitCoDomain.GetExecutorOutput() - parent, hasParent := orc.txnCoordinator.GetParent() - if hasParent { - orc.txnCoordinator = parent - retVal = append(retVal, internaldto.NewNopEmptyExecutorOutput([]string{"OK"})) - return retVal, true - } - noParentErr := fmt.Errorf("%s", noParentMessage) - retVal = append(retVal, internaldto.NewErroneousExecutorOutput(noParentErr)) - return retVal, true - } else if transactStatement.IsRollback() { - var retVal []internaldto.ExecutorOutput - rollbackREsponse := orc.txnCoordinator.Rollback() - rollbackErr, rollbackErrExists := rollbackREsponse.GetError() - if rollbackErrExists { - retVal = append(retVal, internaldto.NewErroneousExecutorOutput(rollbackErr)) - } - parent, hasParent := orc.txnCoordinator.GetParent() - if hasParent { - orc.txnCoordinator = parent - retVal = append(retVal, internaldto.NewNopEmptyExecutorOutput([]string{"Rollback OK"})) - return retVal, true - } - retVal = append( - retVal, - internaldto.NewErroneousExecutorOutput( - fmt.Errorf("%s", noParentMessage)), - ) - return retVal, true - } - if isReadOnly || orc.txnCoordinator.IsRoot() { - stmtOutput := transactStatement.Execute() - return []internaldto.ExecutorOutput{ - stmtOutput, - }, true - } - orc.txnCoordinator.Enqueue(transactStatement) //nolint:errcheck // TODO: investigate - return []internaldto.ExecutorOutput{ - internaldto.NewNopEmptyExecutorOutput([]string{"mutating statement queued"}), - }, true -} diff --git a/internal/stackql/acid/tsm_physio/txn_provider.go b/internal/stackql/acid/tsm_physio/txn_provider.go deleted file mode 100644 index aa1f6c76..00000000 --- a/internal/stackql/acid/tsm_physio/txn_provider.go +++ /dev/null @@ -1,78 +0,0 @@ -package tsm_physio //nolint:revive,stylecheck // prefer this nomenclature - -import ( - "sync" - - "github.com/stackql/stackql/internal/stackql/acid/tsm" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" - "github.com/stackql/stackql/internal/stackql/handler" -) - -//nolint:gochecknoglobals // singleton pattern -var ( - providerOnce sync.Once - providerSingleton Provider - _ Provider = &standardProvider{} - noParentMessage string = "no parent transaction manager available" //nolint:gochecknoglobals,revive,lll // permissable -) - -const ( - defaultMaxStackDepth = 1 -) - -// The transaction provider is singleton -// that orchestrates transaction managers. -type Provider interface { - // Create a new transaction manager. - getOrchestrator(handler.HandlerContext) (Orchestrator, error) - GetTSM(handlerCtx handler.HandlerContext) (tsm.TSM, error) -} - -type standardProvider struct { - ctx txn_context.ITransactionCoordinatorContext -} - -func (sp *standardProvider) getOrchestrator(handlerCtx handler.HandlerContext) (Orchestrator, error) { - tsmInstance, walError := GetTSM(handlerCtx) - if walError != nil { - return nil, walError - } - txnCoordinator := newTxnCoordinator(tsmInstance, handlerCtx, sp.ctx) - orc, err := newTxnOrchestrator(tsmInstance, handlerCtx, txnCoordinator) - return orc, err -} - -func (sp *standardProvider) GetTSM(handlerCtx handler.HandlerContext) (tsm.TSM, error) { - return GetTSM(handlerCtx) -} - -func newTxnCoordinator(tsmInstance tsm.TSM, handlerCtx handler.HandlerContext, - ctx txn_context.ITransactionCoordinatorContext) Coordinator { - maxTxnDepth := defaultMaxStackDepth - if ctx != nil { - maxTxnDepth = ctx.GetMaxStackDepth() - } - return newCoordinator(tsmInstance, handlerCtx, maxTxnDepth) -} - -func GetProviderInstance(ctx txn_context.ITransactionCoordinatorContext) (Provider, error) { - var err error - providerOnce.Do(func() { - if err != nil { - return - } - providerSingleton = &standardProvider{ - ctx: ctx, - } - }) - return providerSingleton, err -} - -func NewOrchestrator(handlerCtx handler.HandlerContext) (Orchestrator, error) { - txnProvider, txnProviderErr := GetProviderInstance( - handlerCtx.GetTxnCoordinatorCtx()) - if txnProviderErr != nil { - return nil, txnProviderErr - } - return txnProvider.getOrchestrator(handlerCtx) -} diff --git a/internal/stackql/acid/txn_context/txn_context.go b/internal/stackql/acid/txn_context/txn_context.go deleted file mode 100644 index fc029788..00000000 --- a/internal/stackql/acid/txn_context/txn_context.go +++ /dev/null @@ -1,25 +0,0 @@ -package txn_context //nolint:stylecheck,revive // meaning of package name is clear - -var ( - _ ITransactionContext = &transactionContext{} -) - -type ITransactionContext interface { - GetStackDepth() int -} - -type transactionContext struct { - stackDepth int -} - -func NewTransactionContext( - stackDepth int, -) ITransactionContext { - return &transactionContext{ - stackDepth: stackDepth, - } -} - -func (tc *transactionContext) GetStackDepth() int { - return tc.stackDepth -} diff --git a/internal/stackql/acid/txn_context/txn_context_test.go b/internal/stackql/acid/txn_context/txn_context_test.go deleted file mode 100644 index 464a1487..00000000 --- a/internal/stackql/acid/txn_context/txn_context_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package txn_context_test - -import ( - "testing" - - . "github.com/stackql/stackql/internal/stackql/acid/txn_context" -) - -func TestNewTransactionContext(t *testing.T) { - expectedDepth := 1 - tc := NewTransactionContext(expectedDepth) - if tc.GetStackDepth() != expectedDepth { - t.Fatal("test failed, mismatch stack depth for txn coordinator context") - } -} diff --git a/internal/stackql/acid/txn_context/txn_coordinator_context.go b/internal/stackql/acid/txn_context/txn_coordinator_context.go deleted file mode 100644 index a848924e..00000000 --- a/internal/stackql/acid/txn_context/txn_coordinator_context.go +++ /dev/null @@ -1,25 +0,0 @@ -package txn_context //nolint:revive,stylecheck // meaning of package name is clear - -var ( - _ ITransactionCoordinatorContext = &transactionCoordinatorContext{} -) - -type ITransactionCoordinatorContext interface { - GetMaxStackDepth() int -} - -type transactionCoordinatorContext struct { - maxStackDepth int -} - -func NewTransactionCoordinatorContext( - maxStackDepth int, -) ITransactionCoordinatorContext { - return &transactionCoordinatorContext{ - maxStackDepth: maxStackDepth, - } -} - -func (tc *transactionCoordinatorContext) GetMaxStackDepth() int { - return tc.maxStackDepth -} diff --git a/internal/stackql/acid/txn_context/txn_coordinator_context_test.go b/internal/stackql/acid/txn_context/txn_coordinator_context_test.go deleted file mode 100644 index 0e3d89ed..00000000 --- a/internal/stackql/acid/txn_context/txn_coordinator_context_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package txn_context_test - -import ( - "testing" - - . "github.com/stackql/stackql/internal/stackql/acid/txn_context" -) - -func TestNewTransactionCoordinatorContext(t *testing.T) { - expectedMaxDepth := 1 - tcc := NewTransactionCoordinatorContext(expectedMaxDepth) - if tcc.GetMaxStackDepth() != expectedMaxDepth { - t.Fatal("test failed, mismatch max stack depth for txn context") - } -} diff --git a/internal/stackql/bundle/bundle.go b/internal/stackql/bundle/bundle.go index b6aa8962..1b7fe64d 100644 --- a/internal/stackql/bundle/bundle.go +++ b/internal/stackql/bundle/bundle.go @@ -4,7 +4,6 @@ import ( "io" "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" "github.com/stackql/stackql/internal/stackql/dbmsinternal" "github.com/stackql/stackql/internal/stackql/garbagecollector" @@ -28,7 +27,6 @@ type Bundle interface { GetSQLEngine() sqlengine.SQLEngine GetTxnCounterManager() txncounter.Manager GetTxnStore() kstore.KStore - GetTxnCoordinatorContext() txn_context.ITransactionCoordinatorContext GetTypingConfig() typing.Config GetSessionContext() dto.SessionContext GetStdOut() (io.Writer, bool) @@ -47,43 +45,40 @@ func NewBundle( txnCtrMgr txncounter.Manager, authContexts map[string]*dto.AuthCtx, sqlDataSources map[string]sql_datasource.SQLDataSource, - txnCoordintatorContext txn_context.ITransactionCoordinatorContext, typCfg typing.Config, sessionCtx dto.SessionContext, ) Bundle { return &simpleBundle{ - garbageCollector: garbageCollector, - namespaces: namespaces, - sqlEngine: sqlEngine, - sqlSystem: sqlSystem, - txnStore: txnStore, - txnCtrMgr: txnCtrMgr, - formatter: sqlSystem.GetASTFormatter(), - pgInternalRouter: pgInternalRouter, - authContexts: authContexts, - sqlDataSources: sqlDataSources, - txnCoordintatorContext: txnCoordintatorContext, - typCfg: typCfg, - sessionCtx: sessionCtx, + garbageCollector: garbageCollector, + namespaces: namespaces, + sqlEngine: sqlEngine, + sqlSystem: sqlSystem, + txnStore: txnStore, + txnCtrMgr: txnCtrMgr, + formatter: sqlSystem.GetASTFormatter(), + pgInternalRouter: pgInternalRouter, + authContexts: authContexts, + sqlDataSources: sqlDataSources, + typCfg: typCfg, + sessionCtx: sessionCtx, } } type simpleBundle struct { - garbageCollector garbagecollector.GarbageCollector - namespaces tablenamespace.Collection - sqlEngine sqlengine.SQLEngine - sqlSystem sql_system.SQLSystem - txnStore kstore.KStore - txnCtrMgr txncounter.Manager - typCfg typing.Config - formatter sqlparser.NodeFormatter - pgInternalRouter dbmsinternal.Router - sqlDataSources map[string]sql_datasource.SQLDataSource - authContexts map[string]*dto.AuthCtx - txnCoordintatorContext txn_context.ITransactionCoordinatorContext - sessionCtx dto.SessionContext - stdOut io.Writer - stdErr io.Writer + garbageCollector garbagecollector.GarbageCollector + namespaces tablenamespace.Collection + sqlEngine sqlengine.SQLEngine + sqlSystem sql_system.SQLSystem + txnStore kstore.KStore + txnCtrMgr txncounter.Manager + typCfg typing.Config + formatter sqlparser.NodeFormatter + pgInternalRouter dbmsinternal.Router + sqlDataSources map[string]sql_datasource.SQLDataSource + authContexts map[string]*dto.AuthCtx + sessionCtx dto.SessionContext + stdOut io.Writer + stdErr io.Writer } func (sb *simpleBundle) WithStdOut(w io.Writer) Bundle { @@ -108,10 +103,6 @@ func (sb *simpleBundle) GetSessionContext() dto.SessionContext { return sb.sessionCtx } -func (sb *simpleBundle) GetTxnCoordinatorContext() txn_context.ITransactionCoordinatorContext { - return sb.txnCoordintatorContext -} - func (sb *simpleBundle) GetSQLDataSources() map[string]sql_datasource.SQLDataSource { return sb.sqlDataSources } diff --git a/internal/stackql/cmd/exec.go b/internal/stackql/cmd/exec.go index fb1927bf..2cdbb58d 100644 --- a/internal/stackql/cmd/exec.go +++ b/internal/stackql/cmd/exec.go @@ -106,8 +106,10 @@ func (cr *commandRunnerImpl) RunCommand(handlerCtx handler.HandlerContext) { } queryErr := stackqlDriver.ProcessQueryWithError(handlerCtx.GetRawQuery()) var iacFailure interface{ IaCFailure() bool } + var transactionFailure interface{ TransactionFailure() bool } if queryErr != nil && (handlerCtx.GetRuntimeContext().AccessConfigFile != "" || - errors.As(queryErr, &iacFailure) || errors.Is(queryErr, handler.ErrSessionConfiguration)) { + errors.As(queryErr, &iacFailure) || errors.As(queryErr, &transactionFailure) || + errors.Is(queryErr, handler.ErrSessionConfiguration)) { os.Exit(1) // The response handler has already presented the error. } } diff --git a/internal/stackql/cmd/mcp.go b/internal/stackql/cmd/mcp.go index 8c9751d9..3e84047d 100644 --- a/internal/stackql/cmd/mcp.go +++ b/internal/stackql/cmd/mcp.go @@ -28,7 +28,6 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v2" - "github.com/stackql/stackql/internal/stackql/acid/tsm_physio" "github.com/stackql/stackql/internal/stackql/buildinfo" "github.com/stackql/stackql/internal/stackql/diagnostics" "github.com/stackql/stackql/internal/stackql/entryutil" @@ -38,6 +37,7 @@ import ( "github.com/stackql/stackql/internal/stackql/mcpbackend" dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" db_util "github.com/stackql/stackql/internal/stackql/sqlengine/connect" + "github.com/stackql/stackql/internal/stackql/transaction" "github.com/stackql/stackql/pkg/mcp_server" _ "github.com/jackc/pgx/v5" //nolint:revive // canonical driver pattern @@ -206,7 +206,7 @@ func runMCPServer(handlerCtx handler.HandlerContext, config *mcp_server.Config) ) iqlerror.PrintErrorAndExitOneIfError(backendErr) } else { - orchestrator, orchestratorErr := tsm_physio.NewOrchestrator(handlerCtx) + orchestrator, orchestratorErr := transaction.NewOrchestrator(handlerCtx) iqlerror.PrintErrorAndExitOneIfError(orchestratorErr) iqlerror.PrintErrorAndExitOneIfNil(orchestrator, "orchestrator is unexpectedly nil") backend, backendErr = mcpbackend.NewStackqlMCPBackendService( diff --git a/internal/stackql/cmd/root.go b/internal/stackql/cmd/root.go index 5babb53b..10f176bf 100644 --- a/internal/stackql/cmd/root.go +++ b/internal/stackql/cmd/root.go @@ -199,7 +199,6 @@ func init() { rootCmd.PersistentFlags().StringVar(&runtimeCtx.StoreTxnCfgRaw, dto.StoreTxnCfgRawKey, "{}", "JSON / YAML string representing Txn store config") rootCmd.PersistentFlags().StringVar(&runtimeCtx.GCCfgRaw, dto.GCCfgRawKey, "{}", "JSON / YAML string representing GC config") rootCmd.PersistentFlags().StringVar(&runtimeCtx.UpstreamErrorCfgRaw, dto.UpstreamErrorCfgRawKey, "{}", `JSON / YAML string representing upstream errors that reads ignore, eg {"http": {"ignoreStatusCodes": ["5xx"]}}`) - rootCmd.PersistentFlags().StringVar(&runtimeCtx.ACIDCfgRaw, dto.ACIDCfgRawKey, "{}", "JSON / YAML string representing ACID config") rootCmd.PersistentFlags().StringVar(&previewCfgRaw, intrinsic.CfgRawKey, "{}", "JSON string configuring the "+intrinsic.ProviderName+ " provider backend; keys: batchSize, flushInterval, endpoint, unstable") rootCmd.PersistentFlags().StringVar(&otelConfigRaw, output.OTelConfigKey, "{}", "JSON / YAML string configuring the otel output format, "+ diff --git a/internal/stackql/driver/driver.go b/internal/stackql/driver/driver.go index 6a19215e..b24e05d9 100644 --- a/internal/stackql/driver/driver.go +++ b/internal/stackql/driver/driver.go @@ -10,7 +10,6 @@ import ( "github.com/stackql-labs/omnisdk/public/authentication" "github.com/stackql/psql-wire/pkg/sqldata" "github.com/stackql/stackql/internal/stackql/accessmanagement" - "github.com/stackql/stackql/internal/stackql/acid/tsm_physio" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/logging" @@ -18,6 +17,7 @@ import ( "github.com/stackql/stackql/internal/stackql/queryshape" "github.com/stackql/stackql/internal/stackql/responsehandler" "github.com/stackql/stackql/internal/stackql/sqlengine" + "github.com/stackql/stackql/internal/stackql/transaction" "github.com/stackql/stackql/internal/stackql/util" "github.com/stackql/stackql/pkg/txncounter" @@ -54,15 +54,10 @@ func (sdf *basicStackQLDriverFactory) newSQLDriver() (StackQLDriver, error) { return nil, err } clonedCtx := sdf.handlerCtx.NewClientContext() - txnOrchestrator, orcErr := tsm_physio.NewOrchestrator(clonedCtx) + txnOrchestrator, orcErr := transaction.NewOrchestrator(clonedCtx) if orcErr != nil { return nil, orcErr } - tsmInstance, walError := tsm_physio.GetTSM(sdf.handlerCtx) - if walError != nil { - return nil, walError - } - sdf.handlerCtx.SetTSM(tsmInstance) clonedCtx.SetTxnCounterMgr(txCtr) buf := bytes.NewBuffer([]byte{}) if sdf.isCaptureDebug { @@ -157,7 +152,7 @@ type portalMeta struct { type basicStackQLDriver struct { debugBuf *bytes.Buffer handlerCtx handler.HandlerContext - txnOrchestrator tsm_physio.Orchestrator + txnOrchestrator transaction.Orchestrator shapeInferrer queryshape.Inferrer paramDecoder paramdecoder.Decoder stmtCache map[string]*stmtMeta @@ -254,15 +249,10 @@ func (dr *basicStackQLDriver) SplitCompoundQuery(s string) ([]string, error) { } func NewStackQLDriver(handlerCtx handler.HandlerContext) (StackQLDriver, error) { - txnOrchestrator, orcErr := tsm_physio.NewOrchestrator(handlerCtx) + txnOrchestrator, orcErr := transaction.NewOrchestrator(handlerCtx) if orcErr != nil { return nil, orcErr } - tsmInstance, walError := tsm_physio.GetTSM(handlerCtx) - if walError != nil { - return nil, walError - } - handlerCtx.SetTSM(tsmInstance) return &basicStackQLDriver{ handlerCtx: handlerCtx, txnOrchestrator: txnOrchestrator, diff --git a/internal/stackql/entryutil/entryutil.go b/internal/stackql/entryutil/entryutil.go index 65098655..3150a56f 100644 --- a/internal/stackql/entryutil/entryutil.go +++ b/internal/stackql/entryutil/entryutil.go @@ -7,7 +7,6 @@ import ( "path/filepath" "strings" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/bundle" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" "github.com/stackql/stackql/internal/stackql/dbmsinternal" @@ -92,15 +91,10 @@ func BuildInputBundle(runtimeCtx dto.RuntimeCtx) (bundle.Bundle, error) { if err != nil { return nil, fmt.Errorf("error initializing SQL data sources: %w", err) } - txnCoordinatorCfg, err := dto.GetTxnCoordinatorCfgCfg(runtimeCtx.ACIDCfgRaw) - if err != nil { - return nil, fmt.Errorf("error initializing Transaction Coordinator config: %w", err) - } sessionConfig, sessionConfigErr := dto.NewSessionContext(runtimeCtx.SessionCtxRaw) if sessionConfigErr != nil { return nil, fmt.Errorf("error initializing session config: %w", sessionConfigErr) } - txnCoordinatorCtx := txn_context.NewTransactionCoordinatorContext(txnCoordinatorCfg.GetMaxTxnDepth()) return bundle.NewBundle( gc, namespaces, @@ -111,7 +105,6 @@ func BuildInputBundle(runtimeCtx dto.RuntimeCtx) (bundle.Bundle, error) { txnCtrMgr, ac, sqlDataSources, - txnCoordinatorCtx, typCfg, sessionConfig, ), nil diff --git a/internal/stackql/handler/handler.go b/internal/stackql/handler/handler.go index 1137997c..aa8f558e 100644 --- a/internal/stackql/handler/handler.go +++ b/internal/stackql/handler/handler.go @@ -3,7 +3,6 @@ package handler import ( "context" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -17,8 +16,6 @@ import ( "github.com/stackql/stackql/internal/stackql/accessmanagement" "github.com/stackql-labs/omnisdk/pkg/omnisdk" - "github.com/stackql/stackql/internal/stackql/acid/tsm" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/bundle" "github.com/stackql/stackql/internal/stackql/constants" "github.com/stackql/stackql/internal/stackql/datasource/sql_datasource" @@ -33,6 +30,7 @@ import ( "github.com/stackql/stackql/internal/stackql/provider" "github.com/stackql/stackql/internal/stackql/registry" dto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql/internal/stackql/sessionconfig" "github.com/stackql/stackql/internal/stackql/sql_system" "github.com/stackql/stackql/internal/stackql/sqlengine" "github.com/stackql/stackql/internal/stackql/tablenamespace" @@ -46,7 +44,7 @@ import ( ) // ErrSessionConfiguration identifies a rejected session mutation at CLI boundaries. -var ErrSessionConfiguration = errors.New("session configuration failed") +var ErrSessionConfiguration = sessionconfig.ErrSessionConfiguration var ( _ HandlerContext = &standardHandlerContext{} @@ -103,7 +101,6 @@ type HandlerContext interface { //nolint:revive // don't mind stuttering this on // SetOutErrFile(io.Writer) // - GetTxnCoordinatorCtx() txn_context.ITransactionCoordinatorContext GetTypingConfig() typing.Config GetIsolationLevel() constants.IsolationLevel @@ -112,9 +109,6 @@ type HandlerContext interface { //nolint:revive // don't mind stuttering this on GetRollbackType() constants.RollbackType UpdateRollbackType(rollbackTypeStr string) error - GetTSM() (tsm.TSM, bool) - SetTSM(tsm.TSM) - SetExportNamespace(string) GetExportNamespace() string @@ -169,10 +163,8 @@ type standardHandlerContext struct { namespaceCollection tablenamespace.Collection formatter sqlparser.NodeFormatter pgInternalRouter dbmsinternal.Router - txnCoordinatorCtx txn_context.ITransactionCoordinatorContext typCfg typing.Config sessionContext dto.SessionContext - walInstance tsm.TSM exportNamespace string stackqlSemver string defaultHTTPClient *http.Client @@ -212,18 +204,6 @@ func (hc *standardHandlerContext) GetExportNamespace() string { return hc.exportNamespace } -func (hc *standardHandlerContext) GetTSM() (tsm.TSM, bool) { - return hc.walInstance, hc.walInstance != nil -} - -func (hc *standardHandlerContext) SetTSM(w tsm.TSM) { - hc.walInstance = w -} - -func (hc *standardHandlerContext) GetTxnCoordinatorCtx() txn_context.ITransactionCoordinatorContext { - return hc.txnCoordinatorCtx -} - func (hc *standardHandlerContext) GetTypingConfig() typing.Config { return hc.typCfg } @@ -702,7 +682,6 @@ func (hc *standardHandlerContext) Clone() HandlerContext { namespaceCollection: hc.namespaceCollection, formatter: hc.formatter, pgInternalRouter: hc.pgInternalRouter, - txnCoordinatorCtx: hc.txnCoordinatorCtx, typCfg: hc.typCfg, sessionContext: hc.sessionContext, exportNamespace: hc.exportNamespace, @@ -784,7 +763,6 @@ func NewHandlerCtx( formatter: inputBundle.GetSQLSystem().GetASTFormatter(), pgInternalRouter: inputBundle.GetDBMSInternalRouter(), currentProvider: runtimeCtx.ProviderStr, - txnCoordinatorCtx: inputBundle.GetTxnCoordinatorContext(), typCfg: inputBundle.GetTypingConfig(), sessionContext: inputBundle.GetSessionContext(), accessManager: accessManager, diff --git a/internal/stackql/handler/provider_session.go b/internal/stackql/handler/provider_session.go index af77773b..92cddca9 100644 --- a/internal/stackql/handler/provider_session.go +++ b/internal/stackql/handler/provider_session.go @@ -19,13 +19,17 @@ import ( ) type providerSessionState struct { - variables map[string]any - base omnisdk.Args - mu sync.Mutex - client omnisdk.ClientSession - store secrets.Store - bindings map[string]omnisdk.ProviderBinding - documentView string + transactionStarting bool + transactionConfig *ProviderTransactionConfig + transactionSession *omnisdk.TransactionSession + transactionHandle omnisdk.TransactionHandle + variables map[string]any + base omnisdk.Args + mu sync.Mutex + client omnisdk.ClientSession + store secrets.Store + bindings map[string]omnisdk.ProviderBinding + documentView string } func decodeSessionObject(value interface{}, target interface{}) error { @@ -144,6 +148,8 @@ func (hc *standardHandlerContext) setProviderSessionConfig(path string, rhs inte return false, nil } switch parts[0] { + case "transaction": + return true, hc.configureProviderTransaction(parts, rhs, scope) case "providers", "secret_store", "secrets": default: return false, nil diff --git a/internal/stackql/handler/provider_transaction.go b/internal/stackql/handler/provider_transaction.go new file mode 100644 index 00000000..2b118db6 --- /dev/null +++ b/internal/stackql/handler/provider_transaction.go @@ -0,0 +1,112 @@ +package handler + +import ( + "context" + "fmt" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql/internal/stackql/constants" + "github.com/stackql/stackql/internal/stackql/intrinsic" +) + +// ProviderTransactionConfig is host addressing/configuration, not desired state. +// Transaction lifecycle and resource ownership are implemented by Omni. +type ProviderTransactionConfig struct { + omnisdk.TransactionConfig + ResumeID string `json:"resume_id,omitempty"` +} + +func (hc *standardHandlerContext) configureProviderTransaction(parts []string, rhs interface{}, scope string) error { + if scope != "" && scope != "session" { + return fmt.Errorf("transaction configuration requires session scope") + } + state := hc.providerSessionState() + state.mu.Lock() + defer state.mu.Unlock() + if state.transactionSession != nil || state.transactionStarting { + return fmt.Errorf("transaction configuration is frozen until COMMIT or ROLLBACK") + } + if len(parts) != 1 { + return fmt.Errorf("set the complete transaction configuration object") + } + if rhs == nil { + state.transactionConfig = nil + return nil + } + var cfg ProviderTransactionConfig + if err := decodeSessionObject(rhs, &cfg); err != nil { + return err + } + if _, err := intrinsic.ManagedStateDirectory(hc, cfg.State); err != nil { + return err + } + if cfg.Collection == "" { + return fmt.Errorf("transaction configuration requires collection") + } + state.transactionConfig = &cfg + return nil +} +func (hc *standardHandlerContext) ProviderTransaction() (*omnisdk.TransactionSession, omnisdk.TransactionHandle, bool) { + state := hc.providerSessionState() + state.mu.Lock() + defer state.mu.Unlock() + return state.transactionSession, state.transactionHandle, state.transactionSession != nil +} +func (hc *standardHandlerContext) BeginProviderTransaction(ctx context.Context) error { + state := hc.providerSessionState() + state.mu.Lock() + if state.transactionSession != nil || state.transactionStarting { + state.mu.Unlock() + return fmt.Errorf("nested provider transactions are unsupported") + } + cfg := ProviderTransactionConfig{TransactionConfig: omnisdk.TransactionConfig{ + Collection: "sql", DynamicScope: true, + }} + if state.transactionConfig != nil { + cfg = *state.transactionConfig + } + state.transactionStarting = true + state.mu.Unlock() + defer func() { state.mu.Lock(); state.transactionStarting = false; state.mu.Unlock() }() + session, err := omnisdk.NewTransactionSession(hc.GetCaller()) + if err != nil { + return err + } + cfg.State, err = intrinsic.ManagedStateDirectory(hc, cfg.State) + if err != nil { + session.Close() + return err + } + if cfg.Mode == "" { + cfg.Mode = omnisdk.TransactionEager + if hc.GetRollbackType() == constants.NopRollback { + cfg.Mode = omnisdk.TransactionLazy + } + } + var handle omnisdk.TransactionHandle + if cfg.ResumeID != "" { + handle, err = session.Resume(ctx, cfg.TransactionConfig, cfg.ResumeID) + } else { + handle, err = intrinsic.OpenProviderTransaction(hc, session, cfg.TransactionConfig) + } + if err != nil { + session.Close() + return err + } + state.mu.Lock() + state.transactionSession = session + state.transactionHandle = handle + state.mu.Unlock() + return nil +} + +func (hc *standardHandlerContext) ClearProviderTransaction() { + state := hc.providerSessionState() + state.mu.Lock() + defer state.mu.Unlock() + if state.transactionSession != nil { + state.transactionSession.Close() + } + state.transactionSession = nil + state.transactionHandle = omnisdk.TransactionHandle{} +} diff --git a/internal/stackql/intrinsic/doc.go b/internal/stackql/intrinsic/doc.go index a54a5397..e17a40b1 100644 --- a/internal/stackql/intrinsic/doc.go +++ b/internal/stackql/intrinsic/doc.go @@ -371,11 +371,14 @@ func planDocQuery(ctx queryContext, translated docQuery) (omnisdk.Plan, string, if completionErr := configureMutationCompletion(ctx, translated, &args); completionErr != nil { return nil, "", completionErr } - configureMutationObserver(ctx, q.Target() != nil, &args) args.Relations = relations args.Tuning.Limit = translated.getLimit() configureReadHints(translated, &args) configureLegacyReadCache(ctx, q, &args) + transactionPlan, active, transactionErr := planProviderTransaction(ctx, translated, registry, res.Graph(), args) + if active { + return transactionPlan, relation, transactionErr + } plan, planErr := omnisdk.NewGraphQuery(registry, res.Graph(), args) if planErr != nil { return nil, "", planErr diff --git a/internal/stackql/intrinsic/iac.go b/internal/stackql/intrinsic/iac.go index 09d5fafb..ea674a81 100644 --- a/internal/stackql/intrinsic/iac.go +++ b/internal/stackql/intrinsic/iac.go @@ -163,21 +163,9 @@ func ledgerPredicates( "and the correlation stamp, and it is how a later run addresses the same resources", relation, collectionPredicate)) } - root := previewCfg.getIaCStateRoot() - if root == "" { - root = filepath.Join(ctx.GetRuntimeContext().ApplicationFilesRootPath, stateDirName) - } - if ctx.GetRuntimeContext().AccessConfigFile != "" { - host, ok := ctx.(interface{ GetCaller() authorization.Session }) - if !ok || host.GetCaller() == nil { - return nil, nil, refuse(errors.New("authenticated caller required for IaC state")) - } - principal := host.GetCaller().Principal() - owner := "principal:" + principal.ID() - if tenant := principal.Attributes()["tenant"]; tenant != "" { - owner = "tenant:" + tenant - } - root = filepath.Join(root, fmt.Sprintf("%x", sha256.Sum256([]byte(owner)))) + root, rootErr := ManagedStateRoot(ctx) + if rootErr != nil { + return nil, nil, refuse(rootErr) } state, stateErr := resolveStateDir(root, popPredicate(params, statePredicate)) if stateErr != nil { @@ -474,3 +462,32 @@ func (e iacOutcomeError) Error() string { func (iacOutcomeError) IaCFailure() bool { return true } func (e iacOutcomeError) Unwrap() error { return e.cause } + +// ManagedStateRoot is shared by IaC and provider SQL transaction coordination. +func ManagedStateRoot(ctx queryContext) (string, error) { + root := previewCfg.getIaCStateRoot() + if root == "" { + root = filepath.Join(ctx.GetRuntimeContext().ApplicationFilesRootPath, stateDirName) + } + if ctx.GetRuntimeContext().AccessConfigFile != "" { + host, ok := ctx.(interface{ GetCaller() authorization.Session }) + if !ok || host.GetCaller() == nil { + return "", errors.New("authenticated caller required for IaC state") + } + principal := host.GetCaller().Principal() + owner := "principal:" + principal.ID() + if tenant := principal.Attributes()["tenant"]; tenant != "" { + owner = "tenant:" + tenant + } + root = filepath.Join(root, fmt.Sprintf("%x", sha256.Sum256([]byte(owner)))) + } + return root, nil +} + +func ManagedStateDirectory(ctx queryContext, name string) (string, error) { + root, err := ManagedStateRoot(ctx) + if err != nil { + return "", err + } + return resolveStateDir(root, name) +} diff --git a/internal/stackql/intrinsic/intrinsic.go b/internal/stackql/intrinsic/intrinsic.go index 9085965e..4ed7cfda 100644 --- a/internal/stackql/intrinsic/intrinsic.go +++ b/internal/stackql/intrinsic/intrinsic.go @@ -60,10 +60,16 @@ func GeneratePrimitiveFunc( ) (func() internaldto.ExecutorOutput, bool) { current := ctx.GetCurrentProvider() switch node := stmt.(type) { + case *sqlparser.Select: + if action, handled := providerTransactionControlFunc(ctx, node); handled { + return action, true + } case *sqlparser.Auth: return authFunc(ctx, node), true case *sqlparser.AuthRevoke: return authRevokeFunc(ctx, node), true + case *sqlparser.Set: + return sessionSetFunc(ctx, node) case *sqlparser.Sleep: return sleepFunc(ctx, node), true case *sqlparser.Use: diff --git a/internal/stackql/intrinsic/provider_transaction.go b/internal/stackql/intrinsic/provider_transaction.go new file mode 100644 index 00000000..a1b61269 --- /dev/null +++ b/internal/stackql/intrinsic/provider_transaction.go @@ -0,0 +1,222 @@ +package intrinsic + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/util" +) + +// ProviderTransactionMutation prevents a transaction from silently escaping to +// backend/local-table mutation paths that have no Omni inverse contract. +func ProviderTransactionMutation(ctx queryContext, stmt sqlparser.Statement) error { + if _, create := stmt.(*sqlparser.Insert); !create { + return fmt.Errorf("transaction has no supported inverse for this mutation") + } + tables, mutation := mutationTables(stmt) + if !mutation { + return fmt.Errorf("statement is not a supported provider mutation") + } + doc, err := fromDocProviders(tables, ctx.GetCurrentProvider()) + if err != nil { + return err + } + if !doc { + return fmt.Errorf("transaction mutation must use an Omni provider document") + } + return nil +} + +func RollbackProviderTransaction( + ctx queryContext, session *omnisdk.TransactionSession, handle omnisdk.TransactionHandle, +) error { + cfg, err := session.Configuration(executionContext(ctx), handle) + if err != nil { + return err + } + args, err := providerTransactionArgs(ctx, cfg) + if err != nil { + return err + } + return session.Rollback(executionContext(ctx), handle, registryRoot(ctx), args) +} + +// ProviderTransactionControlQuery identifies only the fixed, authorized control +// relations. They remain usable when ordinary execution is blocked for recovery. +func ProviderTransactionControlQuery(stmt sqlparser.Statement) bool { + node, ok := stmt.(*sqlparser.Select) + return ok && providerTransactionRelation(node) != "" +} +func providerTransactionRelation(node *sqlparser.Select) string { + if len(node.From) != 1 { + return "" + } + aliased, ok := node.From[0].(*sqlparser.AliasedTableExpr) + if !ok { + return "" + } + table, ok := aliased.Expr.(sqlparser.TableName) + if !ok { + return "" + } + if table.QualifierSecond.GetRawVal() != "stackql_preview" || table.Qualifier.GetRawVal() != "txn" { + return "" + } + switch table.Name.GetRawVal() { + case "status": + return "status" + case "recover": + return "recover" + } + return "" +} +func providerTransactionControlFunc( + ctx queryContext, node *sqlparser.Select, +) (func() internaldto.ExecutorOutput, bool) { + relation := providerTransactionRelation(node) + if relation == "" { + return nil, false + } + host, ok := ctx.(interface { + ProviderTransaction() (*omnisdk.TransactionSession, omnisdk.TransactionHandle, bool) + }) + if !ok { + return refuse(fmt.Errorf("provider transaction session is unavailable")), true + } + return func() internaldto.ExecutorOutput { + session, handle, active := host.ProviderTransaction() + if !active { + return internaldto.NewErroneousExecutorOutput(fmt.Errorf("no active provider transaction; BEGIN or resume first")) + } + if controlErr := applyTransactionControl(ctx, node, relation, session, handle); controlErr != nil { + return internaldto.NewErroneousExecutorOutput(controlErr) + } + snapshot, err := session.Status(executionContext(ctx), handle) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + pending, err := json.Marshal(snapshot.Pending) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + return prepare(ctx, []string{"id", "status", "version", "pending"}, + map[string]map[string]interface{}{"0": { + "id": snapshot.ID, "status": snapshot.Status, + "version": snapshot.Version, "pending": string(pending), + }}, util.DefaultRowSort) + }, true +} + +func providerTransactionArgs(ctx queryContext, cfg omnisdk.TransactionConfig) (omnisdk.Args, error) { + bundles := []string{} + for address := range cfg.Targets { + bundle, _, _ := strings.Cut(address, ".") + bundles = append(bundles, bundle) + } + cloud := "" + if len(bundles) > 0 { + cloud = bundles[0] + } + args, err := previewArgs(ctx, cloud, nil) + if err != nil { + return omnisdk.Args{}, err + } + args.AuthByProvider, err = queryProviderAuth(ctx, bundles) + if err != nil { + return omnisdk.Args{}, err + } + return args, nil +} +func OpenProviderTransaction( + ctx queryContext, session *omnisdk.TransactionSession, cfg omnisdk.TransactionConfig, +) (omnisdk.TransactionHandle, error) { + args, err := providerTransactionArgs(ctx, cfg) + if err != nil { + return omnisdk.TransactionHandle{}, err + } + return session.BeginWithRegistry(executionContext(ctx), cfg, registryRoot(ctx), args) +} + +func applyTransactionControl(ctx queryContext, node *sqlparser.Select, relation string, + session *omnisdk.TransactionSession, handle omnisdk.TransactionHandle, +) error { + if relation != "recover" { + if node.Where != nil { + return fmt.Errorf("transaction status takes no predicates") + } + return nil + } + recovery, err := transactionRecoveryInputs(node) + if err != nil { + return err + } + return session.Recover(executionContext(ctx), handle, recovery) +} + +func transactionRecoveryInputs(node *sqlparser.Select) (omnisdk.TransactionRecovery, error) { + values, unhandled := equalityPredicates(node.Where) + recovery := omnisdk.TransactionRecovery{Version: values["version"]} + if len(unhandled) > 0 { + return recovery, fmt.Errorf("transaction recovery requires equality predicates") + } + if recovery.Version == "" { + return recovery, fmt.Errorf("transaction recovery requires the observed transaction version") + } + for key, raw := range values { + switch key { + case "version": + case "resources": + if raw != "" { + if err := json.Unmarshal([]byte(raw), &recovery.Resources); err != nil { + return recovery, fmt.Errorf("invalid recovery resources") + } + } + case "continue": + if raw != "" { + value, err := strconv.ParseBool(raw) + if err != nil { + return recovery, fmt.Errorf("continue must be true or false") + } + recovery.Continue = value + } + default: + return recovery, fmt.Errorf("unknown transaction recovery input %s", key) + } + } + return recovery, nil +} + +func planProviderTransaction(ctx queryContext, translated docQuery, registry string, + graph omnisdk.Graph, args omnisdk.Args, +) (omnisdk.Plan, bool, error) { + host, ok := ctx.(interface { + ProviderTransaction() (*omnisdk.TransactionSession, omnisdk.TransactionHandle, bool) + }) + if !ok { + return nil, false, nil + } + session, handle, active := host.ProviderTransaction() + if !active { + return nil, false, nil + } + cfg, err := session.Configuration(executionContext(ctx), handle) + if err != nil { + return nil, true, err + } + var managed []omnisdk.Managed + if target := translated.getQuery().Target(); target != nil && len(cfg.Targets) > 0 { + address := target.Resource().Handle() + binding, found := cfg.Targets[address] + if !found { + return nil, true, fmt.Errorf("transaction target %s has no key/identity/inverse addressing declaration", address) + } + managed = append(managed, omnisdk.NewManaged(binding.Key, []string{target.Resource().Alias()}, + binding.Identity, binding.AddressedBy)) + } + return session.Plan(handle, registry, graph, managed, args), true, nil +} diff --git a/internal/stackql/intrinsic/session_set.go b/internal/stackql/intrinsic/session_set.go new file mode 100644 index 00000000..4a059660 --- /dev/null +++ b/internal/stackql/intrinsic/session_set.go @@ -0,0 +1,29 @@ +package intrinsic + +import ( + "fmt" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/sessionconfig" +) + +func sessionSetFunc(ctx queryContext, node *sqlparser.Set) (func() internaldto.ExecutorOutput, bool) { + host, ok := ctx.(interface { + SetConfigAtPath(string, interface{}, string) error + }) + if !ok { + return nil, false + } + return func() internaldto.ExecutorOutput { + if len(node.Exprs) == 0 { + return internaldto.NewErroneousExecutorOutput(fmt.Errorf("no set expressions found")) + } + for _, expression := range node.Exprs { + if err := sessionconfig.Apply(host, expression); err != nil { + return internaldto.NewErroneousExecutorOutput(fmt.Errorf("%w: %w", sessionconfig.ErrSessionConfiguration, err)) + } + } + return internaldto.NewNopEmptyExecutorOutput(nil) + }, true +} diff --git a/internal/stackql/intrinsic/transaction.go b/internal/stackql/intrinsic/transaction.go index e8343530..fda48a51 100644 --- a/internal/stackql/intrinsic/transaction.go +++ b/internal/stackql/intrinsic/transaction.go @@ -1,196 +1,15 @@ package intrinsic import ( - "context" "errors" "fmt" "io" "net/http" "strings" - "sync" "github.com/stackql-labs/omnisdk/pkg/omnisdk" - "github.com/stackql-labs/omnisdk/public/authorization" - "github.com/stackql/stackql-parser/go/vt/sqlparser" - "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" ) -type completionChain struct{ first, second omnisdk.MutationCompletion } - -func (c completionChain) Complete( - ctx context.Context, address, verb string, inputs map[string]any, row omnisdk.Row, -) (omnisdk.Row, error) { - var err error - if c.first != nil { - row, err = c.first.Complete(ctx, address, verb, inputs, row) - if err != nil { - return nil, err - } - } - return c.second.Complete(ctx, address, verb, inputs, row) -} - -type observedQueryContext struct { - queryContext - observer *inverseCapture -} - -func (c observedQueryContext) mutationObserver() omnisdk.MutationCompletion { return c.observer } - -type inverseCapture struct { - mu sync.Mutex - rows []map[string]string - declaration omnisdk.DeclaredInverse -} - -func (c *inverseCapture) Complete( - _ context.Context, _ string, _ string, inputs map[string]any, row omnisdk.Row, -) (omnisdk.Row, error) { - params := map[string]string{} - for name, value := range inputs { - params[name] = fmt.Sprint(value) - } - for name, token := range c.declaration.Tokens() { - if token.Location != "body" || token.Algorithm != "toString" { - return nil, fmt.Errorf("unsupported inverse token %s: %s/%s", name, token.Location, token.Algorithm) - } - value, ok := row[token.Key] - if !ok || value == nil { - return nil, fmt.Errorf("inverse token %s missing response field %s", name, token.Key) - } - params[name] = fmt.Sprint(value) - } - c.mu.Lock() - c.rows = append(c.rows, params) - c.mu.Unlock() - return row, nil -} - -// GenerateTransactionStreamFunc attaches only document-declared reversals to the -// host transaction graph. Accepted results supply their inverse identity. -// -//nolint:gocognit // Planning handles optional inverse metadata before constructing independent executors. -func GenerateTransactionStreamFunc( - ctx queryContext, stmt sqlparser.Statement, -) (func() internaldto.ExecutorOutput, func() internaldto.ExecutorOutput, bool) { - if _, ok := stmt.(*sqlparser.Insert); !ok { - f, handled := GenerateStreamFunc(ctx, stmt) - return f, nil, handled - } - dialect, err := backendDialect(ctx) - if err != nil { - f, handled := GenerateStreamFunc(ctx, stmt) - return f, nil, handled - } - translated, err := newDocTranslator(ctx.GetCurrentProvider(), dialect).mutation(stmt) - if err != nil { - f, handled := GenerateStreamFunc(ctx, stmt) - return f, nil, handled - } - target := translated.getQuery().Target() - if target == nil { - f, handled := GenerateStreamFunc(ctx, stmt) - return f, nil, handled - } - address := target.Resource().Handle() - declaration, err := omnisdk.DocInverse(registryRoot(ctx), address, "insert") - if err != nil || declaration == nil { - f, handled := GenerateStreamFunc(ctx, stmt) - return f, nil, handled - } - capture := &inverseCapture{declaration: declaration} - forward, handled := GenerateStreamFunc(observedQueryContext{queryContext: ctx, observer: capture}, stmt) - if !handled { - return nil, nil, false - } - run := func() internaldto.ExecutorOutput { - result := forward() - if result.GetError() == nil { - message := "Undo the insert on " + strings.TrimPrefix(address, UnstablePrefix) - result.SetUndoLog(binlog.NewSimpleLogEntry(nil, []string{message})) - } - return result - } - inverse := func() internaldto.ExecutorOutput { return executeDeclaredInverse(ctx, address, declaration, capture) } - return run, inverse, true -} - -func configureMutationObserver(ctx queryContext, mutation bool, args *omnisdk.Args) { - if observed, ok := ctx.(interface { - mutationObserver() omnisdk.MutationCompletion - }); ok && mutation { - args.MutationCompletion = completionChain{first: args.MutationCompletion, second: observed.mutationObserver()} - } -} -func (c observedQueryContext) GetOutErrFile() io.Writer { - if host, ok := c.queryContext.(interface{ GetOutErrFile() io.Writer }); ok { - return host.GetOutErrFile() - } - return io.Discard -} - -func executeDeclaredInverse( - ctx queryContext, address string, declaration omnisdk.DeclaredInverse, capture *inverseCapture, -) internaldto.ExecutorOutput { - capture.mu.Lock() - rows := append([]map[string]string(nil), capture.rows...) - capture.mu.Unlock() - for i := len(rows) - 1; i >= 0; i-- { - params, err := inverseParameters(ctx, address, declaration.Method(), rows[i]) - if err != nil { - return internaldto.NewErroneousExecutorOutput(err) - } - node, e := omnisdk.NewInverseMutationNode("inverse", address, "delete", declaration.Method(), params) - if e != nil { - return internaldto.NewErroneousExecutorOutput(e) - } - graph, e := omnisdk.NewGraph([]omnisdk.Node{node}, nil) - if e != nil { - return internaldto.NewErroneousExecutorOutput(e) - } - args, e := previewArgs(ctx, address, params) - if e != nil { - return internaldto.NewErroneousExecutorOutput(e) - } - p, e := omnisdk.NewGraphQuery(registryRoot(ctx), graph, args) - if e != nil { - return internaldto.NewErroneousExecutorOutput(e) - } - cursor, e := p.Open(executionContext(ctx)) - if e != nil { - return internaldto.NewErroneousExecutorOutput(e) - } - output := drainMutation(cursor) - if e = output.GetError(); e != nil { - return internaldto.NewErroneousExecutorOutput(mutationHTTPFailure(ctx, "undo", e)) - } - } - return internaldto.NewExecutorOutput(nil, nil, nil, nil, nil) -} - -func inverseParameters( - ctx queryContext, address, method string, original map[string]string, -) (map[string]string, error) { - table, err := omnisdk.DescribeMutation(registryRoot(ctx), providerAddress(ctx, address), "delete") - if err != nil { - return nil, err - } - for _, signature := range table.Methods() { - if signature.Method() != method { - continue - } - params := map[string]string{} - for _, p := range signature.Params() { - if value, ok := original[p.Name()]; ok { - params[p.Name()] = value - } - } - return params, nil - } - return nil, fmt.Errorf("declared inverse method %s not found", method) -} - type mutationPresentationError struct { cause error message string @@ -218,13 +37,3 @@ func mutationHTTPFailure(ctx queryContext, verb string, err error) error { ) return mutationPresentationError{cause: err, message: message} } - -func (c observedQueryContext) GetCaller() authorization.Session { - if host, ok := c.queryContext.(interface{ GetCaller() authorization.Session }); ok { - return host.GetCaller() - } - return nil -} -func (c observedQueryContext) GetExecutionContext() context.Context { - return executionContext(c.queryContext) -} diff --git a/internal/stackql/intrinsic/transaction_test.go b/internal/stackql/intrinsic/transaction_test.go index 66db342b..aab839f8 100644 --- a/internal/stackql/intrinsic/transaction_test.go +++ b/internal/stackql/intrinsic/transaction_test.go @@ -1,38 +1,12 @@ package intrinsic //nolint:testpackage // Exercises private reversal binding and error contracts. import ( - "context" "errors" "testing" "github.com/stackql-labs/omnisdk/pkg/omnisdk" ) -type inverseFixture struct{} - -func (inverseFixture) Method() string { return "delete" } -func (inverseFixture) Tokens() map[string]omnisdk.InverseToken { - return map[string]omnisdk.InverseToken{ - "userKey": {Key: "id", Location: "body", Algorithm: "toString"}, - } -} -func TestInverseRequiresAcceptedResponseIdentity(t *testing.T) { - c := &inverseCapture{declaration: inverseFixture{}} - if _, err := c.Complete(context.Background(), "p.s.users", "insert", nil, omnisdk.Row{}); err == nil { - t.Fatal("missing identity accepted") - } - if len(c.rows) != 0 { - t.Fatal("registered reversal without identity") - } - row := omnisdk.Row{"id": "created-user"} - if _, err := c.Complete(context.Background(), "p.s.users", "insert", map[string]any{"region": "east"}, row); err != nil { - t.Fatal(err) - } - if len(c.rows) != 1 || c.rows[0]["userKey"] != "created-user" || c.rows[0]["region"] != "east" { - t.Fatalf("bindings=%v", c.rows) - } -} - type failedMutationError struct{} func (failedMutationError) Error() string { return "upstream failed" } diff --git a/internal/stackql/mcpbackend/mcp_service_stackql.go b/internal/stackql/mcpbackend/mcp_service_stackql.go index 1e798876..77baec15 100644 --- a/internal/stackql/mcpbackend/mcp_service_stackql.go +++ b/internal/stackql/mcpbackend/mcp_service_stackql.go @@ -11,11 +11,11 @@ import ( "github.com/sirupsen/logrus" "github.com/stackql/psql-wire/pkg/sqldata" - "github.com/stackql/stackql/internal/stackql/acid/tsm_physio" "github.com/stackql/stackql/internal/stackql/buildinfo" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" runtimedto "github.com/stackql/stackql/internal/stackql/runtimeconfig" + "github.com/stackql/stackql/internal/stackql/transaction" "github.com/stackql/stackql/pkg/mcp_server" "github.com/stackql/stackql/pkg/mcp_server/dto" @@ -272,7 +272,7 @@ func (s *simpleStackqlInterrogator) GetRegistryPull(provider, version string) (s } type stackqlMCPService struct { - txnOrchestrator tsm_physio.Orchestrator + txnOrchestrator transaction.Orchestrator interrogator StackqlInterrogator handlerCtx handler.HandlerContext logger *logrus.Logger @@ -283,7 +283,7 @@ type stackqlMCPService struct { } func NewStackqlMCPBackendService( - txnOrchestrator tsm_physio.Orchestrator, + txnOrchestrator transaction.Orchestrator, handlerCtx handler.HandlerContext, logger *logrus.Logger, serverInfo serverBuildInfo, diff --git a/internal/stackql/planbuilder/entrypoint.go b/internal/stackql/planbuilder/entrypoint.go index f40c6e22..07fe98f0 100644 --- a/internal/stackql/planbuilder/entrypoint.go +++ b/internal/stackql/planbuilder/entrypoint.go @@ -6,7 +6,6 @@ import ( "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/accessmanagement" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/astanalysis/earlyanalysis" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -27,20 +26,9 @@ type PlanBuilder interface { BuildPlanFromContext(handlerCtx handler.HandlerContext) (plan.Plan, error) } -func NewPlanBuilder(transactionContext txn_context.ITransactionContext) PlanBuilder { - return &standardPlanBuilder{ - transactionContext: transactionContext, - } -} +func NewPlanBuilder() PlanBuilder { return &standardPlanBuilder{} } -type standardPlanBuilder struct { - transactionContext txn_context.ITransactionContext -} - -//nolint:nilnil // TODO: sweep through tech debt -func (pb *standardPlanBuilder) BuildUndoPlanFromContext(_ handler.HandlerContext) (plan.Plan, error) { - return nil, nil -} +type standardPlanBuilder struct{} //nolint:funlen,gocognit,errcheck,gocyclo,cyclop // no big deal func (pb *standardPlanBuilder) BuildPlanFromContext(handlerCtx handler.HandlerContext) (plan.Plan, error) { @@ -124,12 +112,12 @@ func (pb *standardPlanBuilder) BuildPlanFromContext(handlerCtx handler.HandlerCo statement = stmt } - pGBuilder := newPlanGraphBuilder(handlerCtx.GetRuntimeContext().ExecutionConcurrencyLimit, pb.transactionContext) + pGBuilder := newPlanGraphBuilder(handlerCtx.GetRuntimeContext().ExecutionConcurrencyLimit) // An omnisdk data relation streams its rows straight to the output writer, // so it is not backed by a view and analysis would try, and fail, to resolve // it as a registry-backed relation. Plan it here, ahead of that analysis. - executor, inverse, handled := intrinsic.GenerateTransactionStreamFunc(handlerCtx, statement) + executor, handled := intrinsic.GenerateStreamFunc(handlerCtx, statement) if !handled { executor, handled = intrinsic.GeneratePrimitiveFunc(handlerCtx, statement) } @@ -151,11 +139,7 @@ func (pb *standardPlanBuilder) BuildPlanFromContext(handlerCtx handler.HandlerCo }, ), ) - if inverse != nil { - pGBuilder.getPlanGraphHolder().CreateInversePrimitiveNode(primitive.NewLocalPrimitive( - func(_ primitive.IPrimitiveCtx) internaldto.ExecutorOutput { return inverse() }, - )) - } + qPlan.SetInstructions(pGBuilder.getPlanGraphHolder()) if optimiseErr := qPlan.GetInstructions().GetPrimitiveGraph().Optimise(); optimiseErr != nil { return createErroneousPlan(handlerCtx, qPlan, rowSort, optimiseErr) diff --git a/internal/stackql/planbuilder/plan_builder.go b/internal/stackql/planbuilder/plan_builder.go index 84788240..0bd9e26d 100644 --- a/internal/stackql/planbuilder/plan_builder.go +++ b/internal/stackql/planbuilder/plan_builder.go @@ -1,13 +1,11 @@ package planbuilder import ( - "encoding/json" "fmt" "sort" "strconv" "strings" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/builder_input" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" @@ -23,6 +21,7 @@ import ( "github.com/stackql/stackql/internal/stackql/primitivegenerator" "github.com/stackql/stackql/internal/stackql/primitivegraph" "github.com/stackql/stackql/internal/stackql/registry" + "github.com/stackql/stackql/internal/stackql/sessionconfig" "github.com/stackql/stackql/internal/stackql/util" "golang.org/x/mod/semver" @@ -58,7 +57,6 @@ type planGraphBuilder interface { type standardPlanGraphBuilder struct { planGraphHolder primitivegraph.PrimitiveGraphHolder rootPrimitiveGenerator primitivegenerator.PrimitiveGenerator - transactionContext txn_context.ITransactionContext preBuiltIndirectCollection []primitivebuilder.Builder } @@ -83,10 +81,9 @@ func (pgb *standardPlanGraphBuilder) getPlanGraphHolder() primitivegraph.Primiti return pgb.planGraphHolder } -func newPlanGraphBuilder(concurrencyLimit int, transactionContext txn_context.ITransactionContext) planGraphBuilder { +func newPlanGraphBuilder(concurrencyLimit int) planGraphBuilder { return &standardPlanGraphBuilder{ - planGraphHolder: primitivegraph.NewPrimitiveGraphHolder(concurrencyLimit), - transactionContext: transactionContext, + planGraphHolder: primitivegraph.NewPrimitiveGraphHolder(concurrencyLimit), } } @@ -224,58 +221,8 @@ func (pgb *standardPlanGraphBuilder) handleExplain(pbi planbuilderinput.PlanBuil return buildErr } -//nolint:exhaustive // Unsupported literal kinds are explicitly rejected by the default branch. func setLogic(pbi planbuilderinput.PlanBuilderInput, setExpr *sqlparser.SetExpr) error { - lhsTrimmed := setExpr.Name.GetRawVal() - config := strings.HasPrefix(lhsTrimmed, "$") - if config { - lhsTrimmed = strings.TrimPrefix(strings.TrimPrefix(lhsTrimmed, "$"), ".") - root, _, _ := strings.Cut(lhsTrimmed, ".") - switch root { - case "auth", "providers", "secret_store", "secrets", "document_merge": - default: - return fmt.Errorf("unknown intrinsic session configuration %q", lhsTrimmed) - } - } else if !strings.HasPrefix(lhsTrimmed, "variables.") { - lhsTrimmed = "variables." + lhsTrimmed - } - if _, reset := setExpr.Expr.(*sqlparser.Default); reset { - return pbi.GetHandlerCtx().SetConfigAtPath(lhsTrimmed, nil, setExpr.Scope) - } - if !config { - var raw json.RawMessage - switch value := setExpr.Expr.(type) { - case *sqlparser.NullVal: - raw = json.RawMessage("null") - case sqlparser.BoolVal: - raw = json.RawMessage(fmt.Sprint(bool(value))) - case *sqlparser.SQLVal: - switch value.Type { - case sqlparser.StrVal: - raw, _ = json.Marshal(string(value.Val)) - case sqlparser.IntVal, sqlparser.FloatVal: - raw = json.RawMessage(value.Val) - default: - return fmt.Errorf("variables accepts scalar literals only") - } - default: - return fmt.Errorf("variables accepts scalar literals only") - } - return pbi.GetHandlerCtx().SetConfigAtPath(lhsTrimmed, raw, setExpr.Scope) - } - value, ok := setExpr.Expr.(*sqlparser.SQLVal) - if !ok || value.Type != sqlparser.StrVal { - return fmt.Errorf("session configuration requires a string literal or DEFAULT") - } - exprStr := string(value.Val) - if strings.HasPrefix(lhsTrimmed, "secrets.") { - return pbi.GetHandlerCtx().SetConfigAtPath(lhsTrimmed, exprStr, setExpr.Scope) - } - var exprObj map[string]interface{} - if err := json.Unmarshal([]byte(exprStr), &exprObj); err == nil && exprObj != nil { - return pbi.GetHandlerCtx().SetConfigAtPath(lhsTrimmed, exprObj, setExpr.Scope) - } - return pbi.GetHandlerCtx().SetConfigAtPath(lhsTrimmed, exprStr, setExpr.Scope) + return sessionconfig.Apply(pbi.GetHandlerCtx(), setExpr) } func (pgb *standardPlanGraphBuilder) handleSet(pbi planbuilderinput.PlanBuilderInput) error { diff --git a/internal/stackql/primitivegraph/graph_holder.go b/internal/stackql/primitivegraph/graph_holder.go index 1ea9f067..198e4a16 100644 --- a/internal/stackql/primitivegraph/graph_holder.go +++ b/internal/stackql/primitivegraph/graph_holder.go @@ -12,25 +12,15 @@ var ( //nolint:revive // acceptable nomenclature type PrimitiveGraphHolder interface { Blank() error - AddInverseTxnControlCounters(t internaldto.TxnControlCounters) AddTxnControlCounters(t internaldto.TxnControlCounters) ContainsIndirect() bool - CreateInversePrimitiveNode(pr primitive.IPrimitive) PrimitiveNode CreatePrimitiveNode(pr primitive.IPrimitive) PrimitiveNode - GetInversePrimitiveGraph() PrimitiveGraph - GetInverseTxnControlCounterSlice() []internaldto.TxnControlCounters GetPrimitiveGraph() PrimitiveGraph GetTxnControlCounterSlice() []internaldto.TxnControlCounters - InverseContainsIndirect() bool - NewInverseDependency(from PrimitiveNode, to PrimitiveNode, weight float64) NewDependency(from PrimitiveNode, to PrimitiveNode, weight float64) SetContainsIndirect(bool) - SetInverseContainsIndirect(bool) SetTxnID(int) - SetInverseTxnID(int) - SetInverseContainsUserManagedRelation(containsUserRelation bool) SetContainsUserManagedRelation(containsUserRelation bool) - InverseContainsUserManagedRelation() bool ContainsUserManagedRelation() bool SetExecutorOutput(id string, output internaldto.ExecutorOutput) error GetExecutorOutput(id string) (internaldto.ExecutorOutput, bool) @@ -40,7 +30,6 @@ type PrimitiveGraphHolder interface { type standardPrimitiveGraphHolder struct { concurrencyLimit int pg PrimitiveGraph - ipg PrimitiveGraph outputRegister internaldto.ExecutorOutputRegister } @@ -52,38 +41,18 @@ func (pgh *standardPrimitiveGraphHolder) SetTxnID(txnID int) { pgh.pg.SetTxnID(txnID) } -func (pgh *standardPrimitiveGraphHolder) SetInverseTxnID(txnID int) { - pgh.ipg.SetTxnID(txnID) -} - -func (pgh *standardPrimitiveGraphHolder) GetInversePrimitiveGraph() PrimitiveGraph { - return pgh.ipg -} - func (pgh *standardPrimitiveGraphHolder) AddTxnControlCounters(t internaldto.TxnControlCounters) { pgh.pg.AddTxnControlCounters(t) } -func (pgh *standardPrimitiveGraphHolder) AddInverseTxnControlCounters(t internaldto.TxnControlCounters) { - pgh.ipg.AddTxnControlCounters(t) -} - func (pgh *standardPrimitiveGraphHolder) CreatePrimitiveNode(pr primitive.IPrimitive) PrimitiveNode { return pgh.pg.CreatePrimitiveNode(pr) } -func (pgh *standardPrimitiveGraphHolder) CreateInversePrimitiveNode(pr primitive.IPrimitive) PrimitiveNode { - return pgh.ipg.CreatePrimitiveNode(pr) -} - func (pgh *standardPrimitiveGraphHolder) NewDependency(from PrimitiveNode, to PrimitiveNode, weight float64) { pgh.pg.NewDependency(from, to, weight) } -func (pgh *standardPrimitiveGraphHolder) NewInverseDependency(from PrimitiveNode, to PrimitiveNode, weight float64) { - pgh.ipg.NewDependency(to, from, weight) -} - func (pgh *standardPrimitiveGraphHolder) SetContainsIndirect(containsView bool) { pgh.pg.SetContainsIndirect(containsView) } @@ -92,38 +61,18 @@ func (pgh *standardPrimitiveGraphHolder) SetContainsUserManagedRelation(contains pgh.pg.SetContainsUserManagedRelation(containsUserRelation) } -func (pgh *standardPrimitiveGraphHolder) SetInverseContainsUserManagedRelation(containsUserRelation bool) { - pgh.ipg.SetContainsUserManagedRelation(containsUserRelation) -} - func (pgh *standardPrimitiveGraphHolder) ContainsIndirect() bool { return pgh.pg.ContainsIndirect() } -func (pgh *standardPrimitiveGraphHolder) InverseContainsIndirect() bool { - return pgh.ipg.ContainsIndirect() -} - func (pgh *standardPrimitiveGraphHolder) ContainsUserManagedRelation() bool { return pgh.pg.ContainsUserManagedRelation() } -func (pgh *standardPrimitiveGraphHolder) InverseContainsUserManagedRelation() bool { - return pgh.ipg.ContainsUserManagedRelation() -} - func (pgh *standardPrimitiveGraphHolder) GetTxnControlCounterSlice() []internaldto.TxnControlCounters { return pgh.pg.GetTxnControlCounterSlice() } -func (pgh *standardPrimitiveGraphHolder) GetInverseTxnControlCounterSlice() []internaldto.TxnControlCounters { - return pgh.ipg.GetTxnControlCounterSlice() -} - -func (pgh *standardPrimitiveGraphHolder) SetInverseContainsIndirect(containsView bool) { - pgh.pg.SetContainsIndirect(containsView) -} - func (pgh *standardPrimitiveGraphHolder) SetExecutorOutput(id string, output internaldto.ExecutorOutput) error { return pgh.outputRegister.SetExecutorOutput(id, output) } @@ -138,17 +87,14 @@ func (pgh *standardPrimitiveGraphHolder) GetRequest(requestType string) (interna func NewPrimitiveGraphHolder(concurrencyLimit int) PrimitiveGraphHolder { pg := newPrimitiveGraph(concurrencyLimit) - ipg := newSequentialPrimitiveGraph(concurrencyLimit) return &standardPrimitiveGraphHolder{ concurrencyLimit: concurrencyLimit, pg: pg, - ipg: ipg, outputRegister: internaldto.NewExecutorOutputRegister(), } } func (pgh *standardPrimitiveGraphHolder) Blank() error { pgh.pg = newPrimitiveGraph(pgh.concurrencyLimit) - pgh.ipg = newSequentialPrimitiveGraph(pgh.concurrencyLimit) return nil } diff --git a/internal/stackql/primitivegraph/sequential_dag.go b/internal/stackql/primitivegraph/sequential_dag.go deleted file mode 100644 index bdd1cce8..00000000 --- a/internal/stackql/primitivegraph/sequential_dag.go +++ /dev/null @@ -1,55 +0,0 @@ -package primitivegraph - -import ( - "sync" - - "github.com/stackql/stackql/internal/stackql/acid/operation" - "github.com/stackql/stackql/internal/stackql/primitive" -) - -var ( - _ PrimitiveGraph = (*sequentialPrimitiveGraph)(nil) - _ primitive.IPrimitive = (*sequentialPrimitiveGraph)(nil) -) - -type sequentialPrimitiveGraph struct { - BasePrimitiveGraph - mutex sync.Mutex - root PrimitiveNode - tail PrimitiveNode -} - -func (pg *sequentialPrimitiveGraph) CreatePrimitiveNode(pr primitive.IPrimitive) PrimitiveNode { - pg.mutex.Lock() - defer pg.mutex.Unlock() - nn := pg.NewNode() - node := &standardPrimitiveNode{ - op: operation.NewReversibleOperation(pr), - id: nn.ID(), - isDone: make(chan bool, 1), - } - var isRoot bool - if pg.Nodes().Len() == 0 { - pg.root = node - isRoot = true - } - existingTail := pg.tail - pg.tail = node - pg.AddNode(node) - if !isRoot { - pg.NewDependency(existingTail, node, 1.0) - } - return node -} - -func newSequentialPrimitiveGraph(concurrencyLimit int) PrimitiveGraph { - baseGraph := newBasePrimitiveGraph(concurrencyLimit) - return &sequentialPrimitiveGraph{ - BasePrimitiveGraph: baseGraph, - } -} - -func (pg *sequentialPrimitiveGraph) NewDependency(from PrimitiveNode, to PrimitiveNode, weight float64) { - e := pg.NewWeightedEdge(from, to, weight) - pg.SetWeightedEdge(e) -} diff --git a/internal/stackql/queryshape/queryshape.go b/internal/stackql/queryshape/queryshape.go index aa8856fc..0a96a816 100644 --- a/internal/stackql/queryshape/queryshape.go +++ b/internal/stackql/queryshape/queryshape.go @@ -92,7 +92,7 @@ func (si *standardInferrer) inferFromPlan(query string) []sqldata.ISQLColumn { clonedCtx := si.handlerCtx.Clone() clonedCtx.SetQuery(query) clonedCtx.SetRawQuery(query) - pb := planbuilder.NewPlanBuilder(nil) + pb := planbuilder.NewPlanBuilder() qPlan, err := pb.BuildPlanFromContext(clonedCtx) if err != nil || qPlan == nil { return nil diff --git a/internal/stackql/querysubmit/querysubmit.go b/internal/stackql/querysubmit/querysubmit.go index 657a30b9..060d01c2 100644 --- a/internal/stackql/querysubmit/querysubmit.go +++ b/internal/stackql/querysubmit/querysubmit.go @@ -7,7 +7,6 @@ import ( "github.com/stackql/stackql-parser/go/vt/sqlparser" "github.com/stackql/stackql/internal/stackql/accessmanagement" "github.com/stackql/stackql/internal/stackql/acid/binlog" - "github.com/stackql/stackql/internal/stackql/acid/txn_context" "github.com/stackql/stackql/internal/stackql/handler" "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" "github.com/stackql/stackql/internal/stackql/logging" @@ -24,7 +23,6 @@ type QuerySubmitter interface { GetStatement() (sqlparser.Statement, bool) PrepareQuery(handlerCtx handler.HandlerContext) error SubmitQuery() internaldto.ExecutorOutput - WithTransactionContext(transactionContext txn_context.ITransactionContext) QuerySubmitter IsReadOnly() bool // Get the redo log entry. GetRedoLog() (binlog.LogEntry, bool) @@ -38,12 +36,9 @@ func NewQuerySubmitter() QuerySubmitter { } type basicQuerySubmitter struct { - queryPlan plan.Plan - undoPlan plan.Plan - undoHandlerCtx handler.HandlerContext - handlerCtx handler.HandlerContext - transactionContext txn_context.ITransactionContext - cancel func() + queryPlan plan.Plan + handlerCtx handler.HandlerContext + cancel func() } func (qs *basicQuerySubmitter) GetRedoLog() (binlog.LogEntry, bool) { @@ -78,13 +73,6 @@ func (qs *basicQuerySubmitter) GetStatement() (sqlparser.Statement, bool) { return qs.queryPlan.GetStatement() } -func (qs *basicQuerySubmitter) WithTransactionContext( - transactionContext txn_context.ITransactionContext, -) QuerySubmitter { - qs.transactionContext = transactionContext - return qs -} - func (qs *basicQuerySubmitter) PrepareQuery(handlerCtx handler.HandlerContext) error { if manager := handlerCtx.GetAccessManager(); manager != nil { caller, runCtx, cancel, err := manager.Establish(handlerCtx.GetExecutionContext()) @@ -105,7 +93,7 @@ func (qs *basicQuerySubmitter) PrepareQuery(handlerCtx handler.HandlerContext) e } qs.handlerCtx = handlerCtx logging.GetLogger().Debugln("PrepareQuery() invoked...") - pb := planbuilder.NewPlanBuilder(qs.transactionContext) + pb := planbuilder.NewPlanBuilder() plan, err := pb.BuildPlanFromContext(handlerCtx) qs.queryPlan = plan if err != nil && qs.cancel != nil { @@ -143,25 +131,6 @@ func (qs *basicQuerySubmitter) SubmitQuery() internaldto.ExecutorOutput { return result } -func (qs *basicQuerySubmitter) PrepareUndoQuery(handlerCtx handler.HandlerContext) error { - qs.undoHandlerCtx = handlerCtx - logging.GetLogger().Debugln("PrepareQuery() invoked...") - pb := planbuilder.NewPlanBuilder(qs.transactionContext) - plan, err := pb.BuildPlanFromContext(handlerCtx) - qs.undoPlan = plan - return err -} - -func (qs *basicQuerySubmitter) UndoQuery() internaldto.ExecutorOutput { - logging.GetLogger().Debugln("UndoQuery() invoked...") - pl := internaldto.NewBasicPrimitiveContext( - nil, - qs.handlerCtx.GetOutfile(), - qs.handlerCtx.GetOutErrFile(), - ) - return qs.queryPlan.GetInstructions().GetInversePrimitiveGraph().Execute(pl) -} - // Read-only snapshots live through streaming, and release their deadline timer at EOF/close. type callerStream struct { sqldata.ISQLResultStream diff --git a/internal/stackql/runtimeconfig/dto.go b/internal/stackql/runtimeconfig/dto.go index 52e040c4..6b0b1324 100644 --- a/internal/stackql/runtimeconfig/dto.go +++ b/internal/stackql/runtimeconfig/dto.go @@ -58,7 +58,6 @@ const ( SessionCtxKey string = "session" GCCfgRawKey string = "gc" UpstreamErrorCfgRawKey string = "upstream.errors" - ACIDCfgRawKey string = "acid" NamespaceCfgRawKey string = "namespaces" SQLBackendCfgRawKey string = "sqlBackend" DBInternalCfgRawKey string = "dbInternal" diff --git a/internal/stackql/runtimeconfig/runtime_ctx.go b/internal/stackql/runtimeconfig/runtime_ctx.go index fac89ab6..275ab95f 100644 --- a/internal/stackql/runtimeconfig/runtime_ctx.go +++ b/internal/stackql/runtimeconfig/runtime_ctx.go @@ -51,7 +51,6 @@ type RuntimeCtx struct { NamespaceCfgRaw string StoreTxnCfgRaw string GCCfgRaw string - ACIDCfgRaw string UpstreamErrorCfgRaw string StrictUpstreamErrors bool QueryCacheSize int @@ -171,8 +170,6 @@ func (rc *RuntimeCtx) Set(key string, val string) error { rc.StoreTxnCfgRaw = val case GCCfgRawKey: rc.GCCfgRaw = val - case ACIDCfgRawKey: - rc.ACIDCfgRaw = val case UpstreamErrorCfgRawKey: rc.UpstreamErrorCfgRaw = val case OutfilePathKey: @@ -271,7 +268,6 @@ func (rc RuntimeCtx) Copy() RuntimeCtx { NamespaceCfgRaw: rc.NamespaceCfgRaw, StoreTxnCfgRaw: rc.StoreTxnCfgRaw, GCCfgRaw: rc.GCCfgRaw, - ACIDCfgRaw: rc.ACIDCfgRaw, UpstreamErrorCfgRaw: rc.UpstreamErrorCfgRaw, StrictUpstreamErrors: rc.StrictUpstreamErrors, QueryCacheSize: rc.QueryCacheSize, diff --git a/internal/stackql/runtimeconfig/txn_coordinator_config.go b/internal/stackql/runtimeconfig/txn_coordinator_config.go deleted file mode 100644 index 8827c9e6..00000000 --- a/internal/stackql/runtimeconfig/txn_coordinator_config.go +++ /dev/null @@ -1,26 +0,0 @@ -package dto - -import ( - "gopkg.in/yaml.v2" -) - -const ( - defaultMaxTxnDepth = -1 -) - -type TxnCoordinatorCfg struct { - MaxTxnDepth *int `json:"maxTransactionDepth" yaml:"maxTransactionDepth"` -} - -func (t TxnCoordinatorCfg) GetMaxTxnDepth() int { - if t.MaxTxnDepth == nil { - return defaultMaxTxnDepth - } - return *t.MaxTxnDepth -} - -func GetTxnCoordinatorCfgCfg(s string) (TxnCoordinatorCfg, error) { - rv := TxnCoordinatorCfg{} - err := yaml.Unmarshal([]byte(s), &rv) - return rv, err -} diff --git a/internal/stackql/sessionconfig/set.go b/internal/stackql/sessionconfig/set.go new file mode 100644 index 00000000..da883427 --- /dev/null +++ b/internal/stackql/sessionconfig/set.go @@ -0,0 +1,81 @@ +package sessionconfig + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" +) + +// Apply is the shared SET/UNSET value decoder; the host applies authorization. +// +//nolint:exhaustive // Unsupported literal kinds are explicitly rejected by the default branch. +func Apply(host interface { + SetConfigAtPath(string, interface{}, string) error +}, setExpr *sqlparser.SetExpr) error { + lhsTrimmed := setExpr.Name.GetRawVal() + config := strings.HasPrefix(lhsTrimmed, "$") + if config { + lhsTrimmed = strings.TrimPrefix(strings.TrimPrefix(lhsTrimmed, "$"), ".") + root, _, _ := strings.Cut(lhsTrimmed, ".") + switch root { + case "auth", "providers", "secret_store", "secrets", "document_merge", "transaction": + default: + return fmt.Errorf("unknown intrinsic session configuration %q", lhsTrimmed) + } + } else if !strings.HasPrefix(lhsTrimmed, "variables.") { + lhsTrimmed = "variables." + lhsTrimmed + } + if _, reset := setExpr.Expr.(*sqlparser.Default); reset { + return host.SetConfigAtPath(lhsTrimmed, nil, setExpr.Scope) + } + if !config { + raw, err := scalarVariable(setExpr.Expr) + if err != nil { + return err + } + return host.SetConfigAtPath(lhsTrimmed, raw, setExpr.Scope) + } + value, ok := setExpr.Expr.(*sqlparser.SQLVal) + if !ok || value.Type != sqlparser.StrVal { + return fmt.Errorf("session configuration requires a string literal or DEFAULT") + } + exprStr := string(value.Val) + if strings.HasPrefix(lhsTrimmed, "secrets.") { + return host.SetConfigAtPath(lhsTrimmed, exprStr, setExpr.Scope) + } + var exprObj map[string]interface{} + if err := json.Unmarshal([]byte(exprStr), &exprObj); err == nil && exprObj != nil { + return host.SetConfigAtPath(lhsTrimmed, exprObj, setExpr.Scope) + } + return host.SetConfigAtPath(lhsTrimmed, exprStr, setExpr.Scope) +} + +var ErrSessionConfiguration = errors.New("session configuration failed") + +func scalarVariable(expr sqlparser.Expr) (json.RawMessage, error) { + switch value := expr.(type) { + case *sqlparser.NullVal: + return json.RawMessage("null"), nil + case sqlparser.BoolVal: + return json.RawMessage(fmt.Sprint(bool(value))), nil + case *sqlparser.SQLVal: + return scalarSQLValue(value) + default: + return nil, fmt.Errorf("variables accepts scalar literals only") + } +} + +//nolint:exhaustive // Non-scalar literal kinds are rejected. +func scalarSQLValue(value *sqlparser.SQLVal) (json.RawMessage, error) { + switch value.Type { + case sqlparser.StrVal: + return json.Marshal(string(value.Val)) + case sqlparser.IntVal, sqlparser.FloatVal: + return json.RawMessage(value.Val), nil + default: + return nil, fmt.Errorf("variables accepts scalar literals only") + } +} diff --git a/internal/stackql/transaction/orchestrator.go b/internal/stackql/transaction/orchestrator.go new file mode 100644 index 00000000..b30200a3 --- /dev/null +++ b/internal/stackql/transaction/orchestrator.go @@ -0,0 +1,278 @@ +package transaction + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/stackql-parser/go/vt/sqlparser" + "github.com/stackql/stackql/internal/stackql/handler" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/intrinsic" + "github.com/stackql/stackql/internal/stackql/querysubmit" +) + +type providerTransactionHost interface { + handler.HandlerContext + ProviderTransaction() (*omnisdk.TransactionSession, omnisdk.TransactionHandle, bool) + BeginProviderTransaction(context.Context) error + ClearProviderTransaction() +} + +// Orchestrator translates SQL control statements; Omni owns transaction semantics. +type Orchestrator interface { + ProcessQueryOrQueries(handler.HandlerContext) ([]internaldto.ExecutorOutput, bool) +} +type omniOrchestrator struct{} +type queuedSQL struct { + SQL string `json:"sql"` +} + +func NewOrchestrator(_ handler.HandlerContext) (Orchestrator, error) { return &omniOrchestrator{}, nil } + +func (o *omniOrchestrator) ProcessQueryOrQueries(hc handler.HandlerContext) ([]internaldto.ExecutorOutput, bool) { + host, ok := hc.(providerTransactionHost) + if !ok { + return providerTransactionError(fmt.Errorf("transaction host is unavailable")), true + } + pieces, err := sqlparser.SplitStatementToPieces(hc.GetRawQuery()) + if err != nil { + return []internaldto.ExecutorOutput{internaldto.NewErroneousExecutorOutput(err)}, true + } + var output []internaldto.ExecutorOutput + for _, sql := range pieces { + if sql == "" { + continue + } + var result []internaldto.ExecutorOutput + _, _, active := host.ProviderTransaction() + statement, parseErr := sqlparser.Parse(sql) + if parseErr != nil { + output = append(output, internaldto.NewErroneousExecutorOutput(parseErr)) + break + } + if active || transactionControl(statement) { + result = markTransactionFailures(o.transactionQuery(host, sql, statement)) + } else { + result = []internaldto.ExecutorOutput{executeProviderSQL(host, host.GetExecutionContext(), sql)} + } + output = append(output, result...) + if transactionOutputFailed(result) { + break + } + } + return output, len(output) > 0 +} +func transactionCaller(host providerTransactionHost) (context.Context, func(), error) { + originalCtx, originalCaller := host.GetExecutionContext(), host.GetCaller() + manager := host.GetAccessManager() + if manager == nil { + return originalCtx, func() {}, nil + } + caller, ctx, cancel, err := manager.Establish(originalCtx) + if err != nil { + return nil, func() {}, err + } + host.SetCaller(caller) + host.SetExecutionContext(ctx) + return ctx, func() { host.SetExecutionContext(originalCtx); host.SetCaller(originalCaller); cancel() }, nil +} +func executeProviderSQL(host providerTransactionHost, ctx context.Context, sql string) internaldto.ExecutorOutput { + local := host.Clone() + local.SetQuery(sql) + local.SetExecutionContext(ctx) + stmt := querysubmit.NewQuerySubmitter() + if err := stmt.PrepareQuery(local); err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + return stmt.SubmitQuery() +} +func (o *omniOrchestrator) transactionQuery( + host providerTransactionHost, sql string, statement sqlparser.Statement, +) []internaldto.ExecutorOutput { + requestCtx := host.GetExecutionContext() + ctx, finish, err := transactionCaller(host) + if err != nil { + return []internaldto.ExecutorOutput{internaldto.NewErroneousExecutorOutput(err)} + } + defer finish() + if _, begin := statement.(*sqlparser.Begin); begin { + if err = host.BeginProviderTransaction(ctx); err != nil { + return providerTransactionError(err) + } + _, handle, _ := host.ProviderTransaction() + return []internaldto.ExecutorOutput{ + internaldto.NewNopEmptyExecutorOutput([]string{"transaction begun: " + handle.ID()}), + } + } + session, handle, active := host.ProviderTransaction() + if !active { + return providerTransactionError(fmt.Errorf("no active provider transaction")) + } + if intrinsic.ProviderTransactionControlQuery(statement) { + return []internaldto.ExecutorOutput{executeProviderSQL(host, requestCtx, sql)} + } + var output []internaldto.ExecutorOutput + drain := func() error { + return drainProviderStatements(host, ctx, session, handle, &output) + } + + switch statement.(type) { + case *sqlparser.Rollback: + if err = intrinsic.RollbackProviderTransaction(host, session, handle); err != nil { + return providerTransactionError(err) + } + host.ClearProviderTransaction() + return []internaldto.ExecutorOutput{internaldto.NewNopEmptyExecutorOutput([]string{"Rollback OK"})} + case *sqlparser.Commit: + if err = drain(); err == nil { + err = session.Commit(ctx, handle) + } + if err != nil { + return append(output, internaldto.NewErroneousExecutorOutput(err)) + } + host.ClearProviderTransaction() + return append(output, internaldto.NewNopEmptyExecutorOutput([]string{"transaction committed"})) + case *sqlparser.Insert, *sqlparser.Update, *sqlparser.Delete: + queued, admissionErr := queueProviderMutation(host, ctx, session, handle, statement) + if admissionErr != nil { + _ = session.Abort(ctx, handle, admissionErr) + return providerTransactionError(admissionErr) + } + if queued { + return []internaldto.ExecutorOutput{ + internaldto.NewNopEmptyExecutorOutput([]string{"mutating statement queued"}), + } + } + case *sqlparser.Select, *sqlparser.Union, *sqlparser.ParenSelect: + // A read is a dependency barrier: preceding writes must be observed. + default: + err = fmt.Errorf("statement cannot execute inside a provider transaction") + _ = session.Abort(ctx, handle, err) + return providerTransactionError(err) + } + if err = drain(); err != nil { + return append(output, internaldto.NewErroneousExecutorOutput(err)) + } + // Normal submission owns its fresh authentication context and streaming lifetime. + result := executeProviderSQL(host, requestCtx, sql) + if result.GetError() != nil { + _ = session.Abort(ctx, handle, result.GetError()) + } + return append(output, result) +} + +// Preserve identifier boundaries when persisting SQL for later parsing. +func formatQueuedSQL(buf *sqlparser.TrackedBuffer, node sqlparser.SQLNode) { + if table, ok := node.(sqlparser.TableName); ok { + first := true + parts := []sqlparser.TableIdent{table.QualifierThird, table.QualifierSecond, table.Qualifier, table.Name} + for _, part := range parts { + if part.IsEmpty() { + continue + } + if !first { + buf.WriteByte('.') + } + buf.WriteByte('"') + buf.WriteString(strings.ReplaceAll(part.GetRawVal(), "\"", "\"\"")) + buf.WriteByte('"') + first = false + } + return + } + node.Format(buf) +} + +func transactionOutputFailed(output []internaldto.ExecutorOutput) bool { + for _, result := range output { + if result.GetError() != nil { + return true + } + } + return false +} + +func queueProviderMutation(host providerTransactionHost, ctx context.Context, + session *omnisdk.TransactionSession, handle omnisdk.TransactionHandle, statement sqlparser.Statement, +) (bool, error) { + if err := intrinsic.ProviderTransactionMutation(host, statement); err != nil { + return false, err + } + cfg, err := session.Configuration(ctx, handle) + if err != nil { + return false, err + } + if cfg.Mode != omnisdk.TransactionLazy || mutationReturnsRows(statement) { + return false, nil + } + if err = host.ResolveVariables(statement); err != nil { + return false, err + } + formatted := sqlparser.NewTrackedBuffer(formatQueuedSQL) + formatted.Myprintf("%v", statement) + payload, err := json.Marshal(queuedSQL{SQL: formatted.String()}) + if err != nil { + return false, err + } + if err = session.Queue(ctx, handle, payload); err != nil { + return false, err + } + return true, nil +} + +func mutationReturnsRows(statement sqlparser.Statement) bool { + switch node := statement.(type) { + case *sqlparser.Insert: + return len(node.SelectExprs) > 0 + case *sqlparser.Update: + return len(node.SelectExprs) > 0 + case *sqlparser.Delete: + return len(node.SelectExprs) > 0 + default: + return false + } +} + +func drainProviderStatements(host providerTransactionHost, ctx context.Context, + session *omnisdk.TransactionSession, handle omnisdk.TransactionHandle, output *[]internaldto.ExecutorOutput, +) error { + return session.Drain(ctx, handle, func(runCtx context.Context, payload json.RawMessage) error { + var queued queuedSQL + if err := json.Unmarshal(payload, &queued); err != nil { + return err + } + result := executeProviderSQL(host, runCtx, queued.SQL) + *output = append(*output, result) + return result.GetError() + }) +} + +func providerTransactionError(err error) []internaldto.ExecutorOutput { + return []internaldto.ExecutorOutput{internaldto.NewErroneousExecutorOutput(err)} +} + +func transactionControl(statement sqlparser.Statement) bool { + switch statement.(type) { + case *sqlparser.Begin, *sqlparser.Commit, *sqlparser.Rollback: + return true + default: + return false + } +} + +// Transaction failures must remain failures even when caller auth is disabled. +type transactionError struct{ error } + +func (e transactionError) Unwrap() error { return e.error } +func (e transactionError) TransactionFailure() bool { return true } +func markTransactionFailures(results []internaldto.ExecutorOutput) []internaldto.ExecutorOutput { + for i, result := range results { + if err := result.GetError(); err != nil { + results[i] = internaldto.NewErroneousExecutorOutput(transactionError{err}) + } + } + return results +} diff --git a/internal/stackql/transaction/orchestrator_test.go b/internal/stackql/transaction/orchestrator_test.go new file mode 100644 index 00000000..b47c163b --- /dev/null +++ b/internal/stackql/transaction/orchestrator_test.go @@ -0,0 +1,50 @@ +//nolint:testpackage // Tests the private SQL snapshot formatter without exposing it as API. +package transaction + +import ( + "testing" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" +) + +func TestQueuedSQLPreservesQualifiedIdentifiers(t *testing.T) { + for _, query := range []string{ + "INSERT INTO stackql_unstable_accessdemo.service.items(name) SELECT 'transaction-item'", + "SELECT * FROM a.b.c.d", + "SELECT * FROM a.b.\"name.with.dot\"", + } { + original, err := sqlparser.Parse(query) + if err != nil { + t.Fatal(err) + } + buf := sqlparser.NewTrackedBuffer(formatQueuedSQL) + buf.Myprintf("%v", original) + restored, err := sqlparser.Parse(buf.String()) + if err != nil { + t.Fatalf("snapshot %q: %v", buf.String(), err) + } + var before, after []sqlparser.TableName + collect := func(dst *[]sqlparser.TableName) sqlparser.Visit { + return func(node sqlparser.SQLNode) (bool, error) { + if name, ok := node.(sqlparser.TableName); ok { + *dst = append(*dst, name) + } + return true, nil + } + } + if err := sqlparser.Walk(collect(&before), original); err != nil { + t.Fatal(err) + } + if err := sqlparser.Walk(collect(&after), restored); err != nil { + t.Fatal(err) + } + if len(before) != len(after) { + t.Fatalf("table count changed: %q", buf.String()) + } + for i := range before { + if before[i] != after[i] { + t.Fatalf("identifier changed: %#v -> %#v", before[i], after[i]) + } + } + } +} diff --git a/test/python/stackql_test_tooling/access_management_keywords.py b/test/python/stackql_test_tooling/access_management_keywords.py index a8e45345..e368699f 100644 --- a/test/python/stackql_test_tooling/access_management_keywords.py +++ b/test/python/stackql_test_tooling/access_management_keywords.py @@ -146,6 +146,8 @@ def do_GET(self): if scenario.startswith("query_deadline"): time.sleep(2) counts["read"] += 1 + if scenario.startswith("provider_transaction"): + counts.setdefault("read_after_mutations", []).append(counts["mutation"]) self.send_response(200) self.send_header("Content-Type", "application/json") self.end_headers() @@ -166,7 +168,10 @@ def do_POST(self): self.end_headers() identity = "managed-" + str(counts["mutation"]) managed[identity] = dict(body, id=identity) - self.wfile.write(json.dumps(managed[identity]).encode()) + response = dict(managed[identity]) + if scenario == "provider_transaction_recovery": + response.pop("id", None) + self.wfile.write(json.dumps(response).encode()) def do_PATCH(self): counts["mutation"] += 1 @@ -242,6 +247,8 @@ def do_DELETE(self): resource["sqlVerbs"]["select"].append({"$ref": "#/components/x-stackQL-resources/items/methods/get"}) resource["sqlVerbs"]["update"] = [{"$ref": "#/components/x-stackQL-resources/items/methods/update"}] resource["sqlVerbs"]["delete"] = [{"$ref": "#/components/x-stackQL-resources/items/methods/delete"}] + if scenario.startswith("provider_transaction") and scenario!="provider_transaction_missing_inverse": + resource["methods"]["create"]["inverse"]={"sqlVerb":{"$ref":"#/components/x-stackQL-resources/items/methods/delete"},"tokens":{"id":{"key":"id","location":"body","algorithm":"toString"}}} service_file.write_text(yaml.safe_dump(service)) def token(subject="reader", updates=None, remove=(), signer=None, kid="local"): @@ -360,6 +367,11 @@ def denied(sql, credential=None, concurrent=False): else: assert code == 0 and "known-item" in output, "default deadline interrupted query: " + output return + if scenario.startswith("provider_transaction"): + from stackql_test_tooling.provider_transaction_scenarios import verify_provider_transaction + verify_provider_transaction(scenario, run, token, managed, counts, root, + base=base, config_file=config_file) + return if scenario == "config": self._verify_configuration(base, config_file, config, reader, query, counts) return diff --git a/test/python/stackql_test_tooling/provider_transaction_scenarios.py b/test/python/stackql_test_tooling/provider_transaction_scenarios.py new file mode 100644 index 00000000..09324b33 --- /dev/null +++ b/test/python/stackql_test_tooling/provider_transaction_scenarios.py @@ -0,0 +1,174 @@ +"""Provider transaction scenarios sharing the authenticated access fixture.""" +import json +import re +import time +import os +import subprocess +import threading +from pathlib import Path + + +def verify_provider_transaction(scenario, run, token, managed, counts, root, base, config_file): + config = {'collection': 'sql-test', 'keys': ['item'], 'targets': { + 'stackql_unstable_accessdemo.service.items': {'key': 'item', 'identity': 'id', 'addressed_by': 'id'}}} + if scenario == 'provider_transaction_eager_rollback': + config['mode'] = 'eager' + configure = 'SET "$.transaction" = \'' + json.dumps(config) + "';" + insert = "INSERT INTO stackql_unstable_accessdemo.service.items(name) SELECT 'transaction-item';" + read = 'SELECT id FROM stackql_unstable_accessdemo.service.items;' + operator = token(subject='operator') + if scenario == 'provider_transaction_default': + # No $.transaction SET and no caller-auth opt-in. + code, output = run('BEGIN;' + insert + insert + read + 'ROLLBACK;', protected=False) + assert code == 0 and 'Rollback OK' in output, output + assert not managed and counts['mutation'] == 4, (managed, counts) + code, output = run('BEGIN;' + insert + 'COMMIT;', protected=False) + assert code == 0 and len(managed) == 1 and counts['mutation'] == 5, output + records = list((Path(root) / 'iac').rglob('ledger/sql/.transactions/*.json')) + assert records, 'default SQL transaction did not persist Omni checkpoints' + code, output = run('ROLLBACK;', protected=False) + assert code != 0 and counts['mutation'] == 5, f'standalone rollback code={code}: {output}' + return + if scenario == 'provider_transaction_auth_disabled': + for mode in ('eager', 'lazy'): + config['mode'] = mode + configure = 'SET "$.transaction" = \' ' + json.dumps(config) + "';" + code, output = run(configure + 'BEGIN;' + insert + read + 'ROLLBACK;', protected=False) + assert code == 0 and 'Rollback OK' in output and not managed, output + assert counts['mutation'] == 4, counts + return + if scenario in ('provider_transaction_resume', 'provider_transaction_resume_queue', 'provider_transaction_recovery'): + config['lease_seconds'] = 3 + config['mode'] = 'lazy' if scenario == 'provider_transaction_resume_queue' else 'eager' + configure = 'SET "$.transaction" = \' ' + json.dumps(config) + "';" + code, output = run(configure + 'BEGIN;' + insert, operator) + if scenario == 'provider_transaction_recovery': + assert code != 0 and 'inverse token' in output, output + else: + assert code == 0, output + match = re.search(r'transaction begun: (\S+)', output) + assert match, output + transaction_id = match.group(1) + config['resume_id'] = transaction_id + configure = 'SET "$.transaction" = \' ' + json.dumps(config) + "';" + # The process exited with an open transaction. Reclaim its actual durable + # lease by expiry, without rewriting state files or force-unlock tricks. + time.sleep(4) + if scenario == 'provider_transaction_resume_queue': + assert not managed and counts['mutation'] == 0, (managed, counts) + code, output = run(configure + 'BEGIN;COMMIT;', operator) + assert code == 0 and len(managed) == 1 and counts['mutation'] == 1, output + elif scenario == 'provider_transaction_resume': + assert len(managed) == 1 and counts['mutation'] == 1, (managed, counts) + code, output = run(configure + 'BEGIN;ROLLBACK;', operator) + assert code == 0 and not managed and counts['mutation'] == 2, output + else: + assert len(managed) == 1, managed + output = recover_in_shell(base, config_file, operator, configure, next(iter(managed))) + assert 'Rollback OK' in output and not managed and counts['mutation'] == 2, output + + # A closed durable transaction ID cannot replay its work. + before = counts['mutation'] + code, output = run(configure + 'BEGIN;', operator) + assert code != 0 and counts['mutation'] == before, output + return + if scenario == 'provider_transaction_lazy_rollback': + code, output = run(configure + 'BEGIN;' + insert + 'ROLLBACK;', operator) + assert code == 0 and 'Rollback OK' in output, output + assert not managed and counts['mutation'] == 0, (managed, counts) + elif scenario == 'provider_transaction_eager_rollback': + code, output = run(configure + 'BEGIN;' + insert + 'ROLLBACK;', operator) + assert code == 0 and 'Rollback OK' in output, output + assert not managed and counts['mutation'] == 2, (managed, counts) + elif scenario == 'provider_transaction_returning': + returning = insert.rstrip(';') + ' RETURNING name;' + code, output = run(configure + 'BEGIN;' + returning + 'ROLLBACK;', operator) + assert code == 0 and 'transaction-item' in output and 'Rollback OK' in output, output + assert not managed and counts['mutation'] == 2, (managed, counts) + elif scenario == 'provider_transaction_missing_inverse': + code, output = run(configure + 'BEGIN;' + insert + 'COMMIT;', operator) + assert code != 0 and 'declared inverse' in output, output + assert not managed and counts['mutation'] == 0, (managed, counts) + elif scenario == 'provider_transaction_status': + code, output = run(configure + 'BEGIN;SELECT * FROM stackql_preview.txn.status;ROLLBACK;', operator) + assert code == 0 and 'version' in output and 'active' in output, output + assert not managed and counts['mutation'] == 0, (managed, counts) + elif scenario == 'provider_transaction_read_barrier': + code, output = run(configure + 'BEGIN;' + insert + read + 'ROLLBACK;', operator) + assert code == 0 and 'Rollback OK' in output, output + assert counts['read_after_mutations'] and counts['read_after_mutations'][0] == 1, counts + assert not managed and counts['mutation'] == 2, (managed, counts) + elif scenario == 'provider_transaction_commit': + code, output = run(configure + 'BEGIN;' + insert + 'COMMIT;', operator) + assert code == 0 and 'transaction committed' in output, output + assert len(managed) == 1 and counts['mutation'] == 1, (managed, counts) + records = list((Path(root) / 'iac').rglob('ledger/sql-test/.transactions/*.json')) + assert records, 'transaction status was not persisted in the IaC ledger' + assert list((Path(root) / 'iac').rglob('journal/txn-*.jsonl')), 'mutation was not journalled' + elif scenario == 'provider_transaction_irreversible': + code, output = run(configure + "BEGIN; DELETE FROM stackql_unstable_accessdemo.service.items WHERE id = 'existing';", operator) + assert code != 0 and 'inverse' in output.lower(), output + assert not managed and counts['mutation'] == 0, (managed, counts) + elif scenario == 'provider_transaction_denied': + code, output = run(configure + 'BEGIN;' + insert + 'COMMIT;', token(subject='reader')) + assert code != 0 and 'denied' in output.lower(), output + assert counts['mutation'] == 0 and not managed, (managed, counts) + else: + raise AssertionError('unknown transaction scenario: ' + scenario) + + +def status_row(output): + for match in re.finditer(r'\[\s*\{', output): + try: + rows, _ = json.JSONDecoder().raw_decode(output[match.start():]) + except ValueError: + continue + if rows and 'version' in rows[0] and 'pending' in rows[0]: + return rows[0] + raise AssertionError('missing transaction status: ' + output) + + +def recover_in_shell(base, config_file, credential, configure, identity): + env = os.environ.copy() + env['STACKQL_CALLER_TOKEN'] = credential + process = subprocess.Popen(base + ['--access.config', json.dumps({'path': str(config_file)}), 'shell'], + env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) + lines = [] + def read_output(): + # JSON output need not end in a newline; inspect it without waiting for EOF. + while True: + char = process.stdout.read(1) + if not char: + return + lines.append(char) + reader = threading.Thread(target=read_output, daemon=True) + reader.start() + try: + process.stdin.write(configure + '\nBEGIN;\nSELECT * FROM stackql_preview.txn.status;\n') + process.stdin.flush() + deadline = time.monotonic() + 25 + while True: + output = ''.join(lines) + try: + snapshot = status_row(output) + break + except AssertionError: + assert time.monotonic() < deadline and process.poll() is None, output + time.sleep(0.02) + pending = json.loads(snapshot['pending']) + assert len(pending) == 1, snapshot + resources = json.dumps([{'key': 'item', 'outcome': 'applied', 'version': pending[0]['version'], 'identity': identity}]) + command = "SELECT * FROM stackql_preview.txn.recover WHERE version = '" + snapshot['version'] + "' AND resources = '" + resources + "' AND continue = 'false';" + process.stdin.write(command + '\nROLLBACK;\nexit\n') + process.stdin.flush() + process.wait(timeout=25) + reader.join(3) + output = ''.join(lines) + assert credential not in output, 'caller credential leaked' + return output + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + process.stdin.close() + process.stdout.close() diff --git a/test/robot/functional/stackql_provider_transactions.robot b/test/robot/functional/stackql_provider_transactions.robot new file mode 100644 index 00000000..3d52229b --- /dev/null +++ b/test/robot/functional/stackql_provider_transactions.robot @@ -0,0 +1,46 @@ +*** Settings *** +Resource ${CURDIR}/stackql.resource +Library stackql_test_tooling.access_management_keywords.AccessManagementKeywords + +*** Test Cases *** +Provider Lazy Rollback Discards Unexecuted Mutations + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_lazy_rollback + +Provider Read Drains Preceding Mutation And Rollback Uses IaC Inverse + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_read_barrier + +Provider Commit Persists Journal And Transaction Status + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_commit + +Provider Irreversible Mutation Is Rejected Before Dispatch + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_irreversible + +Provider Transaction Begin Requires Authorization + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_denied + +Provider Eager Rollback Uses The Shared Inverse + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_eager_rollback + +Provider Returning Preserves Inverse Identity Outside Projection + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_returning + +Provider Create Without Declared Inverse Never Dispatches + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_missing_inverse + +Provider Transaction Status Is An Authorized Session Bound Read + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_status + +Provider Committed Effects Resume After Process Exit And Lease Expiry + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_resume + +Provider Deferred Queue Survives Process Exit Without Duplicate Creation + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_resume_queue + +Provider Missing Handle Requires Explicit Recovery Before Rollback + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_recovery + +Provider Transactions Work Without Opting Into Caller Auth + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_auth_disabled + +Unconfigured SQL Transactions Use Omni And Keep Creates Distinct + Verify Caller Access ${STACKQL_EXE} ${SQL_BACKEND_CFG_STR_CANONICAL} ${EXECUTION_PLATFORM} provider_transaction_default