Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export REPO_ROOT := $(shell pwd)
# path, so adding a provider is mostly adding a directory — see
# service/submitqueue/demo/provider/README.md.
#
# fake a change is a URI; nothing merges anywhere. Needs nothing.
# fake a change is a URI; nothing lands anywhere. Needs nothing.
# git branches in a bare repository on disk; real fetch, cherry-pick, push.
# github real pull requests. Needs a repository and GITHUB_TOKEN.
PROVIDER ?= fake
Expand All @@ -60,7 +60,7 @@ PROVIDER_COMPOSE_FILE_git = service/submitqueue/docker-compose.git.yml
PROVIDER_COMPOSE_FILE_github = service/submitqueue/docker-compose.provider.yml
PROVIDER_COMPOSE_FILE = $(PROVIDER_COMPOSE_FILE_$(PROVIDER))

# Where PROVIDER=git keeps the bare repository it merges into. Outside the
# Where PROVIDER=git keeps the bare repository it lands into. Outside the
# repository, so a demo leaves nothing in a checkout, and bind-mounted rather
# than kept in a volume so `git log` on the host can show what landed.
#
Expand Down Expand Up @@ -266,7 +266,7 @@ deps: tidy-go ## Download and tidy Go dependencies
e2e-git-test: ## Run the hermetic git E2E (real merger against a bare repo; no credentials)
@echo "Running hermetic git end-to-end tests..."
@$(BAZEL) test //test/e2e/submitqueue:go_default_test --test_output=errors \
--test_filter='TestGitMergeE2E'
--test_filter='TestGitLandE2E'

e2e-test: ## Run end-to-end tests (hermetic; Bazel builds all inputs; runs in parallel)
@echo "Running end-to-end tests (parallel)..."
Expand Down Expand Up @@ -519,7 +519,7 @@ local-submitqueue-start: build-all-linux ## Start full stack (PROVIDER=fake|git|
@echo ""
@echo "Gateway gRPC port: $$(docker port $(SUBMITQUEUE_LOCAL_PROJECT)-gateway-service-1 8080 2>/dev/null | cut -d: -f2 || echo 'unknown')"
@if [ "$(PROVIDER)" = "git" ]; then \
echo "Merge target: $(SQ_GIT_SANDBOX_DIR)/sandbox.git"; \
echo "Land target: $(SQ_GIT_SANDBOX_DIR)/sandbox.git"; \
fi
@echo ""
@echo "Generate traffic with:"
Expand Down Expand Up @@ -579,7 +579,7 @@ local-stovepipe-stop: ## Stop the Stovepipe service

mocks: ## Generate mock files using mockgen
@echo "Generating mocks..."
@$(BAZEL) run @rules_go//go -- generate ./submitqueue/extension/storage/... ./submitqueue/extension/buildrunner/... ./submitqueue/extension/changeprovider/... ./platform/extension/counter/... ./platform/extension/consumergate/... ./platform/extension/hook/... ./platform/extension/messagequeue/... ./submitqueue/extension/queueconfig/... ./submitqueue/extension/conflict/... ./submitqueue/extension/speculation/... ./submitqueue/extension/validator/... ./platform/consumer/... ./stovepipe/core/requestlog/... ./stovepipe/extension/storage/... ./stovepipe/extension/sourcecontrol/...
@$(BAZEL) run @rules_go//go -- generate ./submitqueue/extension/storage/... ./submitqueue/extension/buildrunner/... ./submitqueue/extension/changeprovider/... ./platform/extension/counter/... ./platform/extension/consumergate/... ./platform/extension/hook/... ./platform/extension/messagequeue/... ./submitqueue/extension/queueconfig/... ./runway/extension/merger/... ./submitqueue/extension/conflict/... ./submitqueue/extension/speculation/... ./submitqueue/extension/validator/... ./platform/consumer/... ./stovepipe/core/requestlog/... ./stovepipe/extension/storage/... ./stovepipe/extension/sourcecontrol/...
@echo "Mocks generated successfully!"

proto: ## Generate protobuf files from .proto definitions
Expand Down
10 changes: 5 additions & 5 deletions platform/publish/publish_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ func TestIntentID(t *testing.T) {
{
name: "single cause",
entityID: "batch-1",
cause: []string{"merged"},
want: "batch-1/merged",
cause: []string{"landed"},
want: "batch-1/landed",
},
{
name: "multiple causes join in order",
Expand All @@ -164,9 +164,9 @@ func TestIntentID(t *testing.T) {
// The convention only works if the same cause is repeatable and a different
// cause is distinguishable — the two properties every call site relies on.
func TestIntentID_StableAcrossCallsAndDistinctPerCause(t *testing.T) {
assert.Equal(t, IntentID("batch-1", "merged"), IntentID("batch-1", "merged"))
assert.NotEqual(t, IntentID("batch-1", "merged"), IntentID("batch-1"))
assert.NotEqual(t, IntentID("batch-1", "merged"), IntentID("batch-1", "cancelling"))
assert.Equal(t, IntentID("batch-1", "landed"), IntentID("batch-1", "landed"))
assert.NotEqual(t, IntentID("batch-1", "landed"), IntentID("batch-1"))
assert.NotEqual(t, IntentID("batch-1", "landed"), IntentID("batch-1", "cancelling"))
}

func TestUniqueID(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions service/submitqueue/demo/requests/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
//
// - fake (default): a change is a URI and nothing else. No repository, no
// credential, no I/O — the fastest way to put traffic through the queue.
// - git: a branch pushed to the sandbox repository the stack merges into.
// - git: a branch pushed to the sandbox repository the stack lands into.
// Real commits, still no credential.
// - github: a real pull request over the REST API, which needs no clone and
// no git binary, only GITHUB_TOKEN — the same credential the stack uses.
Expand Down Expand Up @@ -130,7 +130,7 @@ func parseFlags() config {
flag.BoolVar(&c.tls, "tls", false, "dial the gateway with transport security")
flag.StringVar(&c.tokenEnv, "token-env", client.DefaultTokenEnv, "environment variable holding the gateway bearer token")
flag.StringVar(&c.queue, "queue", "demo-queue", "queue to land on")
flag.StringVar(&c.strategy, "strategy", "SQUASH_REBASE", "merge strategy")
flag.StringVar(&c.strategy, "strategy", "SQUASH_REBASE", "land strategy")
flag.Parse()

// Only the GitHub source reads a credential; the other two must not fail,
Expand Down
6 changes: 3 additions & 3 deletions service/submitqueue/gateway/server/mapper/land.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ var errUnknownStrategy = errors.New("unknown land strategy in proto message")
// ProtoToLandRequest maps the wire LandRequest to the entity.LandRequest the controller operates on.
// The ID is left empty; the controller assigns it.
func ProtoToLandRequest(req *pb.LandRequest) (entity.LandRequest, error) {
strategy, err := resolveMergeStrategy(req.GetStrategy())
strategy, err := resolveLandStrategy(req.GetStrategy())
if err != nil {
return entity.LandRequest{}, fmt.Errorf("failed to map land strategy: %w", err)
}
Expand All @@ -47,8 +47,8 @@ func ProtoToLandRequest(req *pb.LandRequest) (entity.LandRequest, error) {
}, nil
}

// resolveMergeStrategy maps a proto Strategy enum to the shared mergestrategy.MergeStrategy.
func resolveMergeStrategy(s mergestrategypb.Strategy) (mergestrategy.MergeStrategy, error) {
// resolveLandStrategy maps a proto Strategy enum to the shared mergestrategy.MergeStrategy.
func resolveLandStrategy(s mergestrategypb.Strategy) (mergestrategy.MergeStrategy, error) {
switch s {
case mergestrategypb.Strategy_DEFAULT:
// TODO: resolve default strategy based on queue configuration
Expand Down
4 changes: 2 additions & 2 deletions service/submitqueue/gateway/server/mapper/land_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func TestProtoToLandRequest(t *testing.T) {
}
}

func TestResolveMergeStrategy(t *testing.T) {
func TestResolveLandStrategy(t *testing.T) {
tests := []struct {
name string
in mergestrategypb.Strategy
Expand All @@ -100,7 +100,7 @@ func TestResolveMergeStrategy(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := resolveMergeStrategy(tt.in)
got, err := resolveLandStrategy(tt.in)
if tt.errMsg != "" {
assert.ErrorContains(t, err, tt.errMsg)
return
Expand Down
2 changes: 1 addition & 1 deletion submitqueue/core/batch/transition_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func TestEnsureRecord(t *testing.T) {
batch := entity.Batch{
ID: "monorepo/batch/7",
Queue: "monorepo",
State: entity.BatchStateMerging,
State: entity.BatchStateLanding,
Version: 5,
}
storeErr := errors.New("storage failed")
Expand Down
4 changes: 2 additions & 2 deletions submitqueue/core/request/request_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ func TestGetCurrentStateFromRequestLog(t *testing.T) {
name: "terminal error status with last error",
logs: []entity.RequestLog{
{RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}},
{RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 4, LastError: "merge conflict", Metadata: map[string]string{"step": "merge"}},
{RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 4, LastError: "merge conflict", Metadata: map[string]string{"step": "land"}},
},
expected: CurrentState{Status: entity.RequestStatusError, LastError: "merge conflict", Metadata: map[string]string{"step": "merge"}},
expected: CurrentState{Status: entity.RequestStatusError, LastError: "merge conflict", Metadata: map[string]string{"step": "land"}},
},
{
name: "multiple terminal records picks highest version",
Expand Down
8 changes: 4 additions & 4 deletions submitqueue/core/topickey/topickey.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,16 @@ const (
// so the state machine re-evaluates, and holds the delivery for the next
// poll when the build has not yet reached a terminal state.
TopicKeyBuildSignal TopicKey = "buildsignal"
// TopicKeyMerge is the pipeline stage where speculated batches are published for merging.
TopicKeyMerge TopicKey = "submitqueue-merge"
// TopicKeyConclude is the pipeline stage where merged requests are published for conclusion.
// TopicKeyLand is the pipeline stage where speculated batches are published for landing.
TopicKeyLand TopicKey = "submitqueue-land"
// TopicKeyConclude is the pipeline stage where landed requests are published for conclusion.
TopicKeyConclude TopicKey = "conclude"
// TopicKeyLog is the pipeline stage where per-request logs are written.
TopicKeyLog TopicKey = "log"
)

// MetadataKeyFailureReason is the conclude message's metadata attribute carrying
// a failed batch's human-readable reason. Set by the failure sites (merge and
// a failed batch's human-readable reason. Set by the failure sites (land and
// speculate) on the conclude publish and read by conclude to stamp the request's
// terminal log; absent on the landed and cancelled paths.
const MetadataKeyFailureReason = "failure_reason"
18 changes: 9 additions & 9 deletions submitqueue/entity/batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@ const (
BatchStateCreated BatchState = "created"
// BatchStateSpeculating is the state of a batch that is undergoing speculative execution.
BatchStateSpeculating BatchState = "speculating"
// BatchStateMerging is the state of a batch that is being merged after speculative execution.
BatchStateMerging BatchState = "merging"
// BatchStateLanding is the state of a batch that is being landed after speculative execution.
BatchStateLanding BatchState = "landing"
// BatchStateSucceeded is the terminal state of a batch that has been successfully landed.
BatchStateSucceeded BatchState = "succeeded"
// BatchStateFailed is the terminal state of a batch that has failed.
BatchStateFailed BatchState = "failed"
// BatchStateCancelling is the non-terminal intent state set when a cancel has been requested but the
// batch has not yet been transitioned to BatchStateCancelled. A batch in this state may still reach
// BatchStateSucceeded or BatchStateFailed if a concurrent merge wins the race (e.g. the push had
// BatchStateSucceeded or BatchStateFailed if a concurrent land wins the race (e.g. the push had
// already completed before the cancel CAS observed the batch); those terminal states prevail.
// Forward-progress controllers must treat this state as halted (no new work). The speculate
// controller owns the transition to the terminal BatchStateCancelled and the downstream fan-out
Expand Down Expand Up @@ -67,7 +67,7 @@ func (s BatchState) IsTerminal() bool {
var nonCancellableBatchStates = map[BatchState]bool{
BatchStateUnknown: true,
BatchStateCreating: true,
BatchStateMerging: true,
BatchStateLanding: true,
BatchStateSucceeded: true,
BatchStateFailed: true,
BatchStateCancelled: true,
Expand All @@ -80,7 +80,7 @@ func (s BatchState) IsCancellable() bool {
}

// IsBatchStateHalted returns true if the batch is either terminal or in the process of being cancelled.
// Forward-progress controllers (build, buildsignal, speculate, merge) use this to short-circuit
// Forward-progress controllers (build, buildsignal, speculate, land) use this to short-circuit
// work for batches that the user has asked to cancel — even though Cancelling is non-terminal, no
// further pipeline work should start (cancel will write the terminal state and fan out).
func IsBatchStateHalted(s BatchState) bool {
Expand All @@ -94,7 +94,7 @@ func AllBatchStates() []BatchState {
BatchStateCreating,
BatchStateCreated,
BatchStateSpeculating,
BatchStateMerging,
BatchStateLanding,
BatchStateSucceeded,
BatchStateFailed,
BatchStateCancelling,
Expand All @@ -108,7 +108,7 @@ func ActiveBatchStates() []BatchState {
return []BatchState{
BatchStateCreated,
BatchStateSpeculating,
BatchStateMerging,
BatchStateLanding,
BatchStateCancelling,
}
}
Expand All @@ -127,11 +127,11 @@ func DependencyBatchStates() []BatchState {
return []BatchState{
BatchStateCreated,
BatchStateSpeculating,
BatchStateMerging,
BatchStateLanding,
}
}

// Batch represents a group of requests to land (merge into target branch of the source control repository).
// Batch represents a group of requests to land on the source control repository's target branch.
type Batch struct {
// ID is the globally unique identifier for the batch. Format: "<queue>/batch/<counter_value>".
ID string
Expand Down
4 changes: 2 additions & 2 deletions submitqueue/entity/batch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestBatchState_IsTerminal(t *testing.T) {
{name: "creating", state: BatchStateCreating, terminal: false},
{name: "created", state: BatchStateCreated, terminal: false},
{name: "speculating", state: BatchStateSpeculating, terminal: false},
{name: "merging", state: BatchStateMerging, terminal: false},
{name: "landing", state: BatchStateLanding, terminal: false},
{name: "succeeded", state: BatchStateSucceeded, terminal: true},
{name: "failed", state: BatchStateFailed, terminal: true},
{name: "cancelled", state: BatchStateCancelled, terminal: true},
Expand All @@ -52,7 +52,7 @@ func TestIsCancellable(t *testing.T) {
assert.True(t, BatchState("future").IsCancellable())
assert.False(t, BatchStateUnknown.IsCancellable())
assert.False(t, BatchStateCreating.IsCancellable())
assert.False(t, BatchStateMerging.IsCancellable())
assert.False(t, BatchStateLanding.IsCancellable())
assert.False(t, BatchStateSucceeded.IsCancellable())
assert.False(t, BatchStateFailed.IsCancellable())
assert.False(t, BatchStateCancelled.IsCancellable())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ func TestBestFirst_OnlySpeculatingHeadsProduceCandidates(t *testing.T) {
{state: entity.BatchStateUnknown},
{state: entity.BatchStateCreated},
{state: entity.BatchStateSpeculating, want: true},
{state: entity.BatchStateMerging},
{state: entity.BatchStateLanding},
{state: entity.BatchStateSucceeded},
{state: entity.BatchStateFailed},
{state: entity.BatchStateCancelling},
Expand Down Expand Up @@ -417,13 +417,13 @@ func TestBestFirst_NeverScoresAnAbsentDependency(t *testing.T) {
assert.InDelta(t, math.Log(defaultProbability), cands[0].RankingScore, 1e-9)
}

// A merging dependency is still in progress — the merge can fail — so it stays
// A landing dependency is still in progress — the land can fail — so it stays
// an open question here like any other. Whether a path betting against it is
// worth funding is a matter of price, which is the scorer's to say, not a
// state the search hard-codes.
func TestBestFirst_MergingDependencyStaysOpen(t *testing.T) {
func TestBestFirst_LandingDependencyStaysOpen(t *testing.T) {
batches := []entity.Batch{
{ID: "q/landing", State: entity.BatchStateMerging},
{ID: "q/landing", State: entity.BatchStateLanding},
{ID: "q/H", State: entity.BatchStateSpeculating, Dependencies: []string{"q/landing"}},
}
sc := newCountingScorer(map[string]float64{"q/landing": 0.9})
Expand All @@ -432,8 +432,8 @@ func TestBestFirst_MergingDependencyStaysOpen(t *testing.T) {
require.NoError(t, err)
cands := drainAll(t, iter)

assert.Equal(t, 1, sc.calls["q/landing"], "a merging dependency is priced like any other")
require.Len(t, cands, 2, "both sides of a merge that has not landed yet")
assert.Equal(t, 1, sc.calls["q/landing"], "a landing dependency is priced like any other")
require.Len(t, cands, 2, "both sides of a land that has not completed yet")
assert.Equal(t, entity.DependencyAssumptionSucceeds, assumptionFor(cands[0].Path, "q/landing"))
assert.Equal(t, entity.DependencyAssumptionFails, assumptionFor(cands[1].Path, "q/landing"))
}
Expand Down
2 changes: 1 addition & 1 deletion submitqueue/extension/storage/mysql/batch_store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func TestBatchStore_Update(t *testing.T) {
Queue: "monorepo",
Contains: []string{"monorepo/3", "monorepo/4"},
Dependencies: []string{"monorepo/batch/1", "monorepo/batch/2"},
State: entity.BatchStateMerging,
State: entity.BatchStateLanding,
Version: oldVersion,
}
containsJSON, err := json.Marshal(batch.Contains)
Expand Down
6 changes: 3 additions & 3 deletions submitqueue/orchestrator/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ go_library(
"//submitqueue/orchestrator/controller/conclude:go_default_library",
"//submitqueue/orchestrator/controller/dependencyanalysis:go_default_library",
"//submitqueue/orchestrator/controller/dlq:go_default_library",
"//submitqueue/orchestrator/controller/merge:go_default_library",
"//submitqueue/orchestrator/controller/mergeconflictsignal:go_default_library",
"//submitqueue/orchestrator/controller/mergesignal:go_default_library",
"//submitqueue/orchestrator/controller/land:go_default_library",
"//submitqueue/orchestrator/controller/landconflictsignal:go_default_library",
"//submitqueue/orchestrator/controller/landsignal:go_default_library",
"//submitqueue/orchestrator/controller/speculate:go_default_library",
"//submitqueue/orchestrator/controller/start:go_default_library",
"//submitqueue/orchestrator/controller/validate:go_default_library",
Expand Down
Loading
Loading