a few improvements from formalization

checks/captures.go

@@ -0,0 +1,91 @@
+package checks
+
+import (
+	"fmt"
+	"slices"
+
+	api "github.com/bootdotdev/bootdev/client"
+)
+
+func captureNames(step api.CLIStep) []string {
+	var names []string
+	if step.CLICommand != nil {
+		for _, capture := range step.CLICommand.StdoutVariables {
+			names = append(names, capture.Name)
+		}
+	}
+	if step.HTTPRequest != nil {
+		for _, capture := range step.HTTPRequest.ResponseVariables {
+			names = append(names, capture.Name)
+		}
+		for _, capture := range step.HTTPRequest.ResponseHeaderVariables {
+			names = append(names, capture.Name)
+		}
+	}
+	return names
+}
+
+// Missing and invalidated workflow bindings both block execution. Keeping the
+// namespace separate from values prevents failed or forward captures being
+// mistaken for shell variables.
+func missingDependencies(step api.CLIStep, namespace map[string]bool, values map[string]string) []string {
+	var names []string
+	add := func(text string) {
+		for _, name := range InterpolationNames(text) {
+			if _, available := values[name]; namespace[name] && !available {
+				names = append(names, name)
+			}
+		}
+	}
+	var visit func(any)
+	visit = func(value any) {
+		switch value := value.(type) {
+		case string:
+			add(value)
+		case []any:
+			for _, item := range value {
+				visit(item)
+			}
+		case map[string]any:
+			for _, item := range value {
+				visit(item)
+			}
+		}
+	}
+	if step.CLICommand != nil {
+		add(step.CLICommand.Command)
+	}
+	if step.HTTPRequest != nil {
+		request := step.HTTPRequest.Request
+		add(request.FullURL)
+		for _, value := range request.Headers {
+			add(value)
+		}
+		if request.BodyJSON != nil {
+			visit(request.BodyJSON)
+		} else {
+			for _, value := range request.BodyForm {
+				add(value)
+			}
+		}
+	}
+	slices.Sort(names)
+	return slices.Compact(names)
+}
+
+// Finalize even on early execution errors so previous output values cannot leak
+// into later steps. Extraction writes only to captured, never to values.
+func publishCaptures(names []string, values, captured map[string]string, captureErr string) string {
+	for _, name := range names {
+		delete(values, name)
+		if _, found := captured[name]; !found && captureErr == "" {
+			captureErr = fmt.Sprintf("missing value for variable '%s'", name)
+		}
+	}
+	if captureErr == "" {
+		for _, name := range names {
+			values[name] = captured[name]
+		}
+	}
+	return captureErr
+}

checks/cli.go

@@ -85,6 +85,11 @@ func runCLICommandWithOutputLimit(
 	maxOutputBytesPerStream int,
 	shell commandShell,
 ) (result api.CLICommandResult) {
+	captured := make(map[string]string)
+	defer func() {
+		result.Err = publishCaptures(captureNames(api.CLIStep{CLICommand: &command}), variables, captured, result.Err)
+		result.Variables = maps.Clone(variables)
+	}()
 	finalCommand := InterpolateVariables(command.Command, variables)
 	result.FinalCommand = finalCommand
 	result.Command = command
@@ -103,21 +108,19 @@ func runCLICommandWithOutputLimit(
 	} else if err != nil {
 		result.ExitCode = -2
 	}
+	if result.ExitCode < 0 && err != nil {
+		result.Err = err.Error()
+	}
 
 	result.Stdout = strings.TrimRight(stdout.String(), " \n\t\r")
 	result.Stderr = strings.TrimRight(stderr.String(), " \n\t\r")
 	if command.StdoutFilterTmdl != nil {
 		result.Stdout = ExtractTmdlBlock(result.Stdout, *command.StdoutFilterTmdl)
 	}
 
-	if stdout.truncated || stderr.truncated {
-		result.Err = fmt.Sprintf("command output exceeded the %d-byte per-stream limit", maxOutputBytesPerStream)
-		result.ExitCode = -2
-	} else if err := parseStdoutVariables(result.Stdout, command.StdoutVariables, variables); err != nil {
+	if err := parseStdoutVariables(result.Stdout, command.StdoutVariables, captured); err != nil {
 		result.Err = err.Error()
 	}
-	result.Variables = maps.Clone(variables)
-
 	return result
 }
 

checks/cli_test.go

@@ -2,43 +2,75 @@ package checks
 
 import (
 	"runtime"
-	"strings"
 	"testing"
 
 	api "github.com/bootdotdev/bootdev/client"
 )
 
 func TestRunCLICommandCapsOutput(t *testing.T) {
-	command := `printf 'abcdefgh'`
-	if runtime.GOOS == "windows" {
-		command = `[Console]::Out.Write('abcdefgh')`
-	}
-
-	variables := map[string]string{}
-	result := runCLICommandWithOutputLimit(
-		api.CLIStepCLICommand{
-			Command: command,
-			StdoutVariables: []api.CLICommandStdoutVariable{{
-				Name:  "partial",
-				Regex: `(abcd)`,
-			}},
+	for _, tc := range []struct {
+		name, command, windowsCommand, stdout, stderr, pattern, value string
+		exitCode                                                      int
+		wantFailure                                                   bool
+	}{
+		{
+			name:           "stdout overflow",
+			command:        `printf 'abcdefgh'`,
+			windowsCommand: `[Console]::Out.Write('abcdefgh')`,
+			stdout:         "abcd", pattern: `^(abcd)$`, value: "abcd",
 		},
-		variables,
-		4,
-		defaultShell(),
-	)
-
-	if !strings.Contains(result.Err, "per-stream limit") {
-		t.Fatalf("command error = %q, want per-stream output limit error", result.Err)
-	}
-	if result.ExitCode >= 0 {
-		t.Fatalf("exit code = %d, want internal failure", result.ExitCode)
-	}
-	if result.Stdout != "abcd" {
-		t.Fatalf("stdout = %q, want capped output %q", result.Stdout, "abcd")
-	}
-	if _, ok := variables["partial"]; ok {
-		t.Fatal("truncated output unexpectedly populated a stdout variable")
+		{
+			name:           "stderr overflow",
+			command:        `printf 'ok'; printf 'abcdefgh' >&2`,
+			windowsCommand: `[Console]::Out.Write('ok'); [Console]::Error.Write('abcdefgh')`,
+			stdout:         "ok", stderr: "abcd", pattern: `^(ok)$`, value: "ok",
+		},
+		{
+			name:           "nonzero exit",
+			command:        `printf 'abcdefgh'; exit 7`,
+			windowsCommand: `[Console]::Out.Write('abcdefgh'); exit 7`,
+			stdout:         "abcd", pattern: `^(abcd)$`, value: "abcd", exitCode: 7,
+		},
+		{
+			name:           "capture beyond limit",
+			command:        `printf 'abcdefgh'`,
+			windowsCommand: `[Console]::Out.Write('abcdefgh')`,
+			stdout:         "abcd", pattern: `(efgh)`, wantFailure: true,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			command := tc.command
+			if runtime.GOOS == "windows" {
+				command = tc.windowsCommand
+			}
+			step := api.CLIStepCLICommand{
+				Command:         command,
+				StdoutVariables: []api.CLICommandStdoutVariable{{Name: "token", Regex: tc.pattern}},
+			}
+			variables := map[string]string{"token": "old"}
+			result := runCLICommandWithOutputLimit(step, variables, 4, defaultShell())
+			if result.ExitCode != tc.exitCode {
+				t.Fatalf("exit code = %d, want %d", result.ExitCode, tc.exitCode)
+			}
+			if result.Stdout != tc.stdout || result.Stderr != tc.stderr {
+				t.Fatalf("stdout/stderr = %q/%q, want %q/%q", result.Stdout, result.Stderr, tc.stdout, tc.stderr)
+			}
+			if tc.wantFailure {
+				if result.Err == "" {
+					t.Fatal("missing capture should fail")
+				}
+				if _, found := variables["token"]; found {
+					t.Fatal("failed capture retained the old token")
+				}
+				return
+			}
+			if result.Err != "" {
+				t.Fatalf("unexpected command error: %s", result.Err)
+			}
+			if variables["token"] != tc.value {
+				t.Fatalf("capture = %q, want %q", variables["token"], tc.value)
+			}
+		})
 	}
 }
 

checks/http.go

@@ -28,9 +28,20 @@ func runHTTPRequest(
 ) (
 	result api.HTTPRequestResult,
 ) {
-	finalBaseURL := strings.TrimSuffix(baseURL, "/")
-	interpolatedURL := InterpolateVariables(requestStep.Request.FullURL, variables)
-	completeURL := strings.Replace(interpolatedURL, api.BaseURLPlaceholder, finalBaseURL, 1)
+	captured := make(map[string]string)
+	defer func() {
+		result.Err = publishCaptures(captureNames(api.CLIStep{HTTPRequest: &requestStep}), variables, captured, result.Err)
+		result.Variables = maps.Clone(variables)
+		result.Request = requestStep
+	}()
+	requestVariables := maps.Clone(variables)
+	if requestVariables == nil {
+		requestVariables = make(map[string]string)
+	}
+	if _, ok := requestVariables["baseURL"]; !ok {
+		requestVariables["baseURL"] = strings.TrimSuffix(baseURL, "/")
+	}
+	completeURL := InterpolateVariables(requestStep.Request.FullURL, requestVariables)
 
 	var requestBody io.Reader
 	var contentType string
@@ -103,21 +114,20 @@ func runHTTPRequest(
 	}
 
 	bodyString := truncateAndStringifyBody(body)
-	if err := parseVariables([]byte(bodyString), requestStep.ResponseVariables, variables); err != nil {
-		return api.HTTPRequestResult{Err: fmt.Sprintf("Failed to parse response variable: %s", err)}
-	}
-	if err := parseHeaderVariables(headers, requestStep.ResponseHeaderVariables, variables); err != nil {
-		return api.HTTPRequestResult{Err: fmt.Sprintf("Failed to parse response header variable: %s", err)}
-	}
-
 	result = api.HTTPRequestResult{
 		StatusCode:       resp.StatusCode,
 		ResponseHeaders:  headers,
 		ResponseTrailers: trailers,
 		BodyString:       bodyString,
-		Variables:        maps.Clone(variables),
-		Request:          requestStep,
 	}
+	if err := parseVariables([]byte(bodyString), requestStep.ResponseVariables, captured); err != nil {
+		result.Err = fmt.Sprintf("Failed to parse response variable: %s", err)
+		return result
+	}
+	if err := parseHeaderVariables(headers, requestStep.ResponseHeaderVariables, captured); err != nil {
+		result.Err = fmt.Sprintf("Failed to parse response header variable: %s", err)
+	}
+
 	return result
 }
 

checks/local.go

@@ -36,6 +36,23 @@ func EvaluateCLIResults(cliData api.CLIData, results []api.CLIStepResult) *api.S
 	for i, step := range cliData.Steps {
 		actual := results[i]
 
+		variants := 0
+		if actual.CLICommandResult != nil {
+			variants++
+		}
+		if actual.HTTPRequestResult != nil {
+			variants++
+		}
+		if actual.DependencyFailure != nil {
+			variants++
+		}
+		if variants != 1 {
+			return localFailure(i, -1, "invalid step result: expected exactly one result variant")
+		}
+		if actual.DependencyFailure != nil {
+			return localFailure(i, -1, "step skipped: unavailable variables: "+strings.Join(actual.DependencyFailure.Names, ", "))
+		}
+
 		if step.CLICommand != nil && actual.CLICommandResult != nil {
 			verificationErr := evaluateCLICommandTests(i, *step.CLICommand, *actual.CLICommandResult)
 			if verificationErr != nil {
@@ -120,6 +137,19 @@ func evaluateCLICommandTests(stepIndex int, expect api.CLIStepCLICommand, actual
 		}
 	}
 
+	for _, expectedVar := range expect.StdoutVariables {
+		expectedValue, found, err := regexCapture(expectedVar.Regex, actual.Stdout)
+		if err != nil {
+			return localFailure(stepIndex, len(expect.Tests)+1, fmt.Sprintf("invalid regex for stdout variable '%s'", expectedVar.Name))
+		}
+		if !found {
+			return localFailure(stepIndex, len(expect.Tests)+1, fmt.Sprintf("missing value for variable '%s'", expectedVar.Name))
+		}
+		if !capturedVariableMatches(actual.Variables, expectedVar.Name, expectedValue) {
+			return localFailure(stepIndex, len(expect.Tests)+1, fmt.Sprintf("captured variable '%s' did not match expected stdout value", expectedVar.Name))
+		}
+	}
+
 	return nil
 }
 

checks/runner.go

@@ -2,6 +2,7 @@ package checks
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"strings"
 	"time"
@@ -22,6 +23,18 @@ func CLIChecks(cliData api.CLIData, options RunOptions, send func(tea.Msg)) ([]a
 	if err := validateCLIAssertions(cliData); err != nil {
 		return nil, err
 	}
+	namespace := map[string]bool{"baseURL": true}
+	for i, step := range cliData.Steps {
+		seen := make(map[string]bool)
+		for _, name := range captureNames(step) {
+			if name == "" || name == "baseURL" || seen[name] {
+				return nil, fmt.Errorf("step %d: invalid or duplicate capture name %q", i+1, name)
+			}
+			seen[name] = true
+			namespace[name] = true
+		}
+	}
+
 	shell, err := resolveShell(options.Shell)
 	if err != nil {
 		return nil, err
@@ -45,6 +58,15 @@ func CLIChecks(cliData api.CLIData, options RunOptions, send func(tea.Msg)) ([]a
 	}
 
 	for i, step := range cliData.Steps {
+		if missing := missingDependencies(step, namespace, variables); len(missing) > 0 {
+			for _, name := range captureNames(step) {
+				delete(variables, name)
+			}
+			results[i].DependencyFailure = &api.DependencyFailure{Names: missing}
+			send(messages.StartStepMsg{Description: step.Description, NoPenaltyOnFail: step.NoPenaltyOnFail})
+			send(messages.ResolveStepMsg{Index: i, Result: &results[i]})
+			continue
+		}
 		switch {
 		case step.CLICommand != nil:
 			send(messages.StartStepMsg{
@@ -62,8 +84,7 @@ func CLIChecks(cliData api.CLIData, options RunOptions, send func(tea.Msg)) ([]a
 			handleSleep(step.CLICommand.SleepAfterMs, send)
 
 		case step.HTTPRequest != nil:
-			fullURL := strings.Replace(step.HTTPRequest.Request.FullURL, api.BaseURLPlaceholder, baseURL, 1)
-			interpolatedURL := InterpolateVariables(fullURL, variables)
+			interpolatedURL := InterpolateVariables(step.HTTPRequest.Request.FullURL, variables)
 
 			send(messages.StartStepMsg{
 				Description:     step.Description,