marc/courgette
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: support for cascading workflow/job/step env

Browse source
This commit is contained in:
Marc 2024-08-20 23:35:49 -04:00
parent 3b70f47676
commit f6a93c0b55
Signed by: marc
GPG key ID: 048E042F22B5DC79
9 changed files with 199 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
WORKFLOW_SUPPORT.md
View file

@ -9,7 +9,7 @@ syntax](https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for
- [ ] run-name
- [ ] on
- [ ] permissions
- [ ] env
- [x] env
- [ ] defaults
- [x] jobs
- [x] jobs.<job_id>.name
@ -20,7 +20,7 @@ syntax](https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for
- [ ] jobs.<job_id>.environment
- [ ] jobs.<job_id>.concurrency
- [ ] jobs.<job_id>.outputs
- [ ] jobs.<job_id>.env
- [x] jobs.<job_id>.env
- [x] jobs.<job_id>.defaults
- [ ] jobs.<job_id>.run.shell
- [x] jobs.<job_id>.run.working-directory
@ -42,7 +42,7 @@ syntax](https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for
- [x] jobs.<job_id>.steps[*].working-directory
- [ ] jobs.<job_id>.steps[*].shell
- [ ] jobs.<job_id>.steps[*].with
- [ ] jobs.<job_id>.steps[*].env
- [x] jobs.<job_id>.steps[*].env
- [X] jobs.<job_id>.steps[*].continue-on-error
- [ ] jobs.<job_id>.steps[*].timeout-minutes

2
internal/runner/driver.go
View file

@ -8,7 +8,7 @@ type ContainerDriver interface {
Pull(string) error
Start(string, string) error
Stop(string) error
Exec(containerId string, command string, cwd string) CommandResult
Exec(containerId string, command string, cwd string, env map[string]string) CommandResult
}
// Represents the outcome of a command call made by the driver.

4
internal/runner/mock_container_driver.go
View file

@ -49,12 +49,12 @@ func (d *MockDriver) Stop(uri string) error {
}
func (d *MockDriver) Exec(containerName string, command string, cwd string) CommandResult {
func (d *MockDriver) Exec(containerName string, command string, cwd string, env map[string]string) CommandResult {
if _, init := d.calls["Exec"]; !init {
d.calls["Exec"] = []MockCall{}
}
args := []string{containerName, command, cwd}
args := []string{containerName, command, cwd, fmt.Sprintf("%#v", env)}
d.calls["Exec"] = append(d.calls["Exec"], MockCall{fname: "Exec", args: args})
mockKeys := []string{

27
internal/runner/podman_driver.go
View file

@ -79,13 +79,34 @@ func (d PodmanDriver) Stop(containerName string) error {
return nil
}
func (d PodmanDriver) Exec(containerId string, command string, cwd string) CommandResult {
cmd := exec.Command("podman", "exec", "--workdir", cwd, containerId, "bash", "-c", command)
func (d PodmanDriver) Exec(containerId string, command string, cwd string, env map[string]string) CommandResult {
envArgs := []string{}
for key, value := range env {
envArgs = append(envArgs, fmt.Sprintf("-e=%s=%s", key, value))
}
commandArgs := []string{
"exec",
"--workdir",
cwd,
}
commandArgs = append(commandArgs, envArgs...)
commandArgs = append(commandArgs, containerId,
"bash",
"-c",
command,
)
cmd := exec.Command("podman", commandArgs...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
fmt.Printf("%#v", cmd)
return CommandResult{
Error: err,
ExitCode: cmd.ProcessState.ExitCode(),

8
internal/runner/runner.go
View file

@ -103,8 +103,8 @@ func (r Runner) runJob(jobContext context.Context, jobTracker *TaskTracker, jobW
//
// If the command raises an error while in the container or fails to run
// the command at all, an error is returned, otherwise nil.
func (r *Runner) RunCommandInContainer(containerId string, command string, stepCwd string) error {
result := r.Driver.Exec(containerId, command, stepCwd)
func (r *Runner) RunCommandInContainer(containerId string, command string, stepCwd string, stepEnv map[string]string) error {
result := r.Driver.Exec(containerId, command, stepCwd, stepEnv)
if result.Error != nil {
return result.Error
@ -133,12 +133,14 @@ func (r *Runner) RunJobInContainer(imageUri string, containerId string, jobConte
logger.Info("Started %s", containerId)
for stepIndex, step := range job.Steps {
stepCwd := jobContext.Value("workflow").(workflow.Workflow).GetWorkingDirectory(job.Name, stepIndex)
stepEnv := jobContext.Value("workflow").(workflow.Workflow).GetEnv(job.Name, stepIndex)
logger.Info("Run: %s", step.Run)
logger.Info("Using working directory %s", stepCwd)
var stepError error
if step.Run != "" {
stepError = r.RunCommandInContainer(containerId, step.Run, stepCwd)
stepError = r.RunCommandInContainer(containerId, step.Run, stepCwd, stepEnv)
}
if stepError != nil && !step.ContinueOnError {

11
internal/runner/runner_flow_test.go
View file

@ -5,6 +5,7 @@ import (
logger "courgette/internal/logging"
workflow "courgette/internal/workflow"
"errors"
"fmt"
"testing"
)
@ -14,13 +15,13 @@ func init() {
func TestRunnerRunCommandInContainerReturnsErrorFromDriver(t *testing.T) {
mockDriver := NewMockDriver()
mockDriver.WithMockedCall("Exec", CommandResult{ExitCode: 0, Error: errors.New("test")}, "test-container", "test-command", ".")
mockDriver.WithMockedCall("Exec", CommandResult{ExitCode: 0, Error: errors.New("test")}, "test-container", "test-command", ".", fmt.Sprintf("%#v", map[string]string{}))
runner := Runner{
Driver: &mockDriver,
}
err := runner.RunCommandInContainer("test-container", "test-command", ".")
err := runner.RunCommandInContainer("test-container", "test-command", ".", map[string]string{})
if err == nil {
t.Errorf("Expected error, got nil.")
@ -29,13 +30,13 @@ func TestRunnerRunCommandInContainerReturnsErrorFromDriver(t *testing.T) {
func TestRunnerRunCommandInContainerReturnsErrorIfCommandExitCodeNonzero(t *testing.T) {
mockDriver := NewMockDriver()
mockDriver.WithMockedCall("Exec", CommandResult{ExitCode: 1, Error: nil}, "test-container", "test-command", ".")
mockDriver.WithMockedCall("Exec", CommandResult{ExitCode: 1, Error: nil}, "test-container", "test-command", ".", fmt.Sprintf("%#v", map[string]string{}))
runner := Runner{
Driver: &mockDriver,
}
err := runner.RunCommandInContainer("test-container", "test-command", ".")
err := runner.RunCommandInContainer("test-container", "test-command", ".", map[string]string{})
if err == nil {
t.Errorf("Expected error, got nil.")
@ -44,7 +45,7 @@ func TestRunnerRunCommandInContainerReturnsErrorIfCommandExitCodeNonzero(t *test
func TestRunJobInContainerSchedulesStoppingContainers(t *testing.T) {
mockDriver := NewMockDriver()
mockDriver.WithMockedCall("Exec", CommandResult{ExitCode: 1, Error: nil}, "test-container", "test-command", ".")
mockDriver.WithMockedCall("Exec", CommandResult{ExitCode: 1, Error: nil}, "test-container", "test-command", ".", fmt.Sprintf("%#v", map[string]string{}))
runner := NewRunner(&mockDriver, map[string]string{})

9
internal/runner/runner_job_test.go
View file

@ -4,6 +4,7 @@ import (
"context"
workflow "courgette/internal/workflow"
"errors"
"fmt"
"testing"
)
@ -70,7 +71,7 @@ jobs:
workflow, _ := workflow.FromYamlBytes([]byte(workflowSample))
mockDriver := NewMockDriver()
mockDriver.WithMockedCall("Exec", CommandResult{Error: errors.New("exit 1!"), ExitCode: 1}, "testContainer", "exit 1")
mockDriver.WithMockedCall("Exec", CommandResult{Error: errors.New("exit 1!"), ExitCode: 1}, "testContainer", "exit 1", fmt.Sprintf("%#v", map[string]string{}))
runner := NewRunner(&mockDriver, map[string]string{"test": "test"})
jobContext := context.WithValue(context.Background(), "workflow", *workflow)
@ -98,7 +99,7 @@ jobs:
workflow, _ := workflow.FromYamlBytes([]byte(workflowSample))
mockDriver := NewMockDriver()
mockDriver.WithMockedCall("Exec", CommandResult{Error: errors.New("exit 1!"), ExitCode: 1}, "testContainer", "exit 1")
mockDriver.WithMockedCall("Exec", CommandResult{Error: errors.New("exit 1!"), ExitCode: 1}, "testContainer", "exit 1", fmt.Sprintf("%#v", map[string]string{}))
runner := NewRunner(&mockDriver, map[string]string{"test": "test"})
jobContext := context.WithValue(context.Background(), "workflow", *workflow)
@ -130,7 +131,7 @@ jobs:
workflow, _ := workflow.FromYamlBytes([]byte(workflowSample))
mockDriver := NewMockDriver()
mockDriver.WithMockedCall("Exec", CommandResult{Error: errors.New("exit 1!"), ExitCode: 1}, "testContainer", "exit 1", ".")
mockDriver.WithMockedCall("Exec", CommandResult{Error: errors.New("exit 1!"), ExitCode: 1}, "testContainer", "exit 1", ".", fmt.Sprintf("%#v", map[string]string{}))
runner := NewRunner(&mockDriver, map[string]string{"test": "test"})
jobContext := context.WithValue(context.Background(), "workflow", *workflow)
@ -161,7 +162,7 @@ jobs:
workflow, _ := workflow.FromYamlBytes([]byte(workflowSample))
mockDriver := NewMockDriver()
mockDriver.WithMockedCall("Exec", CommandResult{Error: errors.New("exit 1!"), ExitCode: 1}, "testContainer", "exit 1", ".")
mockDriver.WithMockedCall("Exec", CommandResult{Error: errors.New("exit 1!"), ExitCode: 1}, "testContainer", "exit 1", ".", fmt.Sprintf("%#v", map[string]string{}))
runner := NewRunner(&mockDriver, map[string]string{"test": "test"})
jobContext := context.WithValue(context.Background(), "workflow", *workflow)

20
internal/workflow/models.go
View file

@ -2,6 +2,7 @@ package workflow
import (
"errors"
"maps"
)
type Job struct {
@ -12,6 +13,7 @@ type Job struct {
Name string `yaml:"name"`
// If truthy, job-level failure does not bubble up to the workflow.
ContinueOnError bool `yaml:"continue-on-error"`
Env map[string]string `yaml:"env"`
Defaults struct {
Run struct {
WorkingDirectory string `yaml:"working-directory"`
@ -41,6 +43,7 @@ type Step struct {
Run string `yaml:"run"`
WorkingDirectory string `yaml:"working-directory"`
ContinueOnError bool `yaml:"continue-on-error"`
Env map[string]string `yaml:"env"`
}
func (s Step) Validate() []error {
@ -56,6 +59,7 @@ func (s Step) Validate() []error {
type Workflow struct {
SourcePath string
Jobs map[string]Job `yaml:"jobs"`
Env map[string]string `yaml:"env"`
}
// Returns the given workflow's job+step working directory inside the container
@ -79,6 +83,22 @@ func (w Workflow) GetWorkingDirectory(jobName string, stepIndex int) string {
return "."
}
// Returns the merged map of environment variants defined by:
// - Workflow-level "env"
// - Job-level "env"
// - Step-level "env"
// The environment is merged in order, and name collisions overwrite
// previous values such that jobs can overwrite workflows, and steps, jobs.
func (w Workflow) GetEnv(jobName string, stepIndex int) map[string]string {
finalEnv := map[string]string{}
maps.Copy(finalEnv, w.Env)
maps.Copy(finalEnv, w.Jobs[jobName].Env)
maps.Copy(finalEnv, w.Jobs[jobName].Steps[stepIndex].Env)
return finalEnv
}
func (w Workflow) Validate() []error {
validationErrors := []error{}

128
internal/workflow/models_test.go
View file

@ -1,6 +1,7 @@
package workflow
import (
"reflect"
"testing"
)
@ -78,3 +79,130 @@ jobs:
}
}
func TestWorkflowEnv(t *testing.T) {
sample := `
env:
TEST: 1
jobs:
jobA:
runs-on: default
steps:
- run: echo "test"
`
workflow, _ := FromYamlBytes([]byte(sample))
env := workflow.GetEnv("jobA", 0)
if !reflect.DeepEqual(env, map[string]string{"TEST": "1"}) {
t.Errorf("Unexpected env: %#v", env)
}
}
func TestJobEnv(t *testing.T) {
sample := `
jobs:
jobA:
env:
TEST: 1
runs-on: default
steps:
- run: echo "test"
`
workflow, _ := FromYamlBytes([]byte(sample))
env := workflow.GetEnv("jobA", 0)
if !reflect.DeepEqual(env, map[string]string{"TEST": "1"}) {
t.Errorf("Unexpected env: %#v", env)
}
}
func TestStepEnv(t *testing.T) {
sample := `
jobs:
jobA:
runs-on: default
steps:
- run: echo "test"
env:
TEST: 1
`
workflow, _ := FromYamlBytes([]byte(sample))
env := workflow.GetEnv("jobA", 0)
if !reflect.DeepEqual(env, map[string]string{"TEST": "1"}) {
t.Errorf("Unexpected env: %#v", env)
}
}
func TestJobEnvOverwritesWorkflowEnv(t *testing.T) {
sample := `
env:
TEST: 1
jobs:
jobA:
env:
TEST: 2
runs-on: default
steps:
- run: echo "test"
`
workflow, _ := FromYamlBytes([]byte(sample))
env := workflow.GetEnv("jobA", 0)
if !reflect.DeepEqual(env, map[string]string{"TEST": "2"}) {
t.Errorf("Unexpected env: %#v", env)
}
}
func TestStepEnvOverwritesWorkflowEnv(t *testing.T) {
sample := `
env:
TEST: 1
jobs:
jobA:
runs-on: default
steps:
- run: echo "test"
env:
TEST: 2
`
workflow, _ := FromYamlBytes([]byte(sample))
env := workflow.GetEnv("jobA", 0)
if !reflect.DeepEqual(env, map[string]string{"TEST": "2"}) {
t.Errorf("Unexpected env: %#v", env)
}
}
func TestStepEnvOverwritesJobEnv(t *testing.T) {
sample := `
jobs:
jobA:
env:
TEST: 1
runs-on: default
steps:
- run: echo "test"
env:
TEST: 2
`
workflow, _ := FromYamlBytes([]byte(sample))
env := workflow.GetEnv("jobA", 0)
if !reflect.DeepEqual(env, map[string]string{"TEST": "2"}) {
t.Errorf("Unexpected env: %#v", env)
}
}