feat: initial push, support for simple isolated shell workflows

This commit is contained in:
Marc 2024-08-01 18:43:18 -04:00
parent b526ee43a8
commit d2e3412cd8
Signed by: marc
GPG key ID: 048E042F22B5DC79
15 changed files with 756 additions and 2 deletions

View file

@ -1,3 +1,5 @@
# courgette # Courgette
Semi-unnamed homegrown Forgejo Action compatible runner runtime. ## Overview
Courgette is a homegrown Forgejo/Act compatible runner (or aspires to be someday).

19
go.mod Normal file
View file

@ -0,0 +1,19 @@
module runner
go 1.22.2
require (
github.com/goccy/go-yaml v1.12.0
github.com/spf13/cobra v1.8.1
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/fatih/color v1.10.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
)

38
go.sum Normal file
View file

@ -0,0 +1,38 @@
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM=
github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -0,0 +1,51 @@
package commands
import (
"gopkg.in/yaml.v3"
"io/ioutil"
)
type RunnerConfiguration struct {
Labels map[string]string `yaml:"labels"`
}
type ContainerConfiguration struct {
Driver string `yaml:"driver"`
}
type Configuration struct {
Containers ContainerConfiguration `yaml:"containers"`
Runner RunnerConfiguration `yaml:"runner"`
}
func applyConfigDefaults(config Configuration) Configuration {
defaults := Configuration{
Containers: ContainerConfiguration{
Driver: "podman",
},
}
if config.Containers.Driver == "" {
config.Containers.Driver = defaults.Containers.Driver
}
return config
}
func NewConfigFromFile(configPath string) (*Configuration, error) {
configRaw, err := ioutil.ReadFile(configPath)
if err != nil {
return nil, err
}
var config Configuration
if yamlError := yaml.Unmarshal(configRaw, &config); yamlError != nil {
return nil, yamlError
}
config = applyConfigDefaults(config)
return &config, nil
}

View file

@ -0,0 +1,54 @@
package commands
import (
"errors"
"log"
runner "runner/internal/runner"
workflow "runner/internal/workflow"
)
func ExecuteWorkflow(configurationPath string, workflowFile string) error {
configuration, err := NewConfigFromFile(configurationPath)
if err != nil {
return err
}
driver, err := runner.NewDriver(configuration.Containers.Driver)
if err != nil {
return err
}
runnerInstance := runner.NewRunner(
driver,
configuration.Runner.Labels,
)
workflow, err := workflow.FromYamlFile(workflowFile)
if err != nil {
log.Fatalf("%#v", err)
}
validationErrors := workflow.Validate()
if len(validationErrors) > 0 {
for _, err := range validationErrors {
log.Printf("Validation error:: %#v", err)
}
return errors.New("Jobs encountered errors.")
}
jobErrors := runnerInstance.Execute(*workflow)
if len(jobErrors) > 0 {
for job, err := range jobErrors {
log.Printf("Job \"%s\": %#v", job, err)
}
return errors.New("Jobs encountered errors.")
}
return nil
}

View file

@ -0,0 +1,27 @@
package commands
import (
"errors"
"log"
workflow "runner/internal/workflow"
)
func ValidateWorkflow(configurationPath string, workflowPath string) error {
workflow, err := workflow.FromYamlFile(workflowPath)
if err != nil {
log.Fatalf("%#v", err)
}
validationErrors := workflow.Validate()
if len(validationErrors) > 0 {
for _, err := range validationErrors {
log.Printf("Validation error:: %#v", err)
}
return errors.New("Jobs encountered errors.")
}
return nil
}

20
internal/runner/driver.go Normal file
View file

@ -0,0 +1,20 @@
package runner
import (
"errors"
)
type ContainerDriver interface {
Pull(string) error
Start(string, string) error
Stop(string) error
Exec(string, string) error
}
func NewDriver(driverType string) (ContainerDriver, error) {
if driverType == "podman" {
return &PodmanDriver{}, nil
}
return nil, errors.New("Unrecognized driver type.")
}

View file

@ -0,0 +1,55 @@
// Podman driver
//
// Abstracts interactions with Podman commands via the ContainerDriver interface.
package runner
import (
"os"
"os/exec"
)
type PodmanDriver struct{}
func (d PodmanDriver) Pull(uri string) error {
cmd := exec.Command("podman", "pull", uri)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return err
}
return nil
}
func (d PodmanDriver) Start(uri string, containerName string) error {
cmd := exec.Command("podman", "run", "-td", "--name", containerName, uri)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return err
}
return nil
}
func (d PodmanDriver) Stop(containerName string) error {
cmd := exec.Command("podman", "rm", "-f", containerName)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return err
}
return nil
}
func (d PodmanDriver) Exec(containerId string, command string) error {
cmd := exec.Command("podman", "exec", containerId, "bash", "-c", command)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}

102
internal/runner/runner.go Normal file
View file

@ -0,0 +1,102 @@
package runner
import (
"fmt"
"log"
workflow "runner/internal/workflow"
)
type Runner struct {
Labels map[string]string
Driver ContainerDriver
Runs int
}
func NewRunner(driver ContainerDriver, labels map[string]string) Runner {
return Runner{
Driver: driver,
Labels: labels,
}
}
func (r *Runner) GetContainerName() string {
return fmt.Sprintf("runner-%d", r.Runs)
}
func (r *Runner) GetImageUriByLabel(label string) string {
uri, exists := r.Labels[label]
if exists {
return uri
}
return "debian:latest"
}
// Executes a workflow using the runner.
//
// This is the high-level call that will set up the container
// that the jobs will be executed in, run the jobs's steps and
// tear down the container once no longer useful.
func (r *Runner) Execute(workflow workflow.Workflow) map[string]error {
log.Printf("Executing workflow: %s", workflow.SourcePath)
errors := map[string]error{}
for jobLabel, job := range workflow.Jobs {
runnerImage := r.GetImageUriByLabel(job.RunsOn)
containerName := r.GetContainerName()
log.Printf("Using image %s (label: %s)", runnerImage, job.RunsOn)
if pullError := r.PullContainer(runnerImage); pullError != nil {
errors[jobLabel] = pullError
continue
}
if runError := r.RunJobInContainer(runnerImage, containerName, job); runError != nil {
errors[jobLabel] = runError
continue
}
}
return errors
}
// Pulls the container from the registry provided its image uri.
func (r *Runner) PullContainer(uri string) error {
return r.Driver.Pull(uri)
}
// Starts a container from the given image uri with the given name.
func (r *Runner) StartContainer(uri string, containerName string) error {
return r.Driver.Start(uri, containerName)
}
// Executes a command within the given container.
func (r *Runner) RunCommandInContainer(containerId string, command string) error {
return r.Driver.Exec(containerId, command)
}
// Executes a job within a container.
//
// The container is started before the job steps are run and cleaned up after.
func (r *Runner) RunJobInContainer(imageUri string, containerId string, job workflow.Job) error {
r.StartContainer(imageUri, containerId)
defer r.StopContainer(containerId)
log.Printf("Started %s", containerId)
for _, step := range job.Steps {
log.Printf("Run: %s", step.Run)
r.RunCommandInContainer(containerId, step.Run)
}
log.Printf("Cleaning up %s", containerId)
return nil
}
// Stops the given container.
func (r *Runner) StopContainer(containerName string) {
r.Driver.Stop(containerName)
}

View file

@ -0,0 +1,94 @@
package runner
import (
"slices"
"testing"
)
type MockCall struct {
fname string
args []string
}
type MockDriver struct {
calls []MockCall
}
func (d *MockDriver) Pull(uri string) error {
d.calls = append(d.calls, MockCall{fname: "Pull", args: []string{uri}})
return nil
}
func (d *MockDriver) Start(uri string, containerName string) error {
d.calls = append(d.calls, MockCall{fname: "Start", args: []string{uri, containerName}})
return nil
}
func (d *MockDriver) Stop(uri string) error {
d.calls = append(d.calls, MockCall{fname: "Stop", args: []string{uri}})
return nil
}
func (d *MockDriver) Exec(containerName string, command string) error {
d.calls = append(d.calls, MockCall{fname: "Exec", args: []string{containerName, command}})
return nil
}
func TestRunnerPullContainerCallsDriverPull(t *testing.T) {
mockDriver := MockDriver{}
runner := Runner{
Driver: &mockDriver,
}
runner.PullContainer("test-image")
if len(mockDriver.calls) != 1 {
t.Error("Expected pull to have been called.")
}
expectedArgs := []string{"test-image"}
actualArgs := mockDriver.calls[0].args
if !slices.Equal(actualArgs, expectedArgs) {
t.Errorf("Expected call args to be %#v, got %#v instead.", expectedArgs, actualArgs)
}
}
func TestRunnerStartContainerCallsDriverStart(t *testing.T) {
mockDriver := MockDriver{}
runner := Runner{
Driver: &mockDriver,
}
runner.StartContainer("test-image", "container")
if len(mockDriver.calls) != 1 {
t.Error("Expected start to have been called.")
}
expectedArgs := []string{"test-image", "container"}
actualArgs := mockDriver.calls[0].args
if !slices.Equal(actualArgs, expectedArgs) {
t.Errorf("Expected call args to be %#v, got %#v instead.", expectedArgs, actualArgs)
}
}
func TestRunnerStopContainerCallsDriverStop(t *testing.T) {
mockDriver := MockDriver{}
runner := Runner{
Driver: &mockDriver,
}
runner.StopContainer("container")
if len(mockDriver.calls) != 1 {
t.Error("Expected stop to have been called.")
}
expectedArgs := []string{"container"}
actualArgs := mockDriver.calls[0].args
if !slices.Equal(actualArgs, expectedArgs) {
t.Errorf("Expected call args to be %#v, got %#v instead.", expectedArgs, actualArgs)
}
}

View file

@ -0,0 +1,37 @@
package runner
import (
"testing"
)
func TestGetContainerNameReturnsADeterministicName(t *testing.T) {
runner := Runner{}
containerName := runner.GetContainerName()
if containerName != "runner-0" {
t.Errorf("Unexpected container name: %s", containerName)
}
}
func TestGetImageUriByLabelReturnsAUriIfLabelled(t *testing.T) {
labels := map[string]string{
"test-label": "some-image",
}
runner := Runner{
Labels: labels,
}
imageUri := runner.GetImageUriByLabel("test-label")
if uri, _ := labels["test-label"]; imageUri != uri {
t.Errorf("Expected uri %s, got %s instead.", uri, imageUri)
}
}
func TestGetImageUriByLabelReturnsTheDefaultUriIfLabelUnknown(t *testing.T) {
labels := map[string]string{}
runner := Runner{
Labels: labels,
}
imageUri := runner.GetImageUriByLabel("test-label")
if imageUri != "debian:latest" {
t.Errorf("Expected default uri, got %s instead.", imageUri)
}
}

View file

@ -0,0 +1,42 @@
package workflow
import (
"gopkg.in/yaml.v3"
"io/ioutil"
)
// Builds a Workflow from serialized yaml provided as bytes.
//
// If the bytes cannot be unmarshalled into the Workflow, an error
// is returned along with a nil Workflow pointer.
func FromYamlBytes(raw []byte) (*Workflow, error) {
var workflow Workflow
if yamlError := yaml.Unmarshal(raw, &workflow); yamlError != nil {
return nil, yamlError
}
return &workflow, nil
}
// Builds a Workflow from the contents of the given file.
//
// If the file cannot be read, or its content cannot be parsed, an
// error is returned along with a nil Workflow pointer.
func FromYamlFile(workflowPath string) (*Workflow, error) {
workflowRaw, err := ioutil.ReadFile(workflowPath)
if err != nil {
return nil, err
}
workflow, err := FromYamlBytes(workflowRaw)
if err != nil {
return nil, err
}
workflow.SourcePath = workflowPath
return workflow, err
}

View file

@ -0,0 +1,100 @@
package workflow
import (
"os"
"path/filepath"
"testing"
)
func TestFromYamlBytesCreatesWorkflow(t *testing.T) {
sample := `
jobs:
jobA:
runs-on: default
steps:
- run: echo "test"
`
workflow, err := FromYamlBytes([]byte(sample))
if err != nil {
t.Errorf("Expected no errors, got %#v", err)
}
if workflow == nil {
t.Errorf("Expected workflow struct, got nil.")
}
}
func TestFromYamlBytesReturnsErrorIfErrorParsing(t *testing.T) {
sample := `not-yaml!`
workflow, err := FromYamlBytes([]byte(sample))
if err == nil {
t.Error("Expected error errors, got nil")
}
if workflow != nil {
t.Errorf("Expected nil workflow, got %#v.", workflow)
}
}
func TestFromYamlFileCreatesWorkflow(t *testing.T) {
sample := `
jobs:
jobA:
runs-on: default
steps:
- run: echo "test"
`
workflowPath := filepath.Join(t.TempDir(), "workflow.yml")
os.WriteFile(workflowPath, []byte(sample), 0755)
workflow, err := FromYamlFile(workflowPath)
if err != nil {
t.Errorf("Expected no errors, got %#v", err)
}
if workflow == nil {
t.Errorf("Expected workflow struct, got nil.")
}
}
func TestFromYamlFileReturnsErrorIfErrorParsing(t *testing.T) {
sample := `not-yaml!`
workflowPath := filepath.Join(t.TempDir(), "workflow.yml")
os.WriteFile(workflowPath, []byte(sample), 0755)
workflow, err := FromYamlFile(workflowPath)
if err == nil {
t.Error("Expected errors, got nil.")
}
if workflow != nil {
t.Errorf("Expected nil workflow, got %#v.", workflow)
}
}
func TestFromYamlFileSetsSourcePathOnWorkflow(t *testing.T) {
sample := `
jobs:
jobA:
runs-on: default
steps:
- run: echo "test"
`
workflowPath := filepath.Join(t.TempDir(), "workflow.yml")
os.WriteFile(workflowPath, []byte(sample), 0755)
workflow, _ := FromYamlFile(workflowPath)
if workflow.SourcePath != workflowPath {
t.Errorf("Expected workflow file path to be %s, got %s instead.", workflowPath, workflow.SourcePath)
}
}

View file

@ -0,0 +1,61 @@
package workflow
import (
"errors"
)
type Job struct {
RunsOn string `yaml:"runs-on"`
Steps []Step `yaml:"steps"`
}
func (j Job) Validate() []error {
validationErrors := []error{}
if j.RunsOn == "" {
validationErrors = append(validationErrors, errors.New("Missing \"runs-on\" field on job."))
}
if len(j.Steps) == 0 {
validationErrors = append(validationErrors, errors.New("Missing \"steps\" field on job."))
}
for _, step := range j.Steps {
validationErrors = append(validationErrors, step.Validate()...)
}
return validationErrors
}
type Step struct {
Run string `yaml:"run"`
}
func (s Step) Validate() []error {
validationErrors := []error{}
if s.Run == "" {
validationErrors = append(validationErrors, errors.New("Missing \"run\" field on step."))
}
return validationErrors
}
type Workflow struct {
SourcePath string
Jobs map[string]Job `yaml:"jobs"`
}
func (w Workflow) Validate() []error {
validationErrors := []error{}
if len(w.Jobs) == 0 {
validationErrors = append(validationErrors, errors.New("Missing \"jobs\" field on workflow."))
}
for _, job := range w.Jobs {
validationErrors = append(validationErrors, job.Validate()...)
}
return validationErrors
}

52
main.go Normal file
View file

@ -0,0 +1,52 @@
package main
import (
"github.com/spf13/cobra"
"os"
commands "runner/internal/commands"
)
var cli = &cobra.Command{
Use: "runner",
}
var execute = &cobra.Command{
Use: "execute [workflow-file]",
Short: "Executes the provided workflow.",
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
configPath, _ := cmd.Flags().GetString("config")
if err := commands.ExecuteWorkflow(configPath, args[0]); err != nil {
os.Exit(1)
}
},
}
var validate = &cobra.Command{
Use: "validate [workflow-file]",
Short: "Validates the structure of the provided workflow.",
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
configPath, _ := cmd.Flags().GetString("config")
if err := commands.ValidateWorkflow(configPath, args[0]); err != nil {
os.Exit(1)
}
},
}
var knownCommands = []*cobra.Command{
execute,
validate,
}
func main() {
cli.PersistentFlags().StringP("config", "c", "./config.yml", "Declarative runner configuration.")
for _, command := range knownCommands {
cli.AddCommand(command)
}
cli.Execute()
}