From cf26a13eebbe0f52048739427ccb863b8adc42cd Mon Sep 17 00:00:00 2001 From: Marc Cataford Date: Thu, 1 Aug 2024 22:48:54 -0400 Subject: [PATCH] feat: track tasks and individual jobs in runner --- .gitignore | 1 + go.mod | 7 +-- go.sum | 28 +---------- internal/commands/execute_workflow.go | 9 ++-- internal/runner/runner.go | 71 +++++++++++++++++---------- internal/runner/runner_test.go | 4 +- internal/runner/task.go | 67 +++++++++++++++++++++++++ main.go | 3 ++ 8 files changed, 124 insertions(+), 66 deletions(-) create mode 100644 .gitignore create mode 100644 internal/runner/task.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1fb9ef5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tmp/* diff --git a/go.mod b/go.mod index 57d6d8d..103dfe1 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,12 @@ module runner go 1.22.2 require ( - github.com/goccy/go-yaml v1.12.0 + github.com/gofrs/uuid v4.4.0+incompatible github.com/spf13/cobra v1.8.1 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/fatih/color v1.10.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) diff --git a/go.sum b/go.sum index 810e41f..08446c9 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,13 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM= -github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/commands/execute_workflow.go b/internal/commands/execute_workflow.go index 5e87637..01cfce1 100644 --- a/internal/commands/execute_workflow.go +++ b/internal/commands/execute_workflow.go @@ -40,14 +40,11 @@ func ExecuteWorkflow(configurationPath string, workflowFile string) error { return errors.New("Jobs encountered errors.") } - jobErrors := runnerInstance.Execute(*workflow) - if len(jobErrors) > 0 { - for job, err := range jobErrors { - log.Printf("Job \"%s\": %#v", job, err) - } + taskResult := runnerInstance.Execute(*workflow) - return errors.New("Jobs encountered errors.") + for _, job := range taskResult.Context.Jobs { + log.Printf("Job %s: %s", job.Id, job.Status) } return nil diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 2ef8aa0..26609d8 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -9,6 +9,7 @@ import ( type Runner struct { Labels map[string]string Driver ContainerDriver + Tasks map[string]*Task Runs int } @@ -16,13 +17,10 @@ func NewRunner(driver ContainerDriver, labels map[string]string) Runner { return Runner{ Driver: driver, Labels: labels, + Tasks: map[string]*Task{}, } } -func (r *Runner) GetContainerName() string { - return fmt.Sprintf("runner-%d", r.Runs) -} - func (r *Runner) GetImageUriByLabel(label string) string { uri, exists := r.Labels[label] @@ -33,49 +31,55 @@ func (r *Runner) GetImageUriByLabel(label string) string { return "debian:latest" } +func (r *Runner) GetContainerName(jobId string) string { + return fmt.Sprintf("runner-%s", jobId) +} + +func (r *Runner) AddTask() string { + task := NewTask() + r.Tasks[task.Id] = &task + + return task.Id +} + +func (r *Runner) GetTask(taskId string) *Task { + task, _ := r.Tasks[taskId] + + return task +} + // Executes a workflow using the runner. // // This is the high-level call that will set up the container // that the jobs will be executed in, run the jobs's steps and // tear down the container once no longer useful. -func (r *Runner) Execute(workflow workflow.Workflow) map[string]error { +func (r *Runner) Execute(workflow workflow.Workflow) Task { log.Printf("Executing workflow: %s", workflow.SourcePath) + task := r.GetTask(r.AddTask()) - errors := map[string]error{} + for _, job := range workflow.Jobs { + jobContext := task.GetJobContext(task.AddJob()) + jobContext.SetStatus("started") - for jobLabel, job := range workflow.Jobs { runnerImage := r.GetImageUriByLabel(job.RunsOn) - containerName := r.GetContainerName() + containerName := r.GetContainerName(jobContext.Id) log.Printf("Using image %s (label: %s)", runnerImage, job.RunsOn) if pullError := r.PullContainer(runnerImage); pullError != nil { - errors[jobLabel] = pullError + jobContext.SetStatus("failed").SetError(pullError) continue } if runError := r.RunJobInContainer(runnerImage, containerName, job); runError != nil { - errors[jobLabel] = runError + jobContext.SetStatus("failed").SetError(runError) continue } + + jobContext.SetStatus("success") } - return errors -} - -// Pulls the container from the registry provided its image uri. -func (r *Runner) PullContainer(uri string) error { - return r.Driver.Pull(uri) -} - -// Starts a container from the given image uri with the given name. -func (r *Runner) StartContainer(uri string, containerName string) error { - return r.Driver.Start(uri, containerName) -} - -// Executes a command within the given container. -func (r *Runner) RunCommandInContainer(containerId string, command string) error { - return r.Driver.Exec(containerId, command) + return *task } // Executes a job within a container. @@ -96,6 +100,21 @@ func (r *Runner) RunJobInContainer(imageUri string, containerId string, job work return nil } +// Pulls the container from the registry provided its image uri. +func (r *Runner) PullContainer(uri string) error { + return r.Driver.Pull(uri) +} + +// Starts a container from the given image uri with the given name. +func (r *Runner) StartContainer(uri string, containerName string) error { + return r.Driver.Start(uri, containerName) +} + +// Executes a command within the given container. +func (r *Runner) RunCommandInContainer(containerId string, command string) error { + return r.Driver.Exec(containerId, command) +} + // Stops the given container. func (r *Runner) StopContainer(containerName string) { r.Driver.Stop(containerName) diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 5da4a9d..283eb41 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -6,8 +6,8 @@ import ( func TestGetContainerNameReturnsADeterministicName(t *testing.T) { runner := Runner{} - containerName := runner.GetContainerName() - if containerName != "runner-0" { + containerName := runner.GetContainerName("testid") + if containerName != "runner-testid" { t.Errorf("Unexpected container name: %s", containerName) } } diff --git a/internal/runner/task.go b/internal/runner/task.go new file mode 100644 index 0000000..fbfd51a --- /dev/null +++ b/internal/runner/task.go @@ -0,0 +1,67 @@ +// Task tracking +// +// Task and its associated structs implements a task tracker used to record the +// health and outcomes of individual jobs that the runner executes. +package runner + +import ( + "github.com/gofrs/uuid" +) + +type JobContext struct { + Id string + Status string + Error error +} + +type TaskContext struct { + Jobs map[string]*JobContext +} + +type Task struct { + Id string + Context TaskContext +} + +func NewTask() Task { + return Task{ + Id: uuid.Must(uuid.NewV1()).String(), + Context: TaskContext{ + Jobs: map[string]*JobContext{}, + }, + } +} + +func NewJobContext() JobContext { + return JobContext{ + Id: uuid.Must(uuid.NewV1()).String(), + } +} + +func (c *JobContext) SetStatus(newStatus string) *JobContext { + c.Status = newStatus + + return c +} + +func (c *JobContext) SetError(err error) *JobContext { + c.Error = err + return c +} + +func (t *Task) AddJob() string { + jobContext := NewJobContext() + t.Context.Jobs[jobContext.Id] = &jobContext + + return jobContext.Id +} + +func (t *Task) GetJobContext(jobId string) *JobContext { + ctx, exists := t.Context.Jobs[jobId] + + if exists { + return ctx + } + + return nil +} diff --git a/main.go b/main.go index da4f399..11ebe6a 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "github.com/spf13/cobra" + "log" "os" commands "runner/internal/commands" ) @@ -18,6 +19,7 @@ var execute = &cobra.Command{ configPath, _ := cmd.Flags().GetString("config") if err := commands.ExecuteWorkflow(configPath, args[0]); err != nil { + log.Printf("Failure: %s", err) os.Exit(1) } }, @@ -31,6 +33,7 @@ var validate = &cobra.Command{ configPath, _ := cmd.Flags().GetString("config") if err := commands.ValidateWorkflow(configPath, args[0]); err != nil { + log.Printf("Failure: %s", err) os.Exit(1) } },