From 03451535c8a9468d6dd7b7f0566de576e08addc6 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Tue, 16 Apr 2019 11:01:02 +0200 Subject: [PATCH] runservice: rework config format The current config format was thought for future extensions for reusing runtimes and job definitions adding some parameters. After a lot of thoughts this looks like a complex approach: the final result will be a sort of templating without a lot of powers. Other approach like external templating should be an alternative but I really don't think templating yaml is the way to go. A much better approach will to just use jsonnet when we need to create matrix runs and a lot of other use cases. So just make the config a simple yaml/json. User can generate their config using any preferred tool and in future we'll leverage jsonnet automated parsing and provide a lot of jsonnet based examples for most use cases. Main changes: * Runs are now an array and not a map. The run name is in the Name field * Tasks are now an array and not a map. The task name is in the Name field * Use https://github.com/ghodss/yaml so we'll use json struct tags and unmarshall functions --- .agola/config.yml | 128 +++-- go.mod | 1 + internal/config/config.go | 721 ++++++++++++++------------- internal/config/config_test.go | 420 ++++++++++------ internal/runconfig/runconfig.go | 28 +- internal/runconfig/runconfig_test.go | 297 +++++------ 6 files changed, 851 insertions(+), 744 deletions(-) diff --git a/.agola/config.yml b/.agola/config.yml index 07f4d31..88d218a 100644 --- a/.agola/config.yml +++ b/.agola/config.yml @@ -1,74 +1,60 @@ -runtimes: - go1.12: - type: pod - arch: amd64 - containers: - - image: golang:1.12-stretch - environment: - ENV01: envvalue01 - debian: - type: pod - arch: amd64 - containers: - - image: debian:stretch - dind: - type: pod - arch: amd64 - containers: - - image: docker:stable-dind - privileged: true - entrypoint: dockerd - -tasks: - build-go1.12: - runtime: go1.12 - working_dir: /go/src/github.com/sorintlab/agola - environment: - GO111MODULE: "on" - VAR01: - from_variable: var01 - steps: - - run: env - - clone: - - run: SKIP_DOCKER_TESTS=1 go test -v -count 1 ./... - - build-docker-tests-go-1.12: - runtime: go1.12 - working_dir: /go/src/github.com/sorintlab/agola - environment: - GO111MODULE: "on" - steps: - - run: env - - clone: - - run: - name: build docker tests binary - command: CGO_ENABLED=0 go test -c ./internal/services/runservice/executor/driver -o ./bin/docker-tests - environment: - ENV01: envvalue01 - - save_to_workspace: - contents: - - source_dir: ./bin - dest_dir: /bin/ - paths: - - "*" - - test-docker-driver: - runtime: dind - steps: - - run: env - - restore_workspace: - dest_dir: . - - run: sleep 5 - - run: ./bin/docker-tests -test.parallel 1 -test.v - runs: - agola build/test: - elements: - build go1.12: - task: build-go1.12 - build docker tests go1.12: - task: build-docker-tests-go-1.12 - test docker driver: - task: test-docker-driver + - name: agola build/test + tasks: + - name: build go1.12 + runtime: + type: pod + arch: amd64 + containers: + - image: golang:1.12-stretch + environment: + ENV01: envvalue01 + working_dir: /go/src/github.com/sorintlab/agola + environment: + GO111MODULE: "on" + VAR01: + from_variable: var01 + steps: + - run: env + - clone: + - run: SKIP_DOCKER_TESTS=1 go test -v -count 1 ./... + - name: build docker tests go1.12 + runtime: + type: pod + arch: amd64 + containers: + - image: golang:1.12-stretch + environment: + ENV01: envvalue01 + working_dir: /go/src/github.com/sorintlab/agola + environment: + GO111MODULE: "on" + steps: + - run: env + - clone: + - run: + name: build docker tests binary + command: CGO_ENABLED=0 go test -c ./internal/services/runservice/executor/driver -o ./bin/docker-tests + environment: + ENV01: envvalue01 + - save_to_workspace: + contents: + - source_dir: ./bin + dest_dir: /bin/ + paths: + - "*" + - name: test docker driver + runtime: + type: pod + arch: amd64 + containers: + - image: docker:stable-dind + privileged: true + entrypoint: dockerd + steps: + - run: env + - restore_workspace: + dest_dir: . + - run: ./bin/docker-tests -test.parallel 1 -test.v depends: - build docker tests go1.12 \ No newline at end of file diff --git a/go.mod b/go.mod index f2ce41d..8cc5bf2 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.3.3 // indirect github.com/elazarl/go-bindata-assetfs v1.0.0 + github.com/ghodss/yaml v1.0.0 github.com/go-bindata/go-bindata v1.0.0 github.com/go-ini/ini v1.42.0 // indirect github.com/go-sql-driver/mysql v1.4.1 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index b1700c8..196458f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,6 +15,7 @@ package config import ( + "encoding/json" "fmt" "regexp" "strings" @@ -23,8 +24,8 @@ import ( "github.com/sorintlab/agola/internal/services/types" "github.com/sorintlab/agola/internal/util" + "github.com/ghodss/yaml" "github.com/pkg/errors" - yaml "gopkg.in/yaml.v2" ) const ( @@ -38,19 +39,7 @@ var ( ) type Config struct { - Runtimes map[string]*Runtime `yaml:"runtimes"` - Tasks map[string]*Task `yaml:"tasks"` - Runs map[string]*Run `yaml:"runs"` -} - -type Task struct { - Name string `yaml:"name"` - Runtime string `yaml:"runtime"` - Environment map[string]Value `yaml:"environment,omitempty"` - WorkingDir string `yaml:"working_dir"` - Shell string `yaml:"shell"` - User string `yaml:"user"` - Steps []interface{} `yaml:"steps"` + Runs []*Run `json:"runs"` } type RuntimeType string @@ -66,42 +55,46 @@ const ( ) type RegistryAuth struct { - Type RegistryAuthType `yaml:"type"` + Type RegistryAuthType `json:"type"` // default auth - Username Value `yaml:"username"` - Password Value `yaml:"password"` + Username Value `json:"username"` + Password Value `json:"password"` } type Runtime struct { - Name string `yaml:"name"` - Type RuntimeType `yaml:"type,omitempty"` - Auth *RegistryAuth `yaml:"auth"` - Arch common.Arch `yaml:"arch,omitempty"` - Containers []*Container `yaml:"containers,omitempty"` + Type RuntimeType `json:"type,omitempty"` + Auth *RegistryAuth `json:"auth"` + Arch common.Arch `json:"arch,omitempty"` + Containers []*Container `json:"containers,omitempty"` } type Container struct { - Image string `yaml:"image,omitempty"` - Auth *RegistryAuth `yaml:"auth"` - Environment map[string]Value `yaml:"environment,omitempty"` - User string `yaml:"user"` - Privileged bool `yaml:"privileged"` - Entrypoint string `yaml:"entrypoint"` + Image string `json:"image,omitempty"` + Auth *RegistryAuth `json:"auth"` + Environment map[string]Value `json:"environment,omitempty"` + User string `json:"user"` + Privileged bool `json:"privileged"` + Entrypoint string `json:"entrypoint"` } type Run struct { - Name string `yaml:"name"` - Elements map[string]*Element `yaml:"elements"` + Name string `json:"name"` + Tasks []*Task `json:"tasks"` } -type Element struct { - Name string `yaml:"name"` - Task string `yaml:"task"` - Depends []*Depend `yaml:"depends"` - IgnoreFailure bool `yaml:"ignore_failure"` - Approval bool `yaml:"approval"` - When *types.When `yaml:"when"` +type Task struct { + Name string `json:"name"` + Runtime *Runtime `json:"runtime"` + Environment map[string]Value `json:"environment,omitempty"` + WorkingDir string `json:"working_dir"` + Shell string `json:"shell"` + User string `json:"user"` + Steps []interface{} `json:"steps"` + Depends []*Depend `json:"depends"` + IgnoreFailure bool `json:"ignore_failure"` + Approval bool `json:"approval"` + When *types.When `json:"when"` } type DependCondition string @@ -113,26 +106,26 @@ const ( ) type Depend struct { - ElementName string `yaml:"name"` - Conditions []DependCondition `yaml:"conditions"` + TaskName string `json:"task"` + Conditions []DependCondition `json:"conditions"` } type Step struct { - Type string `yaml:"type"` - Name string `yaml:"name"` + Type string `json:"type"` + Name string `json:"name"` } type CloneStep struct { - Step `yaml:",inline"` + Step `json:",inline"` } type RunStep struct { - Step `yaml:",inline"` - Command string `yaml:"command"` - Environment map[string]Value `yaml:"environment,omitempty"` - WorkingDir string `yaml:"working_dir"` - Shell string `yaml:"shell"` - User string `yaml:"user"` + Step `json:",inline"` + Command string `json:"command"` + Environment map[string]Value `json:"environment,omitempty"` + WorkingDir string `json:"working_dir"` + Shell string `json:"shell"` + User string `json:"user"` } type ValueType int @@ -148,63 +141,78 @@ type Value struct { } type SaveContent struct { - SourceDir string `yaml:"source_dir"` - DestDir string `yaml:"dest_dir"` - Paths []string `yaml:"paths"` + SourceDir string `json:"source_dir"` + DestDir string `json:"dest_dir"` + Paths []string `json:"paths"` } type SaveToWorkspaceStep struct { - Step `yaml:",inline"` - Contents []*SaveContent `yaml:"contents"` + Step `json:",inline"` + Contents []*SaveContent `json:"contents"` } type RestoreWorkspaceStep struct { - Step `yaml:",inline"` - DestDir string `yaml:"dest_dir"` + Step `json:",inline"` + DestDir string `json:"dest_dir"` } type SaveCacheStep struct { - Step `yaml:",inline"` - Key string `yaml:"key"` - Contents []*SaveContent `yaml:"contents"` + Step `json:",inline"` + Key string `json:"key"` + Contents []*SaveContent `json:"contents"` } type RestoreCacheStep struct { - Step `yaml:",inline"` - Keys []string `yaml:"keys"` - DestDir string `yaml:"dest_dir"` + Step `json:",inline"` + Keys []string `json:"keys"` + DestDir string `json:"dest_dir"` } -func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { - type task struct { - Name string `yaml:"name"` - Runtime string `yaml:"runtime"` - Environment map[string]Value `yaml:"environment,omitempty"` - WorkingDir string `yaml:"working_dir"` - Shell string `yaml:"shell"` - User string `yaml:"user"` - Steps []map[string]interface{} `yaml:"steps"` +func (t *Task) UnmarshalJSON(b []byte) error { + type when struct { + Branch interface{} `json:"branch"` + Tag interface{} `json:"tag"` + Ref interface{} `json:"ref"` } - var tt *task - if err := unmarshal(&tt); err != nil { + type runtask struct { + Name string `json:"name"` + Runtime *Runtime `json:"runtime"` + Environment map[string]Value `json:"environment,omitempty"` + WorkingDir string `json:"working_dir"` + Shell string `json:"shell"` + User string `json:"user"` + Steps []map[string]interface{} `json:"steps"` + Depends []interface{} `json:"depends"` + IgnoreFailure bool `json:"ignore_failure"` + Approval bool `json:"approval"` + When *when `json:"when"` + } + + var tr *runtask + + if err := json.Unmarshal(b, &tr); err != nil { return err } - t.Name = tt.Name - t.Runtime = tt.Runtime - t.Environment = tt.Environment - t.WorkingDir = tt.WorkingDir - t.Shell = tt.Shell - t.User = tt.User + t.Name = tr.Name + t.Runtime = tr.Runtime + t.Environment = tr.Environment + t.WorkingDir = tr.WorkingDir + t.Shell = tr.Shell + t.User = tr.User + t.IgnoreFailure = tr.IgnoreFailure + t.Approval = tr.Approval - steps := make([]interface{}, len(tt.Steps)) - for i, stepEntry := range tt.Steps { - if len(stepEntry) > 1 { - return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i) - } - for stepType, stepSpec := range stepEntry { - o, err := yaml.Marshal(stepSpec) + steps := make([]interface{}, len(tr.Steps)) + for i, stepEntry := range tr.Steps { + if _, ok := stepEntry["type"]; ok { + // handle default step definition using format { type: "steptype", other steps fields } + stepType, ok := stepEntry["type"].(string) + if !ok { + return errors.Errorf("step type at index %d must be a string", i) + } + o, err := json.Marshal(stepEntry) if err != nil { return err } @@ -216,20 +224,15 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { case "run": var s RunStep - switch stepSpec.(type) { - case string: - s.Command = stepSpec.(string) - default: - if err := yaml.Unmarshal(o, &s); err != nil { - return err - } + if err := json.Unmarshal(o, &s); err != nil { + return err } s.Type = stepType steps[i] = &s case "save_to_workspace": var s SaveToWorkspaceStep - if err := yaml.Unmarshal(o, &s); err != nil { + if err := json.Unmarshal(o, &s); err != nil { return err } s.Type = stepType @@ -237,7 +240,7 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { case "restore_workspace": var s RestoreWorkspaceStep - if err := yaml.Unmarshal(o, &s); err != nil { + if err := json.Unmarshal(o, &s); err != nil { return err } s.Type = stepType @@ -245,7 +248,7 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { case "save_cache": var s SaveCacheStep - if err := yaml.Unmarshal(o, &s); err != nil { + if err := json.Unmarshal(o, &s); err != nil { return err } s.Type = stepType @@ -253,7 +256,7 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { case "restore_cache": var s RestoreCacheStep - if err := yaml.Unmarshal(o, &s); err != nil { + if err := json.Unmarshal(o, &s); err != nil { return err } s.Type = stepType @@ -261,119 +264,181 @@ func (t *Task) UnmarshalYAML(unmarshal func(interface{}) error) error { default: return errors.Errorf("unknown step type: %s", stepType) } + } else { + // handle simpler (for yaml not for json) steps definition using format "steptype": { stepSpecification } + if len(stepEntry) > 1 { + return errors.Errorf("wrong steps description at index %d: more than one step name per list entry", i) + } + for stepType, stepSpec := range stepEntry { + o, err := json.Marshal(stepSpec) + if err != nil { + return err + } + switch stepType { + case "clone": + var s CloneStep + s.Type = stepType + steps[i] = &s + + case "run": + var s RunStep + switch stepSpec.(type) { + case string: + s.Command = stepSpec.(string) + default: + if err := json.Unmarshal(o, &s); err != nil { + return err + } + } + s.Type = stepType + steps[i] = &s + + case "save_to_workspace": + var s SaveToWorkspaceStep + if err := json.Unmarshal(o, &s); err != nil { + return err + } + s.Type = stepType + steps[i] = &s + + case "restore_workspace": + var s RestoreWorkspaceStep + if err := json.Unmarshal(o, &s); err != nil { + return err + } + s.Type = stepType + steps[i] = &s + + case "save_cache": + var s SaveCacheStep + if err := json.Unmarshal(o, &s); err != nil { + return err + } + s.Type = stepType + steps[i] = &s + + case "restore_cache": + var s RestoreCacheStep + if err := json.Unmarshal(o, &s); err != nil { + return err + } + s.Type = stepType + steps[i] = &s + default: + return errors.Errorf("unknown step type: %s", stepType) + } + } } } t.Steps = steps - return nil -} - -func (e *Element) UnmarshalYAML(unmarshal func(interface{}) error) error { - type when struct { - Branch interface{} `yaml:"branch"` - Tag interface{} `yaml:"tag"` - Ref interface{} `yaml:"ref"` - } - - type element struct { - Name string `yaml:"name"` - Task string `yaml:"task"` - Depends []interface{} `yaml:"depends"` - IgnoreFailure bool `yaml:"ignore_failure"` - Approval bool `yaml:"approval"` - When *when `yaml:"when"` - } - - var te *element - - if err := unmarshal(&te); err != nil { - return err - } - - e.Name = te.Name - e.Task = te.Task - e.IgnoreFailure = te.IgnoreFailure - e.Approval = te.Approval - - depends := make([]*Depend, len(te.Depends)) - for i, dependEntry := range te.Depends { + depends := make([]*Depend, len(tr.Depends)) + for i, dependEntry := range tr.Depends { var depend *Depend - switch dependEntry.(type) { + isSimpler := false + switch de := dependEntry.(type) { + // handle simpler (for yaml) depends definition using format "taskname": case string: depend = &Depend{ - ElementName: dependEntry.(string), + TaskName: dependEntry.(string), } - case map[interface{}]interface{}: - type deplist map[string][]DependCondition - var dl deplist - o, err := yaml.Marshal(dependEntry) - if err != nil { - return err + case map[string]interface{}: + if len(de) == 1 { + for _, v := range de { + switch v.(type) { + case []interface{}: + isSimpler = true + case string: + default: + return errors.Errorf("unsupported depend entry format") + } + } } - if err := yaml.Unmarshal(o, &dl); err != nil { - return err - } - if len(dl) != 1 { - return errors.Errorf("unsupported depend format. Must be a string or a list") - } - for k, v := range dl { - depend = &Depend{ - ElementName: k, - Conditions: v, + if !isSimpler { + // handle default depends definition using format "task": "taskname", conditions: [ list of conditions ] + o, err := json.Marshal(dependEntry) + if err != nil { + return err + } + if err := json.Unmarshal(o, &depend); err != nil { + return err + } + } else { + // handle simpler (for yaml) depends definition using format "taskname": [ list of conditions ] + if len(de) != 1 { + return errors.Errorf("unsupported depend entry format") + } + type deplist map[string][]DependCondition + var dl deplist + o, err := json.Marshal(dependEntry) + if err != nil { + return err + } + if err := json.Unmarshal(o, &dl); err != nil { + return err + } + if len(dl) != 1 { + return errors.Errorf("unsupported depend entry format") + } + for k, v := range dl { + depend = &Depend{ + TaskName: k, + Conditions: v, + } } } default: - return errors.Errorf("unsupported depend format. Must be a string or a list") + return errors.Errorf("unsupported depend entry format") } depends[i] = depend } - e.Depends = depends + t.Depends = depends - if te.When != nil { + if tr.When != nil { w := &types.When{} var err error - if te.When.Branch != nil { - w.Branch, err = parseWhenConditions(te.When.Branch) + if tr.When.Branch != nil { + w.Branch, err = parseWhenConditions(tr.When.Branch) if err != nil { return err } } - if te.When.Tag != nil { - w.Tag, err = parseWhenConditions(te.When.Tag) + if tr.When.Tag != nil { + w.Tag, err = parseWhenConditions(tr.When.Tag) if err != nil { return err } } - if te.When.Ref != nil { - w.Ref, err = parseWhenConditions(te.When.Ref) + if tr.When.Ref != nil { + w.Ref, err = parseWhenConditions(tr.When.Ref) if err != nil { return err } } - e.When = w + t.When = w } return nil } -func (val *Value) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (val *Value) UnmarshalJSON(b []byte) error { var ival interface{} - if err := unmarshal(&ival); err != nil { + if err := json.Unmarshal(b, &ival); err != nil { return err } switch valValue := ival.(type) { case string: val.Type = ValueTypeString val.Value = valValue - case map[interface{}]interface{}: + case map[string]interface{}: for k, v := range valValue { if k == "from_variable" { switch v.(type) { @@ -407,13 +472,9 @@ func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) { return nil, err } include = ss - case map[interface{}]interface{}: + case map[string]interface{}: for k, v := range c { - ks, ok := k.(string) - if !ok { - return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, ks) - } - switch ks { + switch k { case "include": include, err = parseStringOrSlice(v) if err != nil { @@ -425,7 +486,7 @@ func parseWhenConditions(wi interface{}) (*types.WhenConditions, error) { return nil, err } default: - return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, ks) + return nil, errors.Errorf(`expected one of "include" or "exclude", got %s`, k) } } default: @@ -514,31 +575,22 @@ func parseSliceString(si []interface{}) ([]string, error) { return ss, nil } -func (c *Config) Runtime(runtimeName string) *Runtime { - for n, r := range c.Runtimes { - if n == runtimeName { +func (c *Config) Run(runName string) *Run { + for _, r := range c.Runs { + if r.Name == runName { return r } } - panic(fmt.Sprintf("runtime %q doesn't exists", runtimeName)) + panic(fmt.Sprintf("run %q doesn't exists", runName)) } -func (c *Config) Task(taskName string) *Task { - for n, t := range c.Tasks { - if n == taskName { +func (r *Run) Task(taskName string) *Task { + for _, t := range r.Tasks { + if t.Name == taskName { return t } } - panic(fmt.Sprintf("task %q doesn't exists", taskName)) -} - -func (c *Config) Run(runName string) *Run { - for n, p := range c.Runs { - if n == runName { - return p - } - } - panic(fmt.Sprintf("run %q doesn't exists", runName)) + panic(fmt.Sprintf("task %q for run %q doesn't exists", taskName, r.Name)) } var DefaultConfig = Config{} @@ -549,92 +601,79 @@ func ParseConfig(configData []byte) (*Config, error) { return nil, errors.Wrapf(err, "failed to unmarshal config") } - if len(config.Runs) == 0 { - return nil, errors.Errorf("no runs defined") - } - - // Set names from maps keys - for n, runtime := range config.Runtimes { - if runtime == nil { - return nil, errors.Errorf("runtime %q is empty", n) - } - runtime.Name = n - } - - for n, task := range config.Tasks { - if task == nil { - return nil, errors.Errorf("task %q is empty", n) - } - task.Name = n - } - - for n, run := range config.Runs { - if run == nil { - return nil, errors.Errorf("run %q is empty", n) - } - run.Name = n - } - - for _, run := range config.Runs { - for n, element := range run.Elements { - if element == nil { - return nil, errors.Errorf("run %q: element %q is empty", run.Name, n) - } - element.Name = n - } - } - - // Set auth type to default if not specified - for _, runtime := range config.Runtimes { - if runtime.Auth != nil { - if runtime.Auth.Type == "" { - runtime.Auth.Type = RegistryAuthTypeDefault - } - } - for _, container := range runtime.Containers { - if container.Auth != nil { - if container.Auth.Type == "" { - container.Auth.Type = RegistryAuthTypeDefault - } - } - } - } - - // set steps defaults - for _, t := range config.Tasks { - for _, s := range t.Steps { - switch step := s.(type) { - // TODO(sgotti) we could use the run step command as step name but when the - // command is very long or multi line it doesn't makes sense and will - // probably be quite unuseful/confusing from an UI point of view - case *SaveCacheStep: - for _, content := range step.Contents { - if len(content.Paths) == 0 { - // default to all files inside the sourceDir - content.Paths = []string{"**"} - } - } - log.Infof("s: %s", util.Dump(s)) - } - } - } - return &config, checkConfig(&config) } func checkConfig(config *Config) error { + if len(config.Runs) == 0 { + return errors.Errorf("no runs defined") + } + + seenRuns := map[string]struct{}{} + for ri, run := range config.Runs { + if run == nil { + return errors.Errorf("run at index %d is empty", ri) + } + + if run.Name == "" { + return errors.Errorf("run at index %d has empty name", ri) + } + + if len(run.Name) > maxRunNameLength { + return errors.Errorf("run name %q too long", run.Name) + } + + if _, ok := seenRuns[run.Name]; ok { + return errors.Errorf("duplicate run name: %s", run.Name) + } + seenRuns[run.Name] = struct{}{} + + seenTasks := map[string]struct{}{} + for ti, task := range run.Tasks { + if task == nil { + return errors.Errorf("run %q: task at index %d is empty", run.Name, ti) + } + + if task.Name == "" { + return errors.Errorf("run %q: task at index %d has empty name", run.Name, ti) + } + + if len(task.Name) > maxTaskNameLength { + return errors.Errorf("task name %q too long", task.Name) + } + + if _, ok := seenTasks[task.Name]; ok { + return errors.Errorf("duplicate task name: %s", task.Name) + } + seenTasks[task.Name] = struct{}{} + + // check tasks runtime + if task.Runtime == nil { + return errors.Errorf("task %q: runtime is not defined", task.Name) + } + + r := task.Runtime + if r.Type != RuntimeTypePod { + return errors.Errorf("task %q runtime: wrong type %q", task.Name, r.Type) + } + if len(r.Containers) == 0 { + return errors.Errorf("task %q runtime: at least one container must be defined", task.Name) + } + } + } + // check broken dependencies for _, run := range config.Runs { // collect all task names - allElements := map[string]struct{}{} - for _, element := range run.Elements { - allElements[element.Name] = struct{}{} + allTasks := map[string]struct{}{} + for _, task := range run.Tasks { + allTasks[task.Name] = struct{}{} } - for _, element := range run.Elements { - for _, dep := range element.Depends { - if _, ok := allElements[dep.ElementName]; !ok { - return errors.Errorf("run element %q needed by element %q doesn't exist", dep.ElementName, element.Name) + for _, task := range run.Tasks { + for _, dep := range task.Depends { + if _, ok := allTasks[dep.TaskName]; !ok { + return errors.Errorf("run task %q needed by task %q doesn't exist", dep.TaskName, task.Name) } } } @@ -643,21 +682,21 @@ func checkConfig(config *Config) error { // check circular dependencies for _, run := range config.Runs { cerrs := &util.Errors{} - for _, element := range run.Elements { - allParents := getAllElementParents(run, element) + for _, task := range run.Tasks { + allParents := getAllTaskParents(run, task) for _, parent := range allParents { - if parent.Name == element.Name { + if parent.Name == task.Name { // TODO(sgotti) get the parent that depends on task to report it dep := []string{} for _, parent := range allParents { - pparents := getElementParents(run, parent) + pparents := getTaskParents(run, parent) for _, pparent := range pparents { - if pparent.Name == element.Name { + if pparent.Name == task.Name { dep = append(dep, fmt.Sprintf("%q", parent.Name)) } } } - cerrs.Append(errors.Errorf("circular dependency between element %q and elements %s", element.Name, strings.Join(dep, " "))) + cerrs.Append(errors.Errorf("circular dependency between task %q and tasks %s", task.Name, strings.Join(dep, " "))) } } } @@ -668,15 +707,15 @@ func checkConfig(config *Config) error { // check that the task and its parent don't have a common dependency for _, run := range config.Runs { - for _, element := range run.Elements { - parents := getElementParents(run, element) + for _, task := range run.Tasks { + parents := getTaskParents(run, task) for _, parent := range parents { - allParents := getAllElementParents(run, element) - allParentParents := getAllElementParents(run, parent) + allParents := getAllTaskParents(run, task) + allParentParents := getAllTaskParents(run, parent) for _, p := range allParents { for _, pp := range allParentParents { if p.Name == pp.Name { - return errors.Errorf("element %s and its dependency %s have both a dependency on element %s", element.Name, parent.Name, p.Name) + return errors.Errorf("task %s and its dependency %s have both a dependency on task %s", task.Name, parent.Name, p.Name) } } } @@ -684,66 +723,66 @@ func checkConfig(config *Config) error { } } - for _, r := range config.Runtimes { - if r.Type != RuntimeTypePod { - return errors.Errorf("runtime %q: wrong type %q", r.Name, r.Type) - } - if len(r.Containers) == 0 { - return errors.Errorf("runtime %q: at least one container must be defined", r.Name) - } - } - - for _, t := range config.Tasks { - if len(t.Name) > maxTaskNameLength { - return errors.Errorf("task name %q too long", t.Name) - } - if t.Runtime == "" { - return errors.Errorf("task %q: undefined runtime", t.Name) - } - if _, ok := config.Runtimes[t.Runtime]; !ok { - return errors.Errorf("runtime %q needed by task %q doesn't exist", t.Runtime, t.Name) - } - for i, s := range t.Steps { - switch step := s.(type) { - // TODO(sgotti) we could use the run step command as step name but when the - // command is very long or multi line it doesn't makes sense and will - // probably be quite unuseful/confusing from an UI point of view - case *RunStep: - if step.Name == "" { - lines, err := util.CountLines(step.Command) - // if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred - if err != nil || lines > 1 { - return errors.Errorf("missing step name for step %d in task %q, required since command is more than one line", i, t.Name) - } - len := len(step.Command) - if len > maxStepNameLength { - len = maxStepNameLength - } - step.Name = step.Command[:len] - } - } - } - } - + // check duplicate task dependencies for _, run := range config.Runs { - if len(run.Name) > maxRunNameLength { - return errors.Errorf("run name %q too long", run.Name) - } - for _, element := range run.Elements { - // check missing tasks reference - if element.Task == "" { - return errors.Errorf("no task defined for run element %q", element.Name) - } - if _, ok := config.Tasks[element.Task]; !ok { - return errors.Errorf("task %q needed by run element %q doesn't exist", element.Task, element.Name) - } + for _, task := range run.Tasks { // check duplicate dependencies in task seenDependencies := map[string]struct{}{} - for _, dep := range element.Depends { - if _, ok := seenDependencies[dep.ElementName]; ok { - return errors.Errorf("duplicate task dependency: %s", element.Name) + for _, dep := range task.Depends { + if _, ok := seenDependencies[dep.TaskName]; ok { + return errors.Errorf("duplicate task dependency: %s", task.Name) + } + seenDependencies[dep.TaskName] = struct{}{} + } + } + } + + // Set defaults + for _, run := range config.Runs { + for _, task := range run.Tasks { + // Set auth type to default if not specified + runtime := task.Runtime + if runtime.Auth != nil { + if runtime.Auth.Type == "" { + runtime.Auth.Type = RegistryAuthTypeDefault + } + } + for _, container := range runtime.Containers { + if container.Auth != nil { + if container.Auth.Type == "" { + container.Auth.Type = RegistryAuthTypeDefault + } + } + } + + // set steps defaults + for i, s := range task.Steps { + switch step := s.(type) { + // TODO(sgotti) we could use the run step command as step name but when the + // command is very long or multi line it doesn't makes sense and will + // probably be quite unuseful/confusing from an UI point of view + case *RunStep: + if step.Name == "" { + lines, err := util.CountLines(step.Command) + // if we failed to count the lines (shouldn't happen) or the number of lines is > 1 then a name is requred + if err != nil || lines > 1 { + return errors.Errorf("missing step name for step %d (run) in task %q, required since command is more than one line", i, task.Name) + } + len := len(step.Command) + if len > maxStepNameLength { + len = maxStepNameLength + } + step.Name = step.Command[:len] + } + + case *SaveCacheStep: + for _, content := range step.Contents { + if len(content.Paths) == 0 { + // default to all files inside the sourceDir + content.Paths = []string{"**"} + } + } } - seenDependencies[dep.ElementName] = struct{}{} } } } @@ -751,13 +790,13 @@ func checkConfig(config *Config) error { return nil } -// getElementParents returns direct parents of element. -func getElementParents(run *Run, element *Element) []*Element { - parents := []*Element{} - for _, el := range run.Elements { +// getTaskParents returns direct parents of task. +func getTaskParents(run *Run, task *Task) []*Task { + parents := []*Task{} + for _, el := range run.Tasks { isParent := false - for _, d := range element.Depends { - if d.ElementName == el.Name { + for _, d := range task.Depends { + if d.TaskName == el.Name { isParent = true } } @@ -768,26 +807,26 @@ func getElementParents(run *Run, element *Element) []*Element { return parents } -// getAllElementParents returns all the parents (both direct and ancestors) of an element. +// getAllTaskParents returns all the parents (both direct and ancestors) of a task. // In case of circular dependency it won't loop forever but will also return -// the element as parent of itself -func getAllElementParents(run *Run, element *Element) []*Element { - pMap := map[string]*Element{} - nextParents := getElementParents(run, element) +// the task as parent of itself +func getAllTaskParents(run *Run, task *Task) []*Task { + pMap := map[string]*Task{} + nextParents := getTaskParents(run, task) for len(nextParents) > 0 { parents := nextParents - nextParents = []*Element{} + nextParents = []*Task{} for _, parent := range parents { if _, ok := pMap[parent.Name]; ok { continue } pMap[parent.Name] = parent - nextParents = append(nextParents, getElementParents(run, parent)...) + nextParents = append(nextParents, getTaskParents(run, parent)...) } } - parents := make([]*Element, 0, len(pMap)) + parents := make([]*Task, 0, len(pMap)) for _, v := range pMap { parents = append(parents, v) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ea2d6f8..6b35ff0 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -47,52 +47,61 @@ func TestParseConfig(t *testing.T) { name: "test empty run", in: ` runs: - run01: + - `, - err: fmt.Errorf(`run "run01" is empty`), + err: fmt.Errorf(`run at index 0 is empty`), }, { - name: "test missing element dependency", + name: "test empty task", in: ` - tasks: - task0k1: - environment: - ENV01: ENV01 - runs: - run01: - elements: - element01: - task: task01 - depends: - - element02 + - name: run01 + tasks: + - `, - err: fmt.Errorf(`run element "element02" needed by element "element01" doesn't exist`), + err: fmt.Errorf(`run "run01": task at index 0 is empty`), }, { - name: "test circular dependency between 2 elements a -> b -> a", + name: "test missing task dependency", in: ` - tasks: - task01: - environment: - ENV01: ENV01 - runs: - run01: - elements: - element01: - task: task01 + - name: run01 + tasks: + - name: task01 + runtime: + type: pod + containers: + - image: busybox depends: - - element02 - element02: - task: task01 + - task02 + `, + err: fmt.Errorf(`run task "task02" needed by task "task01" doesn't exist`), + }, + { + name: "test circular dependency between 2 tasks a -> b -> a", + in: ` + runs: + - name: run01 + tasks: + - name: task01 + runtime: + type: pod + containers: + - image: busybox depends: - - element01 + - task02 + - name: task02 + runtime: + type: pod + containers: + - image: busybox + depends: + - task01 `, err: &util.Errors{ Errs: []error{ - errors.Errorf("circular dependency between element %q and elements %q", "element01", "element02"), - errors.Errorf("circular dependency between element %q and elements %q", "element02", "element01"), + errors.Errorf("circular dependency between task %q and tasks %q", "task01", "task02"), + errors.Errorf("circular dependency between task %q and tasks %q", "task02", "task01"), }, }, }, @@ -129,54 +138,67 @@ func TestParseOutput(t *testing.T) { out *Config }{ { - name: "test element when conditions", + name: "test task all options", in: ` - runtimes: - runtime01: - type: pod - auth: - username: username - password: - from_variable: password - containers: - - image: image01 - auth: - username: - from_variable: username2 - password: password2 + runs: + - name: run01 + tasks: + - name: task01 + runtime: + type: pod + auth: + username: username + password: + from_variable: password + containers: + - image: image01 + auth: + username: + from_variable: username2 + password: password2 + environment: + ENV01: ENV01 + ENVFROMVARIABLE01: + from_variable: variable01 environment: ENV01: ENV01 ENVFROMVARIABLE01: from_variable: variable01 + steps: + # normal step definition + - type: clone + - type: run + command: command01 + - type: run + name: name different than command + command: command02 + - type: run + command: command03 + environment: + ENV01: ENV01 + ENVFROMVARIABLE01: + from_variable: variable01 + - type: save_cache + key: cache-{{ arch }} + contents: + - source_dir: /go/pkg/mod/cache - tasks: - task01: - runtime: runtime01 - environment: - ENV01: ENV01 - ENVFROMVARIABLE01: - from_variable: variable01 - steps: - - run: command01 - - run: - name: name different than command - command: command02 - - run: - command: command03 - environment: - ENV01: ENV01 - ENVFROMVARIABLE01: - from_variable: variable01 - - save_cache: - key: cache-{{ arch }} - contents: - - source_dir: /go/pkg/mod/cache - - runs: - run01: - elements: - element01: - task: task01 + # simpler (for yaml not for json) steps definition + - clone: + - run: command01 + - run: + name: name different than command + command: command02 + - run: + command: command03 + environment: + ENV01: ENV01 + ENVFROMVARIABLE01: + from_variable: variable01 + - save_cache: + key: cache-{{ arch }} + contents: + - source_dir: /go/pkg/mod/cache when: branch: master tag: @@ -185,88 +207,132 @@ func TestParseOutput(t *testing.T) { ref: include: master exclude: [ /branch01/ , branch02 ] + depends: + - task: task02 + conditions: + - on_success + - on_failure + - task03 + - task04: + - on_success + - name: task02 + runtime: + type: pod + containers: + - image: image01 + - name: task03 + runtime: + type: pod + containers: + - image: image01 + - name: task04 + runtime: + type: pod + containers: + - image: image01 `, out: &Config{ - Runtimes: map[string]*Runtime{ - "runtime01": &Runtime{ - Name: "runtime01", - Type: "pod", - Auth: &RegistryAuth{ - Type: RegistryAuthTypeDefault, - Username: Value{Type: ValueTypeString, Value: "username"}, - Password: Value{Type: ValueTypeFromVariable, Value: "password"}, - }, - Arch: "", - Containers: []*Container{ - &Container{ - Image: "image01", - Auth: &RegistryAuth{ - Type: RegistryAuthTypeDefault, - Username: Value{Type: ValueTypeFromVariable, Value: "username2"}, - Password: Value{Type: ValueTypeString, Value: "password2"}, - }, - Environment: map[string]Value{ - "ENV01": Value{Type: ValueTypeString, Value: "ENV01"}, - "ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"}, - }, - User: "", - }, - }, - }, - }, - Tasks: map[string]*Task{ - "task01": &Task{ - Name: "task01", - Runtime: "runtime01", - Environment: map[string]Value{ - "ENV01": Value{Type: ValueTypeString, Value: "ENV01"}, - "ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"}, - }, - WorkingDir: "", - Shell: "", - User: "", - Steps: []interface{}{ - &RunStep{ - Step: Step{ - Type: "run", - Name: "command01", - }, - Command: "command01", - }, - &RunStep{ - Step: Step{ - Type: "run", - Name: "name different than command", - }, - Command: "command02", - }, - &RunStep{ - Step: Step{ - Type: "run", - Name: "command03", - }, - Command: "command03", - Environment: map[string]Value{ - "ENV01": Value{Type: ValueTypeString, Value: "ENV01"}, - "ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"}, - }, - }, - &SaveCacheStep{ - Step: Step{Type: "save_cache"}, - Key: "cache-{{ arch }}", - Contents: []*SaveContent{&SaveContent{SourceDir: "/go/pkg/mod/cache", Paths: []string{"**"}}}, - }, - }, - }, - }, - Runs: map[string]*Run{ - "run01": &Run{ + Runs: []*Run{ + &Run{ Name: "run01", - Elements: map[string]*Element{ - "element01": &Element{ - Name: "element01", - Task: "task01", - Depends: []*Depend{}, + Tasks: []*Task{ + &Task{ + Name: "task01", + Runtime: &Runtime{ + Type: "pod", + Auth: &RegistryAuth{ + Type: RegistryAuthTypeDefault, + Username: Value{Type: ValueTypeString, Value: "username"}, + Password: Value{Type: ValueTypeFromVariable, Value: "password"}, + }, + Arch: "", + Containers: []*Container{ + &Container{ + Image: "image01", + Auth: &RegistryAuth{ + Type: RegistryAuthTypeDefault, + Username: Value{Type: ValueTypeFromVariable, Value: "username2"}, + Password: Value{Type: ValueTypeString, Value: "password2"}, + }, + Environment: map[string]Value{ + "ENV01": Value{Type: ValueTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"}, + }, + User: "", + }, + }, + }, + Environment: map[string]Value{ + "ENV01": Value{Type: ValueTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"}, + }, + WorkingDir: "", + Shell: "", + User: "", + Steps: []interface{}{ + &CloneStep{Step: Step{Type: "clone"}}, + &RunStep{ + Step: Step{ + Type: "run", + Name: "command01", + }, + Command: "command01", + }, + &RunStep{ + Step: Step{ + Type: "run", + Name: "name different than command", + }, + Command: "command02", + }, + &RunStep{ + Step: Step{ + Type: "run", + Name: "command03", + }, + Command: "command03", + Environment: map[string]Value{ + "ENV01": Value{Type: ValueTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"}, + }, + }, + &SaveCacheStep{ + Step: Step{Type: "save_cache"}, + Key: "cache-{{ arch }}", + Contents: []*SaveContent{&SaveContent{SourceDir: "/go/pkg/mod/cache", Paths: []string{"**"}}}, + }, + &CloneStep{Step: Step{Type: "clone"}}, + &RunStep{ + Step: Step{ + Type: "run", + Name: "command01", + }, + Command: "command01", + }, + &RunStep{ + Step: Step{ + Type: "run", + Name: "name different than command", + }, + Command: "command02", + }, + &RunStep{ + Step: Step{ + Type: "run", + Name: "command03", + }, + Command: "command03", + Environment: map[string]Value{ + "ENV01": Value{Type: ValueTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": Value{Type: ValueTypeFromVariable, Value: "variable01"}, + }, + }, + &SaveCacheStep{ + Step: Step{Type: "save_cache"}, + Key: "cache-{{ arch }}", + Contents: []*SaveContent{&SaveContent{SourceDir: "/go/pkg/mod/cache", Paths: []string{"**"}}}, + }, + }, IgnoreFailure: false, Approval: false, When: &types.When{ @@ -291,6 +357,56 @@ func TestParseOutput(t *testing.T) { }, }, }, + Depends: []*Depend{ + &Depend{TaskName: "task02", Conditions: []DependCondition{DependConditionOnSuccess, DependConditionOnFailure}}, + &Depend{TaskName: "task03", Conditions: nil}, + &Depend{TaskName: "task04", Conditions: []DependCondition{DependConditionOnSuccess}}, + }, + }, + &Task{ + Name: "task02", + Runtime: &Runtime{ + Type: "pod", + Arch: "", + Containers: []*Container{ + &Container{ + Image: "image01", + }, + }, + }, + WorkingDir: "", + Steps: []interface{}{}, + Depends: []*Depend{}, + }, + &Task{ + Name: "task03", + Runtime: &Runtime{ + Type: "pod", + Arch: "", + Containers: []*Container{ + &Container{ + Image: "image01", + }, + }, + }, + WorkingDir: "", + Steps: []interface{}{}, + Depends: []*Depend{}, + }, + &Task{ + Name: "task04", + Runtime: &Runtime{ + Type: "pod", + Arch: "", + Containers: []*Container{ + &Container{ + Image: "image01", + }, + }, + }, + WorkingDir: "", + Steps: []interface{}{}, + Depends: []*Depend{}, }, }, }, diff --git a/internal/runconfig/runconfig.go b/internal/runconfig/runconfig.go index c710f60..45358d1 100644 --- a/internal/runconfig/runconfig.go +++ b/internal/runconfig/runconfig.go @@ -25,9 +25,7 @@ import ( "github.com/sorintlab/agola/internal/util" ) -func genRuntime(c *config.Config, runtimeName string, variables map[string]string) *rstypes.Runtime { - ce := c.Runtime(runtimeName) - +func genRuntime(c *config.Config, ce *config.Runtime, variables map[string]string) *rstypes.Runtime { containers := []*rstypes.Container{} for _, cc := range ce.Containers { env := genEnv(cc.Environment, variables) @@ -189,11 +187,8 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string rcts := map[string]*rstypes.RunConfigTask{} - for _, cre := range cr.Elements { - include := types.MatchWhen(cre.When, branch, tag, ref) - - // resolve referenced task - ct := c.Task(cre.Task) + for _, ct := range cr.Tasks { + include := types.MatchWhen(ct.When, branch, tag, ref) steps := make([]interface{}, len(ct.Steps)) for i, cpts := range ct.Steps { @@ -203,18 +198,17 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string tEnv := genEnv(ct.Environment, variables) t := &rstypes.RunConfigTask{ - ID: uuid.New(cre.Name).String(), - // use the element name from the config as the task name - Name: cre.Name, + ID: uuid.New(ct.Name).String(), + Name: ct.Name, Runtime: genRuntime(c, ct.Runtime, variables), Environment: tEnv, WorkingDir: ct.WorkingDir, Shell: ct.Shell, User: ct.User, Steps: steps, - IgnoreFailure: cre.IgnoreFailure, + IgnoreFailure: ct.IgnoreFailure, Skip: !include, - NeedsApproval: cre.Approval, + NeedsApproval: ct.Approval, } rcts[t.ID] = t @@ -222,10 +216,10 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string // populate depends, needs to be done after having created all the tasks so we can resolve their id for _, rct := range rcts { - cre := cr.Elements[rct.Name] + ct := cr.Task(rct.Name) - depends := make(map[string]*rstypes.RunConfigTaskDepend, len(cre.Depends)) - for _, d := range cre.Depends { + depends := make(map[string]*rstypes.RunConfigTaskDepend, len(ct.Depends)) + for _, d := range ct.Depends { conditions := make([]rstypes.RunConfigTaskDependCondition, len(d.Conditions)) // when no conditions are defined default to on_success if len(d.Conditions) == 0 { @@ -243,7 +237,7 @@ func GenRunConfigTasks(uuid util.UUIDGenerator, c *config.Config, runName string } } - drct := getRunConfigTaskByName(rcts, d.ElementName) + drct := getRunConfigTaskByName(rcts, d.TaskName) depends[drct.ID] = &rstypes.RunConfigTaskDepend{ TaskID: drct.ID, Conditions: conditions, diff --git a/internal/runconfig/runconfig_test.go b/internal/runconfig/runconfig_test.go index f5e1dee..1cb016c 100644 --- a/internal/runconfig/runconfig_test.go +++ b/internal/runconfig/runconfig_test.go @@ -632,80 +632,71 @@ func TestGenRunConfig(t *testing.T) { { name: "test runconfig generation", in: &config.Config{ - Runtimes: map[string]*config.Runtime{ - "runtime01": &config.Runtime{ - Name: "runtime01", - Type: "pod", - Auth: &config.RegistryAuth{ - Type: config.RegistryAuthTypeDefault, - Username: config.Value{Type: config.ValueTypeString, Value: "username"}, - Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"}, - }, - Arch: "", - Containers: []*config.Container{ - &config.Container{ - Image: "image01", - Auth: &config.RegistryAuth{ - Type: config.RegistryAuthTypeDefault, - Username: config.Value{Type: config.ValueTypeFromVariable, Value: "registry_username"}, - Password: config.Value{Type: config.ValueTypeString, Value: "password2"}, - }, - Environment: map[string]config.Value{ - "ENV01": config.Value{Type: config.ValueTypeString, Value: "ENV01"}, - "ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"}, - }, - User: "", - }, - }, - }, - }, - Tasks: map[string]*config.Task{ - "task01": &config.Task{ - Name: "task01", - Runtime: "runtime01", - Environment: map[string]config.Value{ - "ENV01": config.Value{Type: config.ValueTypeString, Value: "ENV01"}, - "ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"}, - }, - WorkingDir: "", - Shell: "", - User: "", - Steps: []interface{}{ - &config.RunStep{ - Step: config.Step{ - Type: "run", - Name: "command01", - }, - Command: "command01", - }, - &config.RunStep{ - Step: config.Step{ - Type: "run", - Name: "name different than command", - }, - Command: "command02", - }, - &config.RunStep{ - Step: config.Step{ - Type: "run", - Name: "command03", - }, - Command: "command03", - Environment: map[string]config.Value{ - "ENV01": config.Value{Type: config.ValueTypeString, Value: "ENV01"}, - "ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"}, - }, - }, - }, - }, - }, - Runs: map[string]*config.Run{ - "run01": &config.Run{ + Runs: []*config.Run{ + &config.Run{ Name: "run01", - Elements: map[string]*config.Element{ - "element01": &config.Element{ - Name: "element01", - Task: "task01", + Tasks: []*config.Task{ + &config.Task{ + Name: "task01", + Runtime: &config.Runtime{ + Type: "pod", + Auth: &config.RegistryAuth{ + Type: config.RegistryAuthTypeDefault, + Username: config.Value{Type: config.ValueTypeString, Value: "username"}, + Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"}, + }, + Arch: "", + Containers: []*config.Container{ + &config.Container{ + Image: "image01", + Auth: &config.RegistryAuth{ + Type: config.RegistryAuthTypeDefault, + Username: config.Value{Type: config.ValueTypeFromVariable, Value: "registry_username"}, + Password: config.Value{Type: config.ValueTypeString, Value: "password2"}, + }, + Environment: map[string]config.Value{ + "ENV01": config.Value{Type: config.ValueTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"}, + }, + User: "", + }, + }, + }, + Environment: map[string]config.Value{ + "ENV01": config.Value{Type: config.ValueTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"}, + }, + WorkingDir: "", + Shell: "", + User: "", + Steps: []interface{}{ + &config.RunStep{ + Step: config.Step{ + Type: "run", + Name: "command01", + }, + Command: "command01", + }, + &config.RunStep{ + Step: config.Step{ + Type: "run", + Name: "name different than command", + }, + Command: "command02", + }, + &config.RunStep{ + Step: config.Step{ + Type: "run", + Name: "command03", + }, + Command: "command03", + Environment: map[string]config.Value{ + "ENV01": config.Value{Type: config.ValueTypeString, Value: "ENV01"}, + "ENVFROMVARIABLE01": config.Value{Type: config.ValueTypeFromVariable, Value: "variable01"}, + }, + }, + }, + Depends: []*config.Depend{}, IgnoreFailure: false, Approval: false, @@ -727,9 +718,9 @@ func TestGenRunConfig(t *testing.T) { "registry_username": "yourregistryusername", }, out: map[string]*rstypes.RunConfigTask{ - uuid.New("element01").String(): &rstypes.RunConfigTask{ - ID: uuid.New("element01").String(), - Name: "element01", Depends: map[string]*rstypes.RunConfigTaskDepend{}, + uuid.New("task01").String(): &rstypes.RunConfigTask{ + ID: uuid.New("task01").String(), + Name: "task01", Depends: map[string]*rstypes.RunConfigTaskDepend{}, Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"), Containers: []*rstypes.Container{ { @@ -762,45 +753,35 @@ func TestGenRunConfig(t *testing.T) { { name: "test runtime auth used for container nil auth", in: &config.Config{ - Runtimes: map[string]*config.Runtime{ - "runtime01": &config.Runtime{ - Name: "runtime01", - Type: "pod", - Auth: &config.RegistryAuth{ - Type: config.RegistryAuthTypeDefault, - Username: config.Value{Type: config.ValueTypeString, Value: "username"}, - Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"}, - }, - Arch: "", - Containers: []*config.Container{ - &config.Container{ - Image: "image01", - }, - }, - }, - }, - Tasks: map[string]*config.Task{ - "task01": &config.Task{ - Name: "task01", - Runtime: "runtime01", - Steps: []interface{}{ - &config.RunStep{ - Step: config.Step{ - Type: "run", - Name: "command01", - }, - Command: "command01", - }, - }, - }, - }, - Runs: map[string]*config.Run{ - "run01": &config.Run{ + Runs: []*config.Run{ + &config.Run{ Name: "run01", - Elements: map[string]*config.Element{ - "element01": &config.Element{ - Name: "element01", - Task: "task01", + Tasks: []*config.Task{ + &config.Task{ + Name: "task01", + Runtime: &config.Runtime{ + Type: "pod", + Auth: &config.RegistryAuth{ + Type: config.RegistryAuthTypeDefault, + Username: config.Value{Type: config.ValueTypeString, Value: "username"}, + Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"}, + }, + Arch: "", + Containers: []*config.Container{ + &config.Container{ + Image: "image01", + }, + }, + }, + Steps: []interface{}{ + &config.RunStep{ + Step: config.Step{ + Type: "run", + Name: "command01", + }, + Command: "command01", + }, + }, }, }, }, @@ -811,9 +792,9 @@ func TestGenRunConfig(t *testing.T) { "password": "yourregistrypassword", }, out: map[string]*rstypes.RunConfigTask{ - uuid.New("element01").String(): &rstypes.RunConfigTask{ - ID: uuid.New("element01").String(), - Name: "element01", Depends: map[string]*rstypes.RunConfigTaskDepend{}, + uuid.New("task01").String(): &rstypes.RunConfigTask{ + ID: uuid.New("task01").String(), + Name: "task01", Depends: map[string]*rstypes.RunConfigTaskDepend{}, Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"), Containers: []*rstypes.Container{ { @@ -837,50 +818,40 @@ func TestGenRunConfig(t *testing.T) { { name: "test runtime auth not used for container with auth", in: &config.Config{ - Runtimes: map[string]*config.Runtime{ - "runtime01": &config.Runtime{ - Name: "runtime01", - Type: "pod", - Auth: &config.RegistryAuth{ - Type: config.RegistryAuthTypeDefault, - Username: config.Value{Type: config.ValueTypeString, Value: "username"}, - Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"}, - }, - Arch: "", - Containers: []*config.Container{ - &config.Container{ - Image: "image01", - Auth: &config.RegistryAuth{ - Type: config.RegistryAuthTypeDefault, - Username: config.Value{Type: config.ValueTypeFromVariable, Value: "registry_username"}, - Password: config.Value{Type: config.ValueTypeString, Value: "password2"}, - }, - }, - }, - }, - }, - Tasks: map[string]*config.Task{ - "task01": &config.Task{ - Name: "task01", - Runtime: "runtime01", - Steps: []interface{}{ - &config.RunStep{ - Step: config.Step{ - Type: "run", - Name: "command01", - }, - Command: "command01", - }, - }, - }, - }, - Runs: map[string]*config.Run{ - "run01": &config.Run{ + Runs: []*config.Run{ + &config.Run{ Name: "run01", - Elements: map[string]*config.Element{ - "element01": &config.Element{ - Name: "element01", - Task: "task01", + Tasks: []*config.Task{ + &config.Task{ + Name: "task01", + Runtime: &config.Runtime{ + Type: "pod", + Auth: &config.RegistryAuth{ + Type: config.RegistryAuthTypeDefault, + Username: config.Value{Type: config.ValueTypeString, Value: "username"}, + Password: config.Value{Type: config.ValueTypeFromVariable, Value: "password"}, + }, + Arch: "", + Containers: []*config.Container{ + &config.Container{ + Image: "image01", + Auth: &config.RegistryAuth{ + Type: config.RegistryAuthTypeDefault, + Username: config.Value{Type: config.ValueTypeFromVariable, Value: "registry_username"}, + Password: config.Value{Type: config.ValueTypeString, Value: "password2"}, + }, + }, + }, + }, + Steps: []interface{}{ + &config.RunStep{ + Step: config.Step{ + Type: "run", + Name: "command01", + }, + Command: "command01", + }, + }, }, }, }, @@ -891,9 +862,9 @@ func TestGenRunConfig(t *testing.T) { "registry_username": "yourregistryusername", }, out: map[string]*rstypes.RunConfigTask{ - uuid.New("element01").String(): &rstypes.RunConfigTask{ - ID: uuid.New("element01").String(), - Name: "element01", Depends: map[string]*rstypes.RunConfigTaskDepend{}, + uuid.New("task01").String(): &rstypes.RunConfigTask{ + ID: uuid.New("task01").String(), + Name: "task01", Depends: map[string]*rstypes.RunConfigTaskDepend{}, Runtime: &rstypes.Runtime{Type: rstypes.RuntimeType("pod"), Containers: []*rstypes.Container{ {