From 4c30a5af1c14f874f7282ea980ccc12503517186 Mon Sep 17 00:00:00 2001 From: Simone Gotti Date: Tue, 16 Apr 2019 15:00:37 +0200 Subject: [PATCH] runservice: handle jsonnet and json config files Handle config files with name `config.jsonnet`, `config.json` and `config.yml` and take the first from the repository in this order For a jsonnet file execute it and use the generated output as the config --- go.mod | 2 ++ go.sum | 7 +++- internal/config/config.go | 22 +++++++++++- internal/config/config_test.go | 4 +-- internal/services/gateway/webhook.go | 54 ++++++++++++++++++++-------- 5 files changed, 71 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 8cc5bf2..c7b4f3f 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/go-ini/ini v1.42.0 // indirect github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/google/go-cmp v0.3.0 + github.com/google/go-jsonnet v0.12.1 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect github.com/gorilla/handlers v1.4.0 github.com/gorilla/mux v1.7.0 @@ -38,6 +39,7 @@ require ( github.com/pkg/errors v0.8.0 github.com/sanity-io/litter v1.1.0 github.com/satori/go.uuid v1.2.0 + github.com/sergi/go-diff v1.0.0 // indirect github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c // indirect diff --git a/go.sum b/go.sum index d11497a..a1ea618 100644 --- a/go.sum +++ b/go.sum @@ -62,8 +62,12 @@ github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu4i+jD32SE9jQXyfnOvwhHqlT0= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-jsonnet v0.12.1 h1:v0iUm/b4SBz7lR/diMoz9tLAz8lqtnNRKIwMrmU2HEU= +github.com/google/go-jsonnet v0.12.1/go.mod h1:gVu3UVSfOt5fRFq+dh9duBqXa5905QY8S1QvMNcEIVs= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= @@ -124,7 +128,6 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -154,6 +157,8 @@ github.com/sanity-io/litter v1.1.0 h1:BllcKWa3VbZmOZbDCoszYLk7zCsKHz5Beossi8SUcT github.com/sanity-io/litter v1.1.0/go.mod h1:CJ0VCw2q4qKU7LaQr3n7UOSHzgEMgcGco7N/SkZQPjw= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a h1:u7WP9TGHJIkJoi/dRDhvYPSthMIdUQPDETiZET/Utl8= github.com/sgotti/gexpect v0.0.0-20161123102107-0afc6c19f50a/go.mod h1:HvB0+YQff1QGS1nct9E3/J8wo8s/EVjq+VXrJSDlQEY= github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= diff --git a/internal/config/config.go b/internal/config/config.go index 196458f..241c9e1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,6 +25,7 @@ import ( "github.com/sorintlab/agola/internal/util" "github.com/ghodss/yaml" + "github.com/google/go-jsonnet" "github.com/pkg/errors" ) @@ -34,6 +35,14 @@ const ( maxStepNameLength = 100 ) +type ConfigFormat int + +const ( + // ConfigFormatJSON handles both json or yaml format (since json is a subset of yaml) + ConfigFormatJSON ConfigFormat = iota + ConfigFormatJsonnet +) + var ( regExpDelimiters = []string{"/", "#"} ) @@ -595,7 +604,18 @@ func (r *Run) Task(taskName string) *Task { var DefaultConfig = Config{} -func ParseConfig(configData []byte) (*Config, error) { +func ParseConfig(configData []byte, format ConfigFormat) (*Config, error) { + // Generate json from jsonnet + if format == ConfigFormatJsonnet { + // TODO(sgotti) support custom import files inside the configdir ??? + vm := jsonnet.MakeVM() + out, err := vm.EvaluateSnippet("", string(configData)) + if err != nil { + return nil, errors.Wrapf(err, "failed to evaluate jsonnet config") + } + configData = []byte(out) + } + config := DefaultConfig if err := yaml.Unmarshal(configData, &config); err != nil { return nil, errors.Wrapf(err, "failed to unmarshal config") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6b35ff0..d58e25f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -109,7 +109,7 @@ func TestParseConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if _, err := ParseConfig([]byte(tt.in)); err != nil { + if _, err := ParseConfig([]byte(tt.in), ConfigFormatJSON); err != nil { if tt.err == nil { t.Fatalf("got error: %v, expected no error", err) } @@ -417,7 +417,7 @@ func TestParseOutput(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - out, err := ParseConfig([]byte(tt.in)) + out, err := ParseConfig([]byte(tt.in), ConfigFormatJSON) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/services/gateway/webhook.go b/internal/services/gateway/webhook.go index 10c4043..048d145 100644 --- a/internal/services/gateway/webhook.go +++ b/internal/services/gateway/webhook.go @@ -39,7 +39,10 @@ import ( const ( defaultSSHPort = "22" - agolaDefaultConfigPath = ".agola/config.yml" + agolaDefaultConfigDir = ".agola" + agolaDefaultJsonnetConfigFile = "config.jsonnet" + agolaDefaultJsonConfigFile = "config.json" + agolaDefaultYamlConfigFile = "config.yml" // List of runs annotations AnnotationEventType = "event_type" @@ -252,16 +255,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { h.log.Infof("webhookData: %s", util.Dump(webhookData)) - var data []byte - err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) { - var err error - data, err = gitSource.GetFile(webhookData.Repo.Path, webhookData.CommitSHA, agolaDefaultConfigPath) - if err == nil { - return true, nil - } - h.log.Errorf("get file err: %v", err) - return false, nil - }) + data, filename, err := h.fetchConfigFiles(gitSource, webhookData) if err != nil { return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to fetch config file") } @@ -330,7 +324,7 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { group = genGroup(GroupTypeUser, userID, webhookData) } - if err := h.createRuns(ctx, data, group, annotations, env, variables, webhookData); err != nil { + if err := h.createRuns(ctx, filename, data, group, annotations, env, variables, webhookData); err != nil { return http.StatusInternalServerError, "", errors.Wrapf(err, "failed to create run") } //if err := gitSource.CreateStatus(webhookData.Repo.Owner, webhookData.Repo.Name, webhookData.CommitSHA, gitsource.CommitStatusPending, "localhost:8080", "build %s", "agola"); err != nil { @@ -340,10 +334,42 @@ func (h *webhooksHandler) handleWebhook(r *http.Request) (int, string, error) { return 0, "", nil } -func (h *webhooksHandler) createRuns(ctx context.Context, configData []byte, group string, annotations, staticEnv, variables map[string]string, webhookData *types.WebhookData) error { +// fetchConfigFiles tries to fetch a config file in one of the supported formats. The precedence is for jsonnet, then json and then yml +// TODO(sgotti) For jsonnet, if we'll support custom import files inside the configdir, also fetch them. +func (h *webhooksHandler) fetchConfigFiles(gitSource gitsource.GitSource, webhookData *types.WebhookData) ([]byte, string, error) { + var data []byte + var filename string + err := util.ExponentialBackoff(util.FetchFileBackoff, func() (bool, error) { + for _, filename = range []string{agolaDefaultJsonnetConfigFile, agolaDefaultJsonConfigFile, agolaDefaultYamlConfigFile} { + var err error + data, err = gitSource.GetFile(webhookData.Repo.Path, webhookData.CommitSHA, path.Join(agolaDefaultConfigDir, filename)) + if err == nil { + return true, nil + } + h.log.Errorf("get file err: %v", err) + } + return false, nil + }) + if err != nil { + return nil, "", err + } + return data, filename, nil +} + +func (h *webhooksHandler) createRuns(ctx context.Context, filename string, configData []byte, group string, annotations, staticEnv, variables map[string]string, webhookData *types.WebhookData) error { setupErrors := []string{} - config, err := config.ParseConfig([]byte(configData)) + var configFormat config.ConfigFormat + switch path.Ext(filename) { + case ".jsonnet": + configFormat = config.ConfigFormatJsonnet + case ".json": + fallthrough + case ".yml": + configFormat = config.ConfigFormatJSON + + } + config, err := config.ParseConfig([]byte(configData), configFormat) if err != nil { log.Errorf("failed to parse config: %+v", err)