From 103241a8df94097fd9af35dd20770239e8d78615 Mon Sep 17 00:00:00 2001 From: Julien Duchesne Date: Thu, 26 Aug 2021 08:27:33 -0400 Subject: [PATCH 1/2] Fix `--stream` combined with `--format` for `jsonnet` If both are defined and jsonnet defines a list, it currently only generates one of the resources Also added a test for it --- .drone.yml | 5 + drone/jsonnet/convert.go | 96 ++++++++++++++++++++ drone/jsonnet/convert_test.go | 37 ++++++++ drone/jsonnet/jsonnet.go | 78 +--------------- drone/jsonnet/testdata/stream_format.jsonnet | 27 ++++++ drone/jsonnet/testdata/stream_format.yaml | 32 +++++++ go.mod | 1 + 7 files changed, 202 insertions(+), 74 deletions(-) create mode 100644 drone/jsonnet/convert.go create mode 100644 drone/jsonnet/convert_test.go create mode 100644 drone/jsonnet/testdata/stream_format.jsonnet create mode 100644 drone/jsonnet/testdata/stream_format.yaml diff --git a/.drone.yml b/.drone.yml index 6204acb6..1d29e76b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3,6 +3,11 @@ type: docker name: default steps: +- name: test + image: golang:1.16 + commands: + - go test ./... + - name: build image: golang:1.16 commands: diff --git a/drone/jsonnet/convert.go b/drone/jsonnet/convert.go new file mode 100644 index 00000000..fdda45c6 --- /dev/null +++ b/drone/jsonnet/convert.go @@ -0,0 +1,96 @@ +package jsonnet + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/fatih/color" + "github.com/ghodss/yaml" + "github.com/google/go-jsonnet" +) + +func convert(source string, stringOutput bool, format bool, stream bool, vars []string) (string, error) { + data, err := ioutil.ReadFile(source) + if err != nil { + return "", err + } + + vm := jsonnet.MakeVM() + vm.MaxStack = 500 + vm.StringOutput = stringOutput + vm.ErrorFormatter.SetMaxStackTraceSize(20) + vm.ErrorFormatter.SetColorFormatter( + color.New(color.FgRed).Fprintf, + ) + + // register native functions + RegisterNativeFuncs(vm) + + // extVars + for _, v := range vars { + name, value, err := getVarVal(v) + if err != nil { + return "", err + } + vm.ExtVar(name, value) + } + + formatDoc := func(doc []byte) ([]byte, error) { + // enable yaml output + if format { + formatted, yErr := yaml.JSONToYAML(doc) + if yErr != nil { + return nil, fmt.Errorf("failed to convert to YAML: %v", yErr) + } + return formatted, nil + } + return doc, nil + } + + buf := new(bytes.Buffer) + if stream { + docs, err := vm.EvaluateSnippetStream(source, string(data)) + if err != nil { + return "", err + } + for _, doc := range docs { + formatted, err := formatDoc([]byte(doc)) + if err != nil { + return "", err + } + + buf.WriteString("---") + buf.WriteString("\n") + buf.Write(formatted) + } + } else { + result, err := vm.EvaluateSnippet(source, string(data)) + if err != nil { + return "", err + } + formatted, err := formatDoc([]byte(result)) + if err != nil { + return "", err + } + buf.Write(formatted) + } + + return buf.String(), nil +} + +// https://github.com/google/go-jsonnet/blob/master/cmd/jsonnet/cmd.go#L149 +func getVarVal(s string) (string, string, error) { + parts := strings.SplitN(s, "=", 2) + name := parts[0] + if len(parts) == 1 { + content, exists := os.LookupEnv(name) + if exists { + return name, content, nil + } + return "", "", fmt.Errorf("environment variable %v was undefined", name) + } + return name, parts[1], nil +} diff --git a/drone/jsonnet/convert_test.go b/drone/jsonnet/convert_test.go new file mode 100644 index 00000000..0dce8880 --- /dev/null +++ b/drone/jsonnet/convert_test.go @@ -0,0 +1,37 @@ +package jsonnet + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConvert(t *testing.T) { + testcases := []struct { + name string + jsonnetFile, yamlFile string + stringOutput, format, stream bool + extVars []string + }{ + { + name: "Stream + Format", + jsonnetFile: "stream_format.jsonnet", + yamlFile: "stream_format.yaml", + format: true, stream: true, + }, + } + + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + expected, err := os.ReadFile(filepath.Join("./testdata", tc.yamlFile)) + assert.NoError(t, err) + + result, err := convert(filepath.Join("./testdata", tc.jsonnetFile), tc.stringOutput, tc.format, tc.stream, tc.extVars) + assert.NoError(t, err) + assert.Equal(t, string(expected), result) + }) + } +} diff --git a/drone/jsonnet/jsonnet.go b/drone/jsonnet/jsonnet.go index e3812a6c..4ac1cc15 100644 --- a/drone/jsonnet/jsonnet.go +++ b/drone/jsonnet/jsonnet.go @@ -1,17 +1,11 @@ package jsonnet import ( - "bytes" - "fmt" "io" "io/ioutil" "log" "os" - "strings" - "github.com/fatih/color" - "github.com/ghodss/yaml" - "github.com/google/go-jsonnet" "github.com/urfave/cli" ) @@ -60,81 +54,17 @@ var Command = cli.Command{ } func generate(c *cli.Context) error { - source := c.String("source") - target := c.String("target") - - data, err := ioutil.ReadFile(source) + result, err := convert(c.String("source"), c.Bool("string"), c.Bool("format"), c.Bool("stream"), c.StringSlice("extVar")) if err != nil { return err } - vm := jsonnet.MakeVM() - vm.MaxStack = 500 - vm.StringOutput = c.Bool("string") - vm.ErrorFormatter.SetMaxStackTraceSize(20) - vm.ErrorFormatter.SetColorFormatter( - color.New(color.FgRed).Fprintf, - ) - - // register native functions - RegisterNativeFuncs(vm) - - // extVars - vars := c.StringSlice("extVar") - for _, v := range vars { - name, value, err := getVarVal(v) - if err != nil { - return err - } - vm.ExtVar(name, value) - } - - buf := new(bytes.Buffer) - if c.Bool("stream") { - docs, err := vm.EvaluateSnippetStream(source, string(data)) - if err != nil { - return err - } - for _, doc := range docs { - buf.WriteString("---") - buf.WriteString("\n") - buf.WriteString(doc) - } - } else { - result, err := vm.EvaluateSnippet(source, string(data)) - if err != nil { - return err - } - buf.WriteString(result) - } - // enable yaml output - if c.Bool("format") { - formatted, yErr := yaml.JSONToYAML(buf.Bytes()) - if yErr != nil { - return fmt.Errorf("failed to convert to YAML: %v", yErr) - } - buf.Reset() - buf.Write(formatted) - } // the user can optionally write the yaml to stdout. This is useful for debugging purposes without mutating an existing file. if c.Bool("stdout") { - io.Copy(os.Stdout, buf) + io.WriteString(os.Stdout, result) return nil } - return ioutil.WriteFile(target, buf.Bytes(), 0644) -} - -// https://github.com/google/go-jsonnet/blob/master/cmd/jsonnet/cmd.go#L149 -func getVarVal(s string) (string, string, error) { - parts := strings.SplitN(s, "=", 2) - name := parts[0] - if len(parts) == 1 { - content, exists := os.LookupEnv(name) - if exists { - return name, content, nil - } - return "", "", fmt.Errorf("environment variable %v was undefined", name) - } - return name, parts[1], nil + target := c.String("target") + return ioutil.WriteFile(target, []byte(result), 0644) } diff --git a/drone/jsonnet/testdata/stream_format.jsonnet b/drone/jsonnet/testdata/stream_format.jsonnet new file mode 100644 index 00000000..3252c6c7 --- /dev/null +++ b/drone/jsonnet/testdata/stream_format.jsonnet @@ -0,0 +1,27 @@ +local pipeline(name) = + { + kind: 'pipeline', + type: 'docker', + name: name, + platform: { + os: 'linux', + arch: 'amd64', + }, + steps: [ + { + name: 'test', + image: 'golang:1.16', + commands: ['go test ./...'], + }, + { + name: 'build', + image: 'golang:1.16', + commands: ['go build ./...'], + }, + ], + }; + +[ + pipeline('first'), + pipeline('second'), +] diff --git a/drone/jsonnet/testdata/stream_format.yaml b/drone/jsonnet/testdata/stream_format.yaml new file mode 100644 index 00000000..1728a3f4 --- /dev/null +++ b/drone/jsonnet/testdata/stream_format.yaml @@ -0,0 +1,32 @@ +--- +kind: pipeline +name: first +platform: + arch: amd64 + os: linux +steps: +- commands: + - go test ./... + image: golang:1.16 + name: test +- commands: + - go build ./... + image: golang:1.16 + name: build +type: docker +--- +kind: pipeline +name: second +platform: + arch: amd64 + os: linux +steps: +- commands: + - go test ./... + image: golang:1.16 + name: test +- commands: + - go build ./... + image: golang:1.16 + name: build +type: docker diff --git a/go.mod b/go.mod index 2056111b..75e53c8c 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/mattn/go-colorable v0.1.4 github.com/mattn/go-isatty v0.0.11 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 + github.com/stretchr/testify v1.4.0 github.com/urfave/cli v1.20.0 go.starlark.net v0.0.0-20201118183435-e55f603d8c79 golang.org/x/net v0.0.0-20190603091049-60506f45cf65 From 77752ca2591429ed88393f3480149c44c24e363a Mon Sep 17 00:00:00 2001 From: Julien Duchesne Date: Thu, 26 Aug 2021 08:51:19 -0400 Subject: [PATCH 2/2] Minimize diffs by keeping all in one file --- drone/jsonnet/convert.go | 96 ------------------- drone/jsonnet/jsonnet.go | 89 +++++++++++++++++ .../{convert_test.go => jsonnet_test.go} | 0 3 files changed, 89 insertions(+), 96 deletions(-) delete mode 100644 drone/jsonnet/convert.go rename drone/jsonnet/{convert_test.go => jsonnet_test.go} (100%) diff --git a/drone/jsonnet/convert.go b/drone/jsonnet/convert.go deleted file mode 100644 index fdda45c6..00000000 --- a/drone/jsonnet/convert.go +++ /dev/null @@ -1,96 +0,0 @@ -package jsonnet - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "strings" - - "github.com/fatih/color" - "github.com/ghodss/yaml" - "github.com/google/go-jsonnet" -) - -func convert(source string, stringOutput bool, format bool, stream bool, vars []string) (string, error) { - data, err := ioutil.ReadFile(source) - if err != nil { - return "", err - } - - vm := jsonnet.MakeVM() - vm.MaxStack = 500 - vm.StringOutput = stringOutput - vm.ErrorFormatter.SetMaxStackTraceSize(20) - vm.ErrorFormatter.SetColorFormatter( - color.New(color.FgRed).Fprintf, - ) - - // register native functions - RegisterNativeFuncs(vm) - - // extVars - for _, v := range vars { - name, value, err := getVarVal(v) - if err != nil { - return "", err - } - vm.ExtVar(name, value) - } - - formatDoc := func(doc []byte) ([]byte, error) { - // enable yaml output - if format { - formatted, yErr := yaml.JSONToYAML(doc) - if yErr != nil { - return nil, fmt.Errorf("failed to convert to YAML: %v", yErr) - } - return formatted, nil - } - return doc, nil - } - - buf := new(bytes.Buffer) - if stream { - docs, err := vm.EvaluateSnippetStream(source, string(data)) - if err != nil { - return "", err - } - for _, doc := range docs { - formatted, err := formatDoc([]byte(doc)) - if err != nil { - return "", err - } - - buf.WriteString("---") - buf.WriteString("\n") - buf.Write(formatted) - } - } else { - result, err := vm.EvaluateSnippet(source, string(data)) - if err != nil { - return "", err - } - formatted, err := formatDoc([]byte(result)) - if err != nil { - return "", err - } - buf.Write(formatted) - } - - return buf.String(), nil -} - -// https://github.com/google/go-jsonnet/blob/master/cmd/jsonnet/cmd.go#L149 -func getVarVal(s string) (string, string, error) { - parts := strings.SplitN(s, "=", 2) - name := parts[0] - if len(parts) == 1 { - content, exists := os.LookupEnv(name) - if exists { - return name, content, nil - } - return "", "", fmt.Errorf("environment variable %v was undefined", name) - } - return name, parts[1], nil -} diff --git a/drone/jsonnet/jsonnet.go b/drone/jsonnet/jsonnet.go index 4ac1cc15..97f13bbe 100644 --- a/drone/jsonnet/jsonnet.go +++ b/drone/jsonnet/jsonnet.go @@ -1,11 +1,17 @@ package jsonnet import ( + "bytes" + "fmt" "io" "io/ioutil" "log" "os" + "strings" + "github.com/fatih/color" + "github.com/ghodss/yaml" + "github.com/google/go-jsonnet" "github.com/urfave/cli" ) @@ -68,3 +74,86 @@ func generate(c *cli.Context) error { target := c.String("target") return ioutil.WriteFile(target, []byte(result), 0644) } + +func convert(source string, stringOutput bool, format bool, stream bool, vars []string) (string, error) { + data, err := ioutil.ReadFile(source) + if err != nil { + return "", err + } + + vm := jsonnet.MakeVM() + vm.MaxStack = 500 + vm.StringOutput = stringOutput + vm.ErrorFormatter.SetMaxStackTraceSize(20) + vm.ErrorFormatter.SetColorFormatter( + color.New(color.FgRed).Fprintf, + ) + + // register native functions + RegisterNativeFuncs(vm) + + // extVars + for _, v := range vars { + name, value, err := getVarVal(v) + if err != nil { + return "", err + } + vm.ExtVar(name, value) + } + + formatDoc := func(doc []byte) ([]byte, error) { + // enable yaml output + if format { + formatted, yErr := yaml.JSONToYAML(doc) + if yErr != nil { + return nil, fmt.Errorf("failed to convert to YAML: %v", yErr) + } + return formatted, nil + } + return doc, nil + } + + buf := new(bytes.Buffer) + if stream { + docs, err := vm.EvaluateSnippetStream(source, string(data)) + if err != nil { + return "", err + } + for _, doc := range docs { + formatted, err := formatDoc([]byte(doc)) + if err != nil { + return "", err + } + + buf.WriteString("---") + buf.WriteString("\n") + buf.Write(formatted) + } + } else { + result, err := vm.EvaluateSnippet(source, string(data)) + if err != nil { + return "", err + } + formatted, err := formatDoc([]byte(result)) + if err != nil { + return "", err + } + buf.Write(formatted) + } + + return buf.String(), nil +} + +// https://github.com/google/go-jsonnet/blob/master/cmd/jsonnet/cmd.go#L149 +func getVarVal(s string) (string, string, error) { + parts := strings.SplitN(s, "=", 2) + name := parts[0] + if len(parts) == 1 { + content, exists := os.LookupEnv(name) + if exists { + return name, content, nil + } + return "", "", fmt.Errorf("environment variable %v was undefined", name) + } + return name, parts[1], nil +} diff --git a/drone/jsonnet/convert_test.go b/drone/jsonnet/jsonnet_test.go similarity index 100% rename from drone/jsonnet/convert_test.go rename to drone/jsonnet/jsonnet_test.go