From 4c1540685dda3dc3345af7ad7dee6977c80950c3 Mon Sep 17 00:00:00 2001 From: Matthew Rich Date: Tue, 14 May 2024 11:26:05 -0700 Subject: [PATCH] fix generating shasum for tar sourced files. move enc/decoders to separate pkg --- README.md | 3 ++ cmd/cli/main.go | 7 ++-- internal/resource/command.go | 7 ++-- internal/resource/container.go | 7 ++-- internal/resource/container_network.go | 7 ++-- internal/resource/declaration.go | 7 ++-- internal/resource/decoder.go | 43 -------------------- internal/resource/decoder_test.go | 56 -------------------------- internal/resource/document.go | 9 ++++- internal/resource/encoder.go | 42 ------------------- internal/resource/encoder_test.go | 33 --------------- internal/resource/exec.go | 7 ++-- internal/resource/file.go | 10 ++++- internal/resource/http.go | 7 ++-- internal/resource/iptables.go | 5 ++- internal/resource/network_route.go | 7 ++-- internal/resource/package.go | 7 ++-- internal/resource/user.go | 7 ++-- internal/source/decl.go | 3 +- internal/source/docsource.go | 3 +- internal/source/http.go | 3 +- internal/source/tar.go | 3 +- internal/target/decl.go | 7 ++-- 23 files changed, 64 insertions(+), 226 deletions(-) delete mode 100644 internal/resource/decoder.go delete mode 100644 internal/resource/decoder_test.go delete mode 100644 internal/resource/encoder.go delete mode 100644 internal/resource/encoder_test.go diff --git a/README.md b/README.md index d38febf..904d9d5 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,9 @@ Create the resources specified in a resource document HTTP endpoint. `jx apply http://localhost/resources` +Convert a tar archive into resource definitions and apply them (extracts the contents of a tar). + +`jx apply https://gitea.rosskeen.house/doublejynx/jx/archive/v0.2.1.tar.gz` # Read resource state diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 7909604..f111645 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -15,6 +15,7 @@ _ "gopkg.in/yaml.v3" "decl/internal/resource" "decl/internal/source" "decl/internal/target" + "decl/internal/codec" ) const ( @@ -171,7 +172,7 @@ func ApplySubCommand(cmd *flag.FlagSet, output io.Writer) (err error) { if e := cmd.Parse(os.Args[2:]); e != nil { return e } - var encoder resource.Encoder + var encoder codec.Encoder documents := make([]*resource.Document, 0, 100) for _,source := range cmd.Args() { loaded := LoadSourceURI(source) @@ -189,9 +190,9 @@ func ApplySubCommand(cmd *flag.FlagSet, output io.Writer) (err error) { switch *GlobalOformat { case FormatYaml: - encoder = resource.NewYAMLEncoder(output) + encoder = codec.NewYAMLEncoder(output) case FormatJson: - encoder = resource.NewJSONEncoder(output) + encoder = codec.NewJSONEncoder(output) } if *GlobalQuiet { for _, dr := range d.Resources() { diff --git a/internal/resource/command.go b/internal/resource/command.go index 2251eca..945cf5a 100644 --- a/internal/resource/command.go +++ b/internal/resource/command.go @@ -14,6 +14,7 @@ import ( "os/exec" "strings" "text/template" + "decl/internal/codec" ) type CommandExecutor func(value any) ([]byte, error) @@ -71,13 +72,11 @@ func NewCommand() *Command { } func (c *Command) Load(r io.Reader) error { - decoder := NewYAMLDecoder(r) - return decoder.Decode(c) + return codec.NewYAMLDecoder(r).Decode(c) } func (c *Command) LoadDecl(yamlResourceDeclaration string) error { - decoder := NewYAMLStringDecoder(yamlResourceDeclaration) - return decoder.Decode(c) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(c) } func (c *Command) Template(value any) ([]string, error) { diff --git a/internal/resource/container.go b/internal/resource/container.go index f0be2cc..462ba44 100644 --- a/internal/resource/container.go +++ b/internal/resource/container.go @@ -26,6 +26,7 @@ _ "strings" "encoding/json" "io" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) type ContainerClient interface { @@ -171,13 +172,11 @@ func (c *Container) Apply() error { } func (c *Container) Load(r io.Reader) error { - d := NewYAMLDecoder(r) - return d.Decode(c) + return codec.NewYAMLDecoder(r).Decode(c) } func (c *Container) LoadDecl(yamlResourceDeclaration string) error { - d := NewYAMLStringDecoder(yamlResourceDeclaration) - return d.Decode(c) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(c) } func (c *Container) Create(ctx context.Context) error { diff --git a/internal/resource/container_network.go b/internal/resource/container_network.go index fd5bc1c..df5994e 100644 --- a/internal/resource/container_network.go +++ b/internal/resource/container_network.go @@ -23,6 +23,7 @@ _ "strings" "encoding/json" "io" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) type ContainerNetworkClient interface { @@ -134,13 +135,11 @@ func (n *ContainerNetwork) Apply() error { } func (n *ContainerNetwork) Load(r io.Reader) error { - d := NewYAMLDecoder(r) - return d.Decode(n) + return codec.NewYAMLDecoder(r).Decode(n) } func (n *ContainerNetwork) LoadDecl(yamlResourceDeclaration string) error { - d := NewYAMLStringDecoder(yamlResourceDeclaration) - return d.Decode(n) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(n) } func (n *ContainerNetwork) Create(ctx context.Context) error { diff --git a/internal/resource/declaration.go b/internal/resource/declaration.go index 600a674..8f1b266 100644 --- a/internal/resource/declaration.go +++ b/internal/resource/declaration.go @@ -10,6 +10,7 @@ import ( "gopkg.in/yaml.v3" "log/slog" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) type DeclarationType struct { @@ -45,13 +46,11 @@ func (d *Declaration) Clone() *Declaration { } func (d *Declaration) Load(r io.Reader) error { - c := NewYAMLDecoder(r) - return c.Decode(d) + return codec.NewYAMLDecoder(r).Decode(d) } func (d *Declaration) LoadDecl(yamlResourceDeclaration string) error { - c := NewYAMLStringDecoder(yamlResourceDeclaration) - return c.Decode(d) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(d) } func (d *Declaration) NewResource() error { diff --git a/internal/resource/decoder.go b/internal/resource/decoder.go deleted file mode 100644 index f5362df..0000000 --- a/internal/resource/decoder.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package resource - -import ( - "encoding/json" -_ "fmt" -_ "github.com/xeipuuv/gojsonschema" - "gopkg.in/yaml.v3" - "io" -_ "log" - "strings" -) - -//type JSONDecoder json.Decoder - -type Decoder interface { - Decode(v any) error -} - -func NewDecoder() *Decoder { - return nil -} - -func NewJSONDecoder(r io.Reader) Decoder { - return json.NewDecoder(r) -} - -func NewJSONStringDecoder(s string) Decoder { - return json.NewDecoder(strings.NewReader(s)) -} - -func NewYAMLDecoder(r io.Reader) Decoder { - return yaml.NewDecoder(r) -} - -func NewYAMLStringDecoder(s string) Decoder { - return yaml.NewDecoder(strings.NewReader(s)) -} - -func NewProtoBufDecoder(r io.Reader) Decoder { - return nil -} diff --git a/internal/resource/decoder_test.go b/internal/resource/decoder_test.go deleted file mode 100644 index 95f849d..0000000 --- a/internal/resource/decoder_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package resource - -import ( -_ "fmt" - "github.com/stretchr/testify/assert" -_ "log" - "strings" - "testing" -) - -func TestNewYAMLDecoder(t *testing.T) { - e := NewYAMLDecoder(strings.NewReader("")) - assert.NotNil(t, e) -} - -func TestNewDecoderDecodeJSON(t *testing.T) { - decl := `{ - "name": "testuser", - "uid": "12001", - "group": "12001", - "home": "/home/testuser", - "state": "present" -}` - - jsonReader := strings.NewReader(decl) - user := NewUser() - - e := NewJSONDecoder(jsonReader) - assert.NotNil(t, e) - docErr := e.Decode(user) - assert.Nil(t, docErr) - - s := NewSchema(user.Type()) - - validateErr := s.Validate(decl) - assert.Nil(t, validateErr) -} - -func TestNewJSONStringDecoder(t *testing.T) { - decl := `{ - "name": "testuser", - "uid": "12001", - "group": "12001", - "home": "/home/testuser", - "state": "present" -}` - - user := NewUser() - - e := NewJSONStringDecoder(decl) - assert.NotNil(t, e) - docErr := e.Decode(user) - assert.Nil(t, docErr) -} diff --git a/internal/resource/document.go b/internal/resource/document.go index aad2b0d..70f21b2 100644 --- a/internal/resource/document.go +++ b/internal/resource/document.go @@ -11,6 +11,7 @@ _ "fmt" _ "net/url" "github.com/sters/yaml-diff/yamldiff" "strings" + "decl/internal/codec" ) type Document struct { @@ -42,7 +43,7 @@ func (d *Document) Clone() *Document { } func (d *Document) Load(r io.Reader) error { - c := NewYAMLDecoder(r) + c := codec.NewYAMLDecoder(r) return c.Decode(d); } @@ -84,7 +85,7 @@ func (d *Document) Apply() error { } func (d *Document) Generate(w io.Writer) error { - e := NewYAMLEncoder(w) + e := codec.NewYAMLEncoder(w) err := e.Encode(d); if err == nil { return e.Close() @@ -122,6 +123,10 @@ func (d *Document) YAML() ([]byte, error) { } func (d *Document) Diff(with *Document, output io.Writer) (string, error) { + defer func() { + if r := recover(); r != nil { + } + }() slog.Info("Document.Diff()") opts := []yamldiff.DoOptionFunc{} if output == nil { diff --git a/internal/resource/encoder.go b/internal/resource/encoder.go deleted file mode 100644 index 9eb48e0..0000000 --- a/internal/resource/encoder.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package resource - -import ( - "encoding/json" -_ "fmt" -_ "github.com/xeipuuv/gojsonschema" - "gopkg.in/yaml.v3" - "io" -_ "log" -) - -type JSONEncoder json.Encoder - -type Encoder interface { - Encode(v any) error - Close() error -} - -func NewEncoder() *Encoder { - return nil -} - -func NewJSONEncoder(w io.Writer) Encoder { - return (*JSONEncoder)(json.NewEncoder(w)) -} - -func NewYAMLEncoder(w io.Writer) Encoder { - return yaml.NewEncoder(w) -} - -func NewProtoBufEncoder(w io.Writer) Encoder { - return nil -} - -func (j *JSONEncoder) Encode(v any) error { - return (*json.Encoder)(j).Encode(v) -} -func (j *JSONEncoder) Close() error { - return nil -} diff --git a/internal/resource/encoder_test.go b/internal/resource/encoder_test.go deleted file mode 100644 index f4b42e1..0000000 --- a/internal/resource/encoder_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package resource - -import ( - _ "fmt" - "github.com/stretchr/testify/assert" - _ "log" - "strings" - "testing" -) - -func TestNewYAMLEncoder(t *testing.T) { - var yamlDoc strings.Builder - e := NewYAMLEncoder(&yamlDoc) - assert.NotNil(t, e) -} - -func TestNewEncoderEncodeJSON(t *testing.T) { - var jsonDoc strings.Builder - file := NewFile() - file.Path = "foo" - - e := NewJSONEncoder(&jsonDoc) - assert.NotNil(t, e) - docErr := e.Encode(file) - assert.Nil(t, docErr) - - s := NewSchema(file.Type()) - - validateErr := s.Validate(jsonDoc.String()) - assert.Nil(t, validateErr) -} diff --git a/internal/resource/exec.go b/internal/resource/exec.go index 8ff002c..e451aa9 100644 --- a/internal/resource/exec.go +++ b/internal/resource/exec.go @@ -14,6 +14,7 @@ import ( _ "strings" "io" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) type Exec struct { @@ -82,13 +83,11 @@ func (x *Exec) Apply() error { } func (x *Exec) Load(r io.Reader) error { - c := NewYAMLDecoder(r) - return c.Decode(x) + return codec.NewYAMLDecoder(r).Decode(x) } func (x *Exec) LoadDecl(yamlResourceDeclaration string) error { - c := NewYAMLStringDecoder(yamlResourceDeclaration) - return c.Decode(x) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(x) } func (x *Exec) Type() string { return "exec" } diff --git a/internal/resource/file.go b/internal/resource/file.go index 9391e53..1616777 100644 --- a/internal/resource/file.go +++ b/internal/resource/file.go @@ -19,6 +19,7 @@ import ( "time" "crypto/sha256" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) type FileType string @@ -170,7 +171,7 @@ func (f *File) Apply() error { } func (f *File) LoadDecl(yamlResourceDeclaration string) (err error) { - d := NewYAMLStringDecoder(yamlResourceDeclaration) + d := codec.NewYAMLStringDecoder(yamlResourceDeclaration) err = d.Decode(f) if err == nil { f.UpdateContentAttributes() @@ -365,6 +366,13 @@ func (f *File) Read(ctx context.Context) ([]byte, error) { return yaml.Marshal(f) } +func (f *File) SetContent(r io.Reader) error { + fileContent, ioErr := io.ReadAll(r) + f.Content = string(fileContent) + f.Sha256 = fmt.Sprintf("%x", sha256.Sum256(fileContent)) + return ioErr +} + func (f *File) Type() string { return "file" } func (f *FileType) UnmarshalYAML(value *yaml.Node) error { diff --git a/internal/resource/http.go b/internal/resource/http.go index 3562746..4fb51fe 100644 --- a/internal/resource/http.go +++ b/internal/resource/http.go @@ -15,6 +15,7 @@ _ "os" "strings" "log/slog" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) func init() { @@ -121,14 +122,12 @@ func (h *HTTP) Apply() error { } func (h *HTTP) Load(r io.Reader) error { - c := NewYAMLDecoder(r) - return c.Decode(h) + return codec.NewYAMLDecoder(r).Decode(h) } func (h *HTTP) LoadDecl(yamlResourceDeclaration string) error { slog.Info("LoadDecl()", "yaml", yamlResourceDeclaration) - c := NewYAMLStringDecoder(yamlResourceDeclaration) - return c.Decode(h) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(h) } func (h *HTTP) ResolveId(ctx context.Context) string { diff --git a/internal/resource/iptables.go b/internal/resource/iptables.go index 1ce7230..199d015 100644 --- a/internal/resource/iptables.go +++ b/internal/resource/iptables.go @@ -17,6 +17,7 @@ _ "os/exec" "strings" "log/slog" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) func init() { @@ -251,11 +252,11 @@ func (i *Iptable) Apply() error { } func (i *Iptable) Load(r io.Reader) error { - return NewYAMLDecoder(r).Decode(i) + return codec.NewYAMLDecoder(r).Decode(i) } func (i *Iptable) LoadDecl(yamlResourceDeclaration string) error { - return NewYAMLStringDecoder(yamlResourceDeclaration).Decode(i) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(i) } diff --git a/internal/resource/network_route.go b/internal/resource/network_route.go index 8030f8a..186d5e3 100644 --- a/internal/resource/network_route.go +++ b/internal/resource/network_route.go @@ -16,6 +16,7 @@ import ( _ "strconv" "strings" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) func init() { @@ -192,13 +193,11 @@ func (n *NetworkRoute) Apply() error { } func (n *NetworkRoute) Load(r io.Reader) error { - c := NewYAMLDecoder(r) - return c.Decode(n) + return codec.NewYAMLDecoder(r).Decode(n) } func (n *NetworkRoute) LoadDecl(yamlResourceDeclaration string) error { - c := NewYAMLStringDecoder(yamlResourceDeclaration) - return c.Decode(n) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(n) } func (n *NetworkRoute) ResolveId(ctx context.Context) string { diff --git a/internal/resource/package.go b/internal/resource/package.go index 299a231..c86ac75 100644 --- a/internal/resource/package.go +++ b/internal/resource/package.go @@ -16,6 +16,7 @@ import ( "path/filepath" "strings" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) type PackageType string @@ -189,13 +190,11 @@ func (p *Package) Apply() error { } func (p *Package) Load(r io.Reader) error { - c := NewYAMLDecoder(r) - return c.Decode(p) + return codec.NewYAMLDecoder(r).Decode(p) } func (p *Package) LoadDecl(yamlResourceDeclaration string) error { - c := NewYAMLStringDecoder(yamlResourceDeclaration) - return c.Decode(p) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(p) } func (p *Package) Type() string { return "package" } diff --git a/internal/resource/user.go b/internal/resource/user.go index 0d8eafa..7c45fbd 100644 --- a/internal/resource/user.go +++ b/internal/resource/user.go @@ -16,6 +16,7 @@ _ "os" "encoding/json" "errors" "gitea.rosskeen.house/rosskeen.house/machine" + "decl/internal/codec" ) type decodeUser User @@ -148,13 +149,11 @@ func (u *User) Apply() error { } func (u *User) Load(r io.Reader) error { - c := NewYAMLDecoder(r) - return c.Decode(u) + return codec.NewYAMLDecoder(r).Decode(u) } func (u *User) LoadDecl(yamlResourceDeclaration string) error { - c := NewYAMLStringDecoder(yamlResourceDeclaration) - return c.Decode(u) + return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(u) } func (u *User) AddUserCommand(args *[]string) error { diff --git a/internal/source/decl.go b/internal/source/decl.go index 41959d7..2d4b9fe 100644 --- a/internal/source/decl.go +++ b/internal/source/decl.go @@ -11,6 +11,7 @@ _ "gopkg.in/yaml.v3" "path/filepath" "decl/internal/resource" "decl/internal/transport" + "decl/internal/codec" "regexp" _ "os" "io" @@ -73,7 +74,7 @@ func (d *DeclFile) ExtractResources(filter ResourceSelector) ([]*resource.Docume fileReader = d.transport } - decoder := resource.NewYAMLDecoder(fileReader) + decoder := codec.NewYAMLDecoder(fileReader) slog.Info("ExtractResources()", "documents", documents) index := 0 for { diff --git a/internal/source/docsource.go b/internal/source/docsource.go index 41197c2..06ee55b 100644 --- a/internal/source/docsource.go +++ b/internal/source/docsource.go @@ -17,6 +17,7 @@ _ "strings" "errors" "path/filepath" "decl/internal/resource" + "decl/internal/codec" ) type ResourceSelector func(r resource.Resource) bool @@ -91,7 +92,7 @@ func ExtractResources(uri string, filter ResourceSelector) ([]*resource.Document d.AddResourceDeclaration("file", f) } default: - decoder := resource.NewYAMLDecoder(file) + decoder := codec.NewYAMLDecoder(file) index := 0 for { doc := documents[index] diff --git a/internal/source/http.go b/internal/source/http.go index 6f71577..8d0dec5 100644 --- a/internal/source/http.go +++ b/internal/source/http.go @@ -14,6 +14,7 @@ _ "path/filepath" "decl/internal/iofilter" "decl/internal/signature" "decl/internal/transport" + "decl/internal/codec" _ "os" "io" "errors" @@ -52,7 +53,7 @@ func (h *HTTP) ExtractResources(filter ResourceSelector) ([]*resource.Document, return }) - decoder := resource.NewYAMLDecoder(sumReadData) + decoder := codec.NewYAMLDecoder(sumReadData) index := 0 for { doc := resource.NewDocument() diff --git a/internal/source/tar.go b/internal/source/tar.go index e8dfe66..35afff7 100644 --- a/internal/source/tar.go +++ b/internal/source/tar.go @@ -92,11 +92,10 @@ func (t *Tar) ExtractResources(filter ResourceSelector) ([]*resource.Document, e if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil { return documents, fiErr } - readFileData, readErr := io.ReadAll(tarReader) + readErr := f.SetContent(tarReader) if readErr != nil { return documents, readErr } - f.Content = string(readFileData) d.AddResourceDeclaration("file", f) } } diff --git a/internal/target/decl.go b/internal/target/decl.go index 8784e07..2c0fadc 100644 --- a/internal/target/decl.go +++ b/internal/target/decl.go @@ -10,6 +10,7 @@ _ "gopkg.in/yaml.v3" "net/url" "path/filepath" "decl/internal/resource" + "decl/internal/codec" "os" "compress/gzip" "io" @@ -26,7 +27,7 @@ type DeclFile struct { Path string `yaml:"path" json:"path"` Gzip bool `yaml:"gzip,omitempty" json:"gzip,omitempty"` Format string `yaml:"format,omitempty" json:"format,omitempty"` - encoder resource.Encoder `yaml:"-" json:"-"` + encoder codec.Encoder `yaml:"-" json:"-"` closer func() error `yaml:"-" json:"-"` } @@ -138,11 +139,11 @@ func (d *DeclFile) Open() error { switch d.Format { case FormatJson: - d.encoder = resource.NewJSONEncoder(fileWriter) + d.encoder = codec.NewJSONEncoder(fileWriter) case FormatYaml: fallthrough default: - d.encoder = resource.NewYAMLEncoder(fileWriter) + d.encoder = codec.NewYAMLEncoder(fileWriter) } return nil