fix generating shasum for tar sourced files. move enc/decoders to separate pkg
Some checks failed
Lint / golangci-lint (push) Failing after 9m49s
Declarative Tests / test (push) Successful in 1m19s

This commit is contained in:
Matthew Rich 2024-05-14 11:26:05 -07:00
parent ab46f7d595
commit 4c1540685d
23 changed files with 64 additions and 226 deletions

View File

@ -30,6 +30,9 @@ Create the resources specified in a resource document HTTP endpoint.
`jx apply http://localhost/resources` `jx apply http://localhost/resources`
Convert a tar archive into resource definitions and apply them (extracts the contents of a tar).
`jx apply https://gitea.rosskeen.house/doublejynx/jx/archive/v0.2.1.tar.gz`
# Read resource state # Read resource state

View File

@ -15,6 +15,7 @@ _ "gopkg.in/yaml.v3"
"decl/internal/resource" "decl/internal/resource"
"decl/internal/source" "decl/internal/source"
"decl/internal/target" "decl/internal/target"
"decl/internal/codec"
) )
const ( const (
@ -171,7 +172,7 @@ func ApplySubCommand(cmd *flag.FlagSet, output io.Writer) (err error) {
if e := cmd.Parse(os.Args[2:]); e != nil { if e := cmd.Parse(os.Args[2:]); e != nil {
return e return e
} }
var encoder resource.Encoder var encoder codec.Encoder
documents := make([]*resource.Document, 0, 100) documents := make([]*resource.Document, 0, 100)
for _,source := range cmd.Args() { for _,source := range cmd.Args() {
loaded := LoadSourceURI(source) loaded := LoadSourceURI(source)
@ -189,9 +190,9 @@ func ApplySubCommand(cmd *flag.FlagSet, output io.Writer) (err error) {
switch *GlobalOformat { switch *GlobalOformat {
case FormatYaml: case FormatYaml:
encoder = resource.NewYAMLEncoder(output) encoder = codec.NewYAMLEncoder(output)
case FormatJson: case FormatJson:
encoder = resource.NewJSONEncoder(output) encoder = codec.NewJSONEncoder(output)
} }
if *GlobalQuiet { if *GlobalQuiet {
for _, dr := range d.Resources() { for _, dr := range d.Resources() {

View File

@ -14,6 +14,7 @@ import (
"os/exec" "os/exec"
"strings" "strings"
"text/template" "text/template"
"decl/internal/codec"
) )
type CommandExecutor func(value any) ([]byte, error) type CommandExecutor func(value any) ([]byte, error)
@ -71,13 +72,11 @@ func NewCommand() *Command {
} }
func (c *Command) Load(r io.Reader) error { func (c *Command) Load(r io.Reader) error {
decoder := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(c)
return decoder.Decode(c)
} }
func (c *Command) LoadDecl(yamlResourceDeclaration string) error { func (c *Command) LoadDecl(yamlResourceDeclaration string) error {
decoder := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(c)
return decoder.Decode(c)
} }
func (c *Command) Template(value any) ([]string, error) { func (c *Command) Template(value any) ([]string, error) {

View File

@ -26,6 +26,7 @@ _ "strings"
"encoding/json" "encoding/json"
"io" "io"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
type ContainerClient interface { type ContainerClient interface {
@ -171,13 +172,11 @@ func (c *Container) Apply() error {
} }
func (c *Container) Load(r io.Reader) error { func (c *Container) Load(r io.Reader) error {
d := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(c)
return d.Decode(c)
} }
func (c *Container) LoadDecl(yamlResourceDeclaration string) error { func (c *Container) LoadDecl(yamlResourceDeclaration string) error {
d := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(c)
return d.Decode(c)
} }
func (c *Container) Create(ctx context.Context) error { func (c *Container) Create(ctx context.Context) error {

View File

@ -23,6 +23,7 @@ _ "strings"
"encoding/json" "encoding/json"
"io" "io"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
type ContainerNetworkClient interface { type ContainerNetworkClient interface {
@ -134,13 +135,11 @@ func (n *ContainerNetwork) Apply() error {
} }
func (n *ContainerNetwork) Load(r io.Reader) error { func (n *ContainerNetwork) Load(r io.Reader) error {
d := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(n)
return d.Decode(n)
} }
func (n *ContainerNetwork) LoadDecl(yamlResourceDeclaration string) error { func (n *ContainerNetwork) LoadDecl(yamlResourceDeclaration string) error {
d := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(n)
return d.Decode(n)
} }
func (n *ContainerNetwork) Create(ctx context.Context) error { func (n *ContainerNetwork) Create(ctx context.Context) error {

View File

@ -10,6 +10,7 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"log/slog" "log/slog"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
type DeclarationType struct { type DeclarationType struct {
@ -45,13 +46,11 @@ func (d *Declaration) Clone() *Declaration {
} }
func (d *Declaration) Load(r io.Reader) error { func (d *Declaration) Load(r io.Reader) error {
c := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(d)
return c.Decode(d)
} }
func (d *Declaration) LoadDecl(yamlResourceDeclaration string) error { func (d *Declaration) LoadDecl(yamlResourceDeclaration string) error {
c := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(d)
return c.Decode(d)
} }
func (d *Declaration) NewResource() error { func (d *Declaration) NewResource() error {

View File

@ -1,43 +0,0 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package resource
import (
"encoding/json"
_ "fmt"
_ "github.com/xeipuuv/gojsonschema"
"gopkg.in/yaml.v3"
"io"
_ "log"
"strings"
)
//type JSONDecoder json.Decoder
type Decoder interface {
Decode(v any) error
}
func NewDecoder() *Decoder {
return nil
}
func NewJSONDecoder(r io.Reader) Decoder {
return json.NewDecoder(r)
}
func NewJSONStringDecoder(s string) Decoder {
return json.NewDecoder(strings.NewReader(s))
}
func NewYAMLDecoder(r io.Reader) Decoder {
return yaml.NewDecoder(r)
}
func NewYAMLStringDecoder(s string) Decoder {
return yaml.NewDecoder(strings.NewReader(s))
}
func NewProtoBufDecoder(r io.Reader) Decoder {
return nil
}

View File

@ -1,56 +0,0 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package resource
import (
_ "fmt"
"github.com/stretchr/testify/assert"
_ "log"
"strings"
"testing"
)
func TestNewYAMLDecoder(t *testing.T) {
e := NewYAMLDecoder(strings.NewReader(""))
assert.NotNil(t, e)
}
func TestNewDecoderDecodeJSON(t *testing.T) {
decl := `{
"name": "testuser",
"uid": "12001",
"group": "12001",
"home": "/home/testuser",
"state": "present"
}`
jsonReader := strings.NewReader(decl)
user := NewUser()
e := NewJSONDecoder(jsonReader)
assert.NotNil(t, e)
docErr := e.Decode(user)
assert.Nil(t, docErr)
s := NewSchema(user.Type())
validateErr := s.Validate(decl)
assert.Nil(t, validateErr)
}
func TestNewJSONStringDecoder(t *testing.T) {
decl := `{
"name": "testuser",
"uid": "12001",
"group": "12001",
"home": "/home/testuser",
"state": "present"
}`
user := NewUser()
e := NewJSONStringDecoder(decl)
assert.NotNil(t, e)
docErr := e.Decode(user)
assert.Nil(t, docErr)
}

View File

@ -11,6 +11,7 @@ _ "fmt"
_ "net/url" _ "net/url"
"github.com/sters/yaml-diff/yamldiff" "github.com/sters/yaml-diff/yamldiff"
"strings" "strings"
"decl/internal/codec"
) )
type Document struct { type Document struct {
@ -42,7 +43,7 @@ func (d *Document) Clone() *Document {
} }
func (d *Document) Load(r io.Reader) error { func (d *Document) Load(r io.Reader) error {
c := NewYAMLDecoder(r) c := codec.NewYAMLDecoder(r)
return c.Decode(d); return c.Decode(d);
} }
@ -84,7 +85,7 @@ func (d *Document) Apply() error {
} }
func (d *Document) Generate(w io.Writer) error { func (d *Document) Generate(w io.Writer) error {
e := NewYAMLEncoder(w) e := codec.NewYAMLEncoder(w)
err := e.Encode(d); err := e.Encode(d);
if err == nil { if err == nil {
return e.Close() return e.Close()
@ -122,6 +123,10 @@ func (d *Document) YAML() ([]byte, error) {
} }
func (d *Document) Diff(with *Document, output io.Writer) (string, error) { func (d *Document) Diff(with *Document, output io.Writer) (string, error) {
defer func() {
if r := recover(); r != nil {
}
}()
slog.Info("Document.Diff()") slog.Info("Document.Diff()")
opts := []yamldiff.DoOptionFunc{} opts := []yamldiff.DoOptionFunc{}
if output == nil { if output == nil {

View File

@ -1,42 +0,0 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package resource
import (
"encoding/json"
_ "fmt"
_ "github.com/xeipuuv/gojsonschema"
"gopkg.in/yaml.v3"
"io"
_ "log"
)
type JSONEncoder json.Encoder
type Encoder interface {
Encode(v any) error
Close() error
}
func NewEncoder() *Encoder {
return nil
}
func NewJSONEncoder(w io.Writer) Encoder {
return (*JSONEncoder)(json.NewEncoder(w))
}
func NewYAMLEncoder(w io.Writer) Encoder {
return yaml.NewEncoder(w)
}
func NewProtoBufEncoder(w io.Writer) Encoder {
return nil
}
func (j *JSONEncoder) Encode(v any) error {
return (*json.Encoder)(j).Encode(v)
}
func (j *JSONEncoder) Close() error {
return nil
}

View File

@ -1,33 +0,0 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package resource
import (
_ "fmt"
"github.com/stretchr/testify/assert"
_ "log"
"strings"
"testing"
)
func TestNewYAMLEncoder(t *testing.T) {
var yamlDoc strings.Builder
e := NewYAMLEncoder(&yamlDoc)
assert.NotNil(t, e)
}
func TestNewEncoderEncodeJSON(t *testing.T) {
var jsonDoc strings.Builder
file := NewFile()
file.Path = "foo"
e := NewJSONEncoder(&jsonDoc)
assert.NotNil(t, e)
docErr := e.Encode(file)
assert.Nil(t, docErr)
s := NewSchema(file.Type())
validateErr := s.Validate(jsonDoc.String())
assert.Nil(t, validateErr)
}

View File

@ -14,6 +14,7 @@ import (
_ "strings" _ "strings"
"io" "io"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
type Exec struct { type Exec struct {
@ -82,13 +83,11 @@ func (x *Exec) Apply() error {
} }
func (x *Exec) Load(r io.Reader) error { func (x *Exec) Load(r io.Reader) error {
c := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(x)
return c.Decode(x)
} }
func (x *Exec) LoadDecl(yamlResourceDeclaration string) error { func (x *Exec) LoadDecl(yamlResourceDeclaration string) error {
c := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(x)
return c.Decode(x)
} }
func (x *Exec) Type() string { return "exec" } func (x *Exec) Type() string { return "exec" }

View File

@ -19,6 +19,7 @@ import (
"time" "time"
"crypto/sha256" "crypto/sha256"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
type FileType string type FileType string
@ -170,7 +171,7 @@ func (f *File) Apply() error {
} }
func (f *File) LoadDecl(yamlResourceDeclaration string) (err error) { func (f *File) LoadDecl(yamlResourceDeclaration string) (err error) {
d := NewYAMLStringDecoder(yamlResourceDeclaration) d := codec.NewYAMLStringDecoder(yamlResourceDeclaration)
err = d.Decode(f) err = d.Decode(f)
if err == nil { if err == nil {
f.UpdateContentAttributes() f.UpdateContentAttributes()
@ -365,6 +366,13 @@ func (f *File) Read(ctx context.Context) ([]byte, error) {
return yaml.Marshal(f) return yaml.Marshal(f)
} }
func (f *File) SetContent(r io.Reader) error {
fileContent, ioErr := io.ReadAll(r)
f.Content = string(fileContent)
f.Sha256 = fmt.Sprintf("%x", sha256.Sum256(fileContent))
return ioErr
}
func (f *File) Type() string { return "file" } func (f *File) Type() string { return "file" }
func (f *FileType) UnmarshalYAML(value *yaml.Node) error { func (f *FileType) UnmarshalYAML(value *yaml.Node) error {

View File

@ -15,6 +15,7 @@ _ "os"
"strings" "strings"
"log/slog" "log/slog"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
func init() { func init() {
@ -121,14 +122,12 @@ func (h *HTTP) Apply() error {
} }
func (h *HTTP) Load(r io.Reader) error { func (h *HTTP) Load(r io.Reader) error {
c := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(h)
return c.Decode(h)
} }
func (h *HTTP) LoadDecl(yamlResourceDeclaration string) error { func (h *HTTP) LoadDecl(yamlResourceDeclaration string) error {
slog.Info("LoadDecl()", "yaml", yamlResourceDeclaration) slog.Info("LoadDecl()", "yaml", yamlResourceDeclaration)
c := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(h)
return c.Decode(h)
} }
func (h *HTTP) ResolveId(ctx context.Context) string { func (h *HTTP) ResolveId(ctx context.Context) string {

View File

@ -17,6 +17,7 @@ _ "os/exec"
"strings" "strings"
"log/slog" "log/slog"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
func init() { func init() {
@ -251,11 +252,11 @@ func (i *Iptable) Apply() error {
} }
func (i *Iptable) Load(r io.Reader) error { func (i *Iptable) Load(r io.Reader) error {
return NewYAMLDecoder(r).Decode(i) return codec.NewYAMLDecoder(r).Decode(i)
} }
func (i *Iptable) LoadDecl(yamlResourceDeclaration string) error { func (i *Iptable) LoadDecl(yamlResourceDeclaration string) error {
return NewYAMLStringDecoder(yamlResourceDeclaration).Decode(i) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(i)
} }

View File

@ -16,6 +16,7 @@ import (
_ "strconv" _ "strconv"
"strings" "strings"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
func init() { func init() {
@ -192,13 +193,11 @@ func (n *NetworkRoute) Apply() error {
} }
func (n *NetworkRoute) Load(r io.Reader) error { func (n *NetworkRoute) Load(r io.Reader) error {
c := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(n)
return c.Decode(n)
} }
func (n *NetworkRoute) LoadDecl(yamlResourceDeclaration string) error { func (n *NetworkRoute) LoadDecl(yamlResourceDeclaration string) error {
c := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(n)
return c.Decode(n)
} }
func (n *NetworkRoute) ResolveId(ctx context.Context) string { func (n *NetworkRoute) ResolveId(ctx context.Context) string {

View File

@ -16,6 +16,7 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
type PackageType string type PackageType string
@ -189,13 +190,11 @@ func (p *Package) Apply() error {
} }
func (p *Package) Load(r io.Reader) error { func (p *Package) Load(r io.Reader) error {
c := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(p)
return c.Decode(p)
} }
func (p *Package) LoadDecl(yamlResourceDeclaration string) error { func (p *Package) LoadDecl(yamlResourceDeclaration string) error {
c := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(p)
return c.Decode(p)
} }
func (p *Package) Type() string { return "package" } func (p *Package) Type() string { return "package" }

View File

@ -16,6 +16,7 @@ _ "os"
"encoding/json" "encoding/json"
"errors" "errors"
"gitea.rosskeen.house/rosskeen.house/machine" "gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
) )
type decodeUser User type decodeUser User
@ -148,13 +149,11 @@ func (u *User) Apply() error {
} }
func (u *User) Load(r io.Reader) error { func (u *User) Load(r io.Reader) error {
c := NewYAMLDecoder(r) return codec.NewYAMLDecoder(r).Decode(u)
return c.Decode(u)
} }
func (u *User) LoadDecl(yamlResourceDeclaration string) error { func (u *User) LoadDecl(yamlResourceDeclaration string) error {
c := NewYAMLStringDecoder(yamlResourceDeclaration) return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(u)
return c.Decode(u)
} }
func (u *User) AddUserCommand(args *[]string) error { func (u *User) AddUserCommand(args *[]string) error {

View File

@ -11,6 +11,7 @@ _ "gopkg.in/yaml.v3"
"path/filepath" "path/filepath"
"decl/internal/resource" "decl/internal/resource"
"decl/internal/transport" "decl/internal/transport"
"decl/internal/codec"
"regexp" "regexp"
_ "os" _ "os"
"io" "io"
@ -73,7 +74,7 @@ func (d *DeclFile) ExtractResources(filter ResourceSelector) ([]*resource.Docume
fileReader = d.transport fileReader = d.transport
} }
decoder := resource.NewYAMLDecoder(fileReader) decoder := codec.NewYAMLDecoder(fileReader)
slog.Info("ExtractResources()", "documents", documents) slog.Info("ExtractResources()", "documents", documents)
index := 0 index := 0
for { for {

View File

@ -17,6 +17,7 @@ _ "strings"
"errors" "errors"
"path/filepath" "path/filepath"
"decl/internal/resource" "decl/internal/resource"
"decl/internal/codec"
) )
type ResourceSelector func(r resource.Resource) bool type ResourceSelector func(r resource.Resource) bool
@ -91,7 +92,7 @@ func ExtractResources(uri string, filter ResourceSelector) ([]*resource.Document
d.AddResourceDeclaration("file", f) d.AddResourceDeclaration("file", f)
} }
default: default:
decoder := resource.NewYAMLDecoder(file) decoder := codec.NewYAMLDecoder(file)
index := 0 index := 0
for { for {
doc := documents[index] doc := documents[index]

View File

@ -14,6 +14,7 @@ _ "path/filepath"
"decl/internal/iofilter" "decl/internal/iofilter"
"decl/internal/signature" "decl/internal/signature"
"decl/internal/transport" "decl/internal/transport"
"decl/internal/codec"
_ "os" _ "os"
"io" "io"
"errors" "errors"
@ -52,7 +53,7 @@ func (h *HTTP) ExtractResources(filter ResourceSelector) ([]*resource.Document,
return return
}) })
decoder := resource.NewYAMLDecoder(sumReadData) decoder := codec.NewYAMLDecoder(sumReadData)
index := 0 index := 0
for { for {
doc := resource.NewDocument() doc := resource.NewDocument()

View File

@ -92,11 +92,10 @@ func (t *Tar) ExtractResources(filter ResourceSelector) ([]*resource.Document, e
if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil { if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil {
return documents, fiErr return documents, fiErr
} }
readFileData, readErr := io.ReadAll(tarReader) readErr := f.SetContent(tarReader)
if readErr != nil { if readErr != nil {
return documents, readErr return documents, readErr
} }
f.Content = string(readFileData)
d.AddResourceDeclaration("file", f) d.AddResourceDeclaration("file", f)
} }
} }

View File

@ -10,6 +10,7 @@ _ "gopkg.in/yaml.v3"
"net/url" "net/url"
"path/filepath" "path/filepath"
"decl/internal/resource" "decl/internal/resource"
"decl/internal/codec"
"os" "os"
"compress/gzip" "compress/gzip"
"io" "io"
@ -26,7 +27,7 @@ type DeclFile struct {
Path string `yaml:"path" json:"path"` Path string `yaml:"path" json:"path"`
Gzip bool `yaml:"gzip,omitempty" json:"gzip,omitempty"` Gzip bool `yaml:"gzip,omitempty" json:"gzip,omitempty"`
Format string `yaml:"format,omitempty" json:"format,omitempty"` Format string `yaml:"format,omitempty" json:"format,omitempty"`
encoder resource.Encoder `yaml:"-" json:"-"` encoder codec.Encoder `yaml:"-" json:"-"`
closer func() error `yaml:"-" json:"-"` closer func() error `yaml:"-" json:"-"`
} }
@ -138,11 +139,11 @@ func (d *DeclFile) Open() error {
switch d.Format { switch d.Format {
case FormatJson: case FormatJson:
d.encoder = resource.NewJSONEncoder(fileWriter) d.encoder = codec.NewJSONEncoder(fileWriter)
case FormatYaml: case FormatYaml:
fallthrough fallthrough
default: default:
d.encoder = resource.NewYAMLEncoder(fileWriter) d.encoder = codec.NewYAMLEncoder(fileWriter)
} }
return nil return nil