update resource schemas
Some checks failed
Lint / golangci-lint (push) Failing after 9m53s
Declarative Tests / test (push) Failing after 58s

This commit is contained in:
Matthew Rich 2024-04-09 12:30:05 -07:00
parent dc843d98a7
commit 33dd463180
24 changed files with 495 additions and 66 deletions

View File

@ -55,6 +55,9 @@ func main() {
if e := d.Load(resourceFile); e != nil {
log.Fatal(e)
}
if validationErr := d.Validate(); validationErr != nil {
log.Fatal(validationErr)
}
if applyErr := d.Apply(); applyErr != nil {
log.Fatal(applyErr)
}

View File

@ -3,28 +3,50 @@
package resource
import (
_ "context"
_ "context"
"encoding/json"
"fmt"
"gopkg.in/yaml.v3"
_ "log"
_ "net/url"
_ "os"
"io"
"log/slog"
_ "net/url"
_ "os"
"os/exec"
"strings"
"text/template"
"io"
"encoding/json"
)
type CommandExecutor func(value any) ([]byte, error)
type CommandExtractAttributes func(output []byte, target any) error
type CommandArg string
type Command struct {
Path string `json:"path" yaml:"path"`
Args []CommandArg `json:"args" yaml:"args"`
Path string `json:"path" yaml:"path"`
Args []CommandArg `json:"args" yaml:"args"`
Executor CommandExecutor `json:"-" yaml:"-"`
Extractor CommandExtractAttributes `json:"-" yaml:"-"`
}
func NewCommand() *Command {
return &Command{}
c := &Command{}
c.Executor = func(value any) ([]byte, error) {
args, err := c.Template(value)
if err != nil {
return nil, err
}
cmd := exec.Command(c.Path, args...)
stderr, pipeErr := cmd.StderrPipe()
if pipeErr != nil {
return nil, pipeErr
}
output, err := cmd.Output()
stdErrOutput, _ := io.ReadAll(stderr)
slog.Info("execute()", "path", c.Path, "args", args, "output", output, "error", stdErrOutput)
return output, err
}
return c
}
func (c *Command) Load(r io.Reader) error {
@ -51,11 +73,7 @@ func (c *Command) Template(value any) ([]string, error) {
}
func (c *Command) Execute(value any) ([]byte, error) {
args, err := c.Template(value)
if err != nil {
return nil, err
}
return exec.Command(c.Path, args...).Output()
return c.Executor(value)
}
func (c *CommandArg) UnmarshalValue(value string) error {

View File

@ -21,7 +21,8 @@ import (
_ "os"
_ "os/exec"
"path/filepath"
_ "strings"
_ "strings"
"encoding/json"
)
type ContainerClient interface {
@ -110,6 +111,14 @@ func (c *Container) SetURI(uri string) error {
return e
}
func (c *Container) JSON() ([]byte, error) {
return json.Marshal(c)
}
func (c *Container) Validate() error {
return fmt.Errorf("failed")
}
func (c *Container) Apply() error {
ctx := context.Background()
switch c.State {

View File

@ -4,14 +4,11 @@ package resource
import (
"encoding/json"
"errors"
_ "fmt"
"github.com/xeipuuv/gojsonschema"
_ "fmt"
"gopkg.in/yaml.v3"
"io"
_ "log"
_ "net/url"
"strings"
"log/slog"
_ "net/url"
)
type Document struct {
@ -29,21 +26,20 @@ func (d *Document) Load(r io.Reader) error {
func (d *Document) Validate() error {
jsonDocument, jsonErr := d.JSON()
slog.Info("document.Validate() json", "json", jsonDocument, "err", jsonErr)
if jsonErr == nil {
schemaLoader := gojsonschema.NewReferenceLoader("file://schemas/document.jsonschema")
documentLoader := gojsonschema.NewBytesLoader(jsonDocument)
result, err := gojsonschema.Validate(schemaLoader, documentLoader)
s := NewSchema("document")
err := s.Validate(string(jsonDocument))
if err != nil {
return err
}
if !result.Valid() {
schemaErrors := strings.Builder{}
for _, err := range result.Errors() {
schemaErrors.WriteString(err.String() + "\n")
/*
for i := range d.ResourceDecls {
if e := d.ResourceDecls[i].Resource().Validate(); e != nil {
return fmt.Errorf("failed to validate resource %s; %w", d.ResourceDecls[i].Resource().URI(), e)
}
return errors.New(schemaErrors.String())
}
*/
}
return nil
}

View File

@ -57,6 +57,10 @@ func (x *Exec) ResolveId(ctx context.Context) string {
return ""
}
func (x *Exec) Validate() error {
return fmt.Errorf("failed")
}
func (x *Exec) Apply() error {
return nil
}

View File

@ -79,6 +79,10 @@ func (f *File) SetURI(uri string) error {
return e
}
func (f *File) Validate() error {
return fmt.Errorf("failed")
}
func (f *File) Apply() error {
switch f.State {
case "absent":

View File

@ -25,10 +25,10 @@ func HTTPFactory(u *url.URL) Resource {
// Manage the state of an HTTP endpoint
type HTTP struct {
Endpoint string `yaml:"endpoint"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
Body string `yaml:"body,omitempty"`
State string `yaml:"state"`
Body string `yaml:"body,omitempty" json:"body,omitempty"`
State string `yaml:"state" json:"state"`
}
func NewHTTP() *HTTP {
@ -47,6 +47,10 @@ func (h *HTTP) SetURI(uri string) error {
return nil
}
func (h *HTTP) Validate() error {
return fmt.Errorf("failed")
}
func (h *HTTP) Apply() error {
switch h.State {

View File

@ -131,6 +131,10 @@ func (n *NetworkRoute) SetURI(uri string) error {
return nil
}
func (n *NetworkRoute) Validate() error {
return fmt.Errorf("failed")
}
func (n *NetworkRoute) Apply() error {
switch n.State {

View File

@ -4,23 +4,43 @@ package resource
import (
"context"
"encoding/json"
"errors"
"fmt"
"gopkg.in/yaml.v3"
_ "log"
"net/url"
_ "os"
_ "os/exec"
"io"
"log/slog"
"net/url"
_ "os"
_ "os/exec"
"path/filepath"
_ "strings"
"strings"
)
type PackageType string
const (
PackageTypeApk PackageType = "apk"
PackageTypeApt PackageType = "apt"
PackageTypeDeb PackageType = "deb"
PackageTypeDnf PackageType = "dnf"
PackageTypeRpm PackageType = "rpm"
PackageTypePip PackageType = "pip"
PackageTypeYum PackageType = "yum"
)
type Package struct {
Name string `yaml:"name" json:"name"`
Version string `yaml:"version" json:"version"`
Name string `yaml:"name" json:"name"`
Required string `json:"required" yaml:"required"`
Version string `yaml:"version" json:"version"`
PackageType PackageType `yaml:"type" json:"type"`
CreateCommand *Command `yaml:"-" json:"-"`
ReadCommand *Command `yaml:"-" json:"-"`
UpdateCommand *Command `yaml:"-" json:"-"`
DeleteCommand *Command `yaml:"-" json:"-"`
// state attributes
State string `yaml:"state"`
State string `yaml:"state" json:"state"`
}
func init() {
@ -28,6 +48,34 @@ func init() {
p := NewPackage()
return p
})
ResourceTypes.Register(string(PackageTypeApk), func(u *url.URL) Resource {
p := NewPackage()
return p
})
ResourceTypes.Register(string(PackageTypeApt), func(u *url.URL) Resource {
p := NewPackage()
return p
})
ResourceTypes.Register(string(PackageTypeDeb), func(u *url.URL) Resource {
p := NewPackage()
return p
})
ResourceTypes.Register(string(PackageTypeDnf), func(u *url.URL) Resource {
p := NewPackage()
return p
})
ResourceTypes.Register(string(PackageTypeRpm), func(u *url.URL) Resource {
p := NewPackage()
return p
})
ResourceTypes.Register(string(PackageTypePip), func(u *url.URL) Resource {
p := NewPackage()
return p
})
ResourceTypes.Register(string(PackageTypeYum), func(u *url.URL) Resource {
p := NewPackage()
return p
})
}
func NewPackage() *Package {
@ -35,18 +83,22 @@ func NewPackage() *Package {
}
func (p *Package) URI() string {
return fmt.Sprintf("package://%s?version=%s", p.Name, p.Version)
return fmt.Sprintf("package://%s?version=%s&type=%s", p.Name, p.Version, p.PackageType)
}
func (p *Package) SetURI(uri string) error {
resourceUri, e := url.Parse(uri)
if e == nil {
if resourceUri.Scheme == "package" {
p.Name = filepath.Join(resourceUri.Hostname(), resourceUri.RequestURI())
p.Name = filepath.Join(resourceUri.Hostname(), resourceUri.Path)
p.Version = resourceUri.Query().Get("version")
if p.Version == "" {
p.Version = "latest"
}
p.PackageType = PackageType(resourceUri.Query().Get("type"))
if p.PackageType == "" {
e = fmt.Errorf("%w: %s is not a package known resource ", ErrInvalidResourceURI, uri)
}
} else {
e = fmt.Errorf("%w: %s is not a package resource ", ErrInvalidResourceURI, uri)
}
@ -54,15 +106,36 @@ func (p *Package) SetURI(uri string) error {
return e
}
func (p *Package) JSON() ([]byte, error) {
return json.Marshal(p)
}
func (p *Package) Validate() error {
s := NewSchema(p.Type())
jsonDoc, jsonErr := p.JSON()
if jsonErr == nil {
return s.Validate(string(jsonDoc))
}
return jsonErr
}
func (p *Package) ResolveId(ctx context.Context) string {
return ""
}
func (p *Package) Apply() error {
if p.Version == "latest" {
p.Version = ""
}
_, err := p.CreateCommand.Execute(p)
if err != nil {
return err
}
p.Read(context.Background())
return nil
}
func (p *Package) Load(r io.Reader) error {
c := NewYAMLDecoder(r)
return c.Decode(p)
@ -76,16 +149,81 @@ func (p *Package) LoadDecl(yamlResourceDeclaration string) error {
func (p *Package) Type() string { return "package" }
func (p *Package) Read(ctx context.Context) ([]byte, error) {
out, err := p.ReadCommand.Execute(p)
if err != nil {
return nil, err
}
exErr := p.ReadCommand.Extractor(out, p)
if exErr != nil {
return nil, exErr
}
return yaml.Marshal(p)
}
func (p *Package) UnmarshalJSON(data []byte) error {
if unmarshalErr := json.Unmarshal(data, *p); unmarshalErr != nil {
return unmarshalErr
}
p.CreateCommand, p.ReadCommand, p.UpdateCommand, p.DeleteCommand = p.PackageType.NewCRUD()
return nil
}
func (p *Package) UnmarshalYAML(value *yaml.Node) error {
type decodePackage Package
if unmarshalErr := value.Decode((*decodePackage)(p)); unmarshalErr != nil {
return unmarshalErr
}
p.CreateCommand, p.ReadCommand, p.UpdateCommand, p.DeleteCommand = p.PackageType.NewCRUD()
return nil
}
func (p *PackageType) NewCRUD() (create *Command, read *Command, update *Command, del *Command) {
switch *p {
case PackageTypeApk:
return NewApkCreateCommand(), NewApkReadCommand(), NewApkUpdateCommand(), NewApkDeleteCommand()
case PackageTypeApt:
return NewAptCreateCommand(), NewAptReadCommand(), NewAptUpdateCommand(), NewAptDeleteCommand()
case PackageTypeDeb:
case PackageTypeDnf:
case PackageTypeRpm:
case PackageTypePip:
case PackageTypeYum:
}
return nil, nil, nil, nil
}
func (p *PackageType) UnmarshalValue(value string) error {
switch value {
case string(PackageTypeApk), string(PackageTypeApt), string(PackageTypeDeb), string(PackageTypeDnf), string(PackageTypeRpm), string(PackageTypePip), string(PackageTypeYum):
*p = PackageType(value)
return nil
default:
return errors.New("invalid PackageType value")
}
}
func (p *PackageType) UnmarshalJSON(data []byte) error {
var s string
if unmarshalRouteTypeErr := json.Unmarshal(data, &s); unmarshalRouteTypeErr != nil {
return unmarshalRouteTypeErr
}
return p.UnmarshalValue(s)
}
func (p *PackageType) UnmarshalYAML(value *yaml.Node) error {
var s string
if err := value.Decode(&s); err != nil {
return err
}
return p.UnmarshalValue(s)
}
func NewApkCreateCommand() *Command {
c := NewCommand()
c.Path = "apk"
c.Args = []CommandArg{
CommandArg("add"),
CommandArg("{{ .Name }}={{ .Version }}"),
CommandArg("{{ .Name }}{{ .Required }}"),
}
return c
}
@ -98,6 +236,28 @@ func NewApkReadCommand() *Command {
CommandArg("-ev"),
CommandArg("{{ .Name }}"),
}
c.Extractor = func(out []byte, target any) error {
p := target.(*Package)
pkg := strings.Split(string(out), "-")
if pkg[0] == p.Name {
p.Name = pkg[0]
p.Version = pkg[1]
p.State = "present"
} else {
p.State = "absent"
}
return nil
}
return c
}
func NewApkUpdateCommand() *Command {
c := NewCommand()
c.Path = "apk"
c.Args = []CommandArg{
CommandArg("del"),
CommandArg("{{ .Name }}"),
}
return c
}
@ -110,3 +270,70 @@ func NewApkDeleteCommand() *Command {
}
return c
}
func NewAptCreateCommand() *Command {
c := NewCommand()
c.Path = "apt-get"
c.Args = []CommandArg{
CommandArg("satisfy"),
CommandArg("-y"),
CommandArg("{{ .Name }} ({{ .Required }})"),
}
return c
}
func NewAptReadCommand() *Command {
c := NewCommand()
c.Path = "dpkg"
c.Args = []CommandArg{
CommandArg("-s"),
CommandArg("{{ .Name }}"),
}
c.Extractor = func(out []byte, target any) error {
p := target.(*Package)
slog.Info("Extract()", "out", out)
pkginfo := strings.Split(string(out), "\n")
for _, infofield := range pkginfo {
if len(infofield) > 0 && infofield[0] != ' ' {
fieldKeyValue := strings.SplitN(infofield, ":", 2)
if len(fieldKeyValue) > 1 {
key := strings.TrimSpace(fieldKeyValue[0])
value := strings.TrimSpace(fieldKeyValue[1])
switch key {
case "Package":
if value == p.Name {
p.State = "present"
} else {
p.State = "absent"
return nil
}
case "Version":
p.Version = value
}
}
}
}
return nil
}
return c
}
func NewAptUpdateCommand() *Command {
c := NewCommand()
c.Path = "apt"
c.Args = []CommandArg{
CommandArg("install"),
CommandArg("{{ .Name }}"),
}
return c
}
func NewAptDeleteCommand() *Command {
c := NewCommand()
c.Path = "apt"
c.Args = []CommandArg{
CommandArg("remove"),
CommandArg("{{ .Name }}"),
}
return c
}

View File

@ -4,17 +4,17 @@ package resource
import (
"context"
_ "encoding/json"
_ "fmt"
_ "encoding/json"
_ "fmt"
"github.com/stretchr/testify/assert"
_ "gopkg.in/yaml.v3"
_ "io"
_ "log"
_ "net/http"
_ "net/http/httptest"
_ "net/url"
_ "os"
_ "strings"
_ "gopkg.in/yaml.v3"
_ "io"
"log/slog"
_ "net/http"
_ "net/http/httptest"
_ "net/url"
_ "os"
_ "strings"
"testing"
)
@ -32,7 +32,7 @@ func TestPackageApplyResourceTransformation(t *testing.T) {
}
func TestReadPackage(t *testing.T) {
decl:=`
decl := `
name: vim
version: latest
type: apk
@ -40,13 +40,56 @@ type: apk
p := NewPackage()
assert.NotNil(t, p)
m := &MockCommand{
Executor: func(value any) ([]byte, error) {
return nil, nil
},
Extractor: func(output []byte, target any) error {
p.Name = "vim"
p.Version = "1.1.1"
p.State = "present"
return nil
},
}
loadErr := p.LoadDecl(decl)
assert.Nil(t, loadErr)
assert.Equal(t, "latest", p.Version)
p.ReadCommand = (*Command)(m)
yaml, readErr := p.Read(context.Background())
assert.Nil(t, readErr)
assert.Greater(t, len(yaml), 0)
slog.Info("read()", "yaml", yaml)
assert.Equal(t, "1.1.1", p.Version)
slog.Info("resource: ", "package", p)
assert.Nil(t, p.Validate())
}
func TestReadAptPackage(t *testing.T) {
decl := `
name: vim
required: ">1.1.1"
type: apt
`
p := NewPackage()
assert.NotNil(t, p)
loadErr := p.LoadDecl(decl)
assert.Nil(t, loadErr)
assert.Equal(t, ">1.1.1", p.Required)
p.ReadCommand = NewAptReadCommand()
p.ReadCommand.Executor = func(value any) ([]byte, error) {
return []byte(`
Package: vim
Version: 1.2.2
`), nil
}
yaml, readErr := p.Read(context.Background())
assert.Nil(t, readErr)
assert.Greater(t, len(yaml), 0)
slog.Info("read()", "yaml", yaml)
assert.Equal(t, "1.2.2", p.Version)
assert.Nil(t, p.Validate())
}
func TestReadPackageError(t *testing.T) {
@ -58,7 +101,7 @@ func TestCreatePackage(t *testing.T) {
func TestPackageSetURI(t *testing.T) {
p := NewPackage()
assert.NotNil(t, p)
e := p.SetURI("package://" + "12345_key")
e := p.SetURI("package://" + "12345_key?type=apk")
assert.Nil(t, e)
assert.Equal(t, "package", p.Type())
assert.Equal(t, "12345_key", p.Name)

View File

@ -19,9 +19,9 @@ type Resource interface {
ResourceLoader
StateTransformer
ResourceReader
ResourceValidator
}
// validate the type/uri
type ResourceValidator interface {
Validate() error
}

View File

@ -8,8 +8,14 @@ import (
"fmt"
"github.com/xeipuuv/gojsonschema"
"strings"
"embed"
"net/http"
"log/slog"
)
//go:embed schemas/*.jsonschema
var schemaFiles embed.FS
type Schema struct {
schema gojsonschema.JSONLoader
}
@ -17,7 +23,8 @@ type Schema struct {
func NewSchema(name string) *Schema {
path := fmt.Sprintf("file://schemas/%s.jsonschema", name)
return &Schema{schema: gojsonschema.NewReferenceLoader(path)}
return &Schema{schema: gojsonschema.NewReferenceLoaderFileSystem(path, http.FS(schemaFiles))}
//return &Schema{schema: gojsonschema.NewReferenceLoader(path)}
}
func (s *Schema) Validate(source string) error {
@ -26,9 +33,10 @@ func (s *Schema) Validate(source string) error {
result, err := gojsonschema.Validate(s.schema, loader)
if err != nil {
fmt.Printf("%#v %#v %#v %#v\n", source, loader, result, err)
slog.Info("schema error", "source", source, "schema", s.schema, "result", result, "err", err)
return err
}
slog.Info("schema", "source", source, "schema", s.schema, "result", result, "err", err)
if !result.Valid() {
schemaErrors := strings.Builder{}
@ -39,3 +47,11 @@ func (s *Schema) Validate(source string) error {
}
return nil
}
func (s *Schema) ValidateSchema() error {
sl := gojsonschema.NewSchemaLoader()
sl.Validate = true
schemaErr := sl.AddSchemas(s.schema)
slog.Info("validate schema definition", "schemaloader", sl, "err", schemaErr)
return schemaErr
}

View File

@ -26,7 +26,7 @@ func TestNewSchema(t *testing.T) {
assert.NotEqual(t, nil, s)
}
func TestSchemaValidate(t *testing.T) {
func TestSchemaValidateJSON(t *testing.T) {
ctx := context.Background()
s := NewSchema("file")
assert.NotEqual(t, nil, s)
@ -79,3 +79,10 @@ func TestSchemaValidate(t *testing.T) {
expected := fmt.Sprintf(declarationAttributes, file, cTime.Format(time.RFC3339Nano))
assert.YAMLEq(t, expected, string(r))
}
func TestSchemaValidateSchema(t *testing.T) {
s := NewSchema("document")
assert.NotNil(t, s)
assert.Nil(t, s.ValidateSchema())
}

View File

@ -1,4 +1,5 @@
{
"$id": "document.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "document",
"type": "object",
@ -8,7 +9,12 @@
"type": "array",
"description": "Resources list",
"items": {
"type": "object"
"oneOf": [
{ "$ref": "package-declaration.jsonschema" },
{ "$ref": "file-declaration.jsonschema" },
{ "$ref": "user-declaration.jsonschema" },
{ "$ref": "exec-declaration.jsonschema" }
]
}
}
}

View File

@ -0,0 +1,17 @@
{
"$id": "exec-declaration.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "exec-declaration",
"type": "object",
"required": [ "type", "attributes" ],
"properties": {
"type": {
"type": "string",
"description": "Resource type name.",
"enum": [ "exec" ]
},
"attributes": {
"$ref": "exec.jsonschema"
}
}
}

View File

@ -1,4 +1,5 @@
{
"$id": "exec.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "exec",
"type": "object",

View File

@ -0,0 +1,17 @@
{
"$id": "file-declaration.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "file-declaration",
"type": "object",
"required": [ "type", "attributes" ],
"properties": {
"type": {
"type": "string",
"description": "Resource type name.",
"enum": [ "file" ]
},
"attributes": {
"$ref": "file.jsonschema"
}
}
}

View File

@ -1,4 +1,5 @@
{
"$id": "file.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "file",
"type": "object",

View File

@ -0,0 +1,17 @@
{
"$id": "package-declaration.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "package-declaration",
"type": "object",
"required": [ "type", "attributes" ],
"properties": {
"type": {
"type": "string",
"description": "Resource type name.",
"enum": [ "package" ]
},
"attributes": {
"$ref": "package.jsonschema"
}
}
}

View File

@ -1,11 +1,18 @@
{
"$id": "package.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "package",
"type": "object",
"required": [ "name", "version", "type" ],
"required": [ "name", "type" ],
"properties": {
"name": {
"type": "string"
"type": "string",
"pattern": "^[a-zA-Z0-9][a-zA-Z0-9+.-_]+$"
},
"required": {
"description": "version requirement",
"type": "string",
"pattern": "^([><~=][=]?[a-zA-Z0-9+.-_]+|)$"
},
"version": {
"type": "string"

View File

@ -0,0 +1,17 @@
{
"$id": "user-declaration.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "declaration",
"type": "object",
"required": [ "type", "attributes" ],
"properties": {
"type": {
"type": "string",
"description": "Resource type name.",
"enum": [ "user" ]
},
"attributes": {
"$ref": "user.jsonschema"
}
}
}

View File

@ -1,4 +1,5 @@
{
"$id": "user.jsonschema",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "user",
"description": "A user account",
@ -6,7 +7,8 @@
"required": [ "name" ],
"properties": {
"name": {
"type": "string"
"type": "string",
"pattern": "^[a-z]([-_a-z0-9]{0,31})$"
},
"uid": {
"type": "integer",

View File

@ -50,6 +50,10 @@ func (u *User) ResolveId(ctx context.Context) string {
return LookupUIDString(u.Name)
}
func (u *User) Validate() error {
return fmt.Errorf("failed")
}
func (u *User) Apply() error {
switch u.State {
case "present":

View File

@ -13,6 +13,7 @@ type MockResource struct {
InjectType func() string
InjectResolveId func(ctx context.Context) string
InjectLoadDecl func(string) error
InjectValidate func() error
InjectApply func() error
InjectRead func(context.Context) ([]byte, error)
}
@ -29,6 +30,10 @@ func (m *MockResource) LoadDecl(yamlResourceDeclaration string) error {
return m.InjectLoadDecl(yamlResourceDeclaration)
}
func (m *MockResource) Validate() error {
return m.InjectValidate()
}
func (m *MockResource) Apply() error {
return m.InjectApply()
}