add build support to container_image resource; add more testing

internal/resource/common.go

@@ -0,0 +1,131 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package resource
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"path/filepath"
+	"decl/internal/data"
+	"decl/internal/folio"
+)
+
+type Common struct {
+	includeQueryParamsInURI bool `json:"-" yaml:"-"`
+	resourceType TypeName `json:"-" yaml:"-"`
+	Uri folio.URI `json:"uri,omitempty" yaml:"uri,omitempty"`
+	parsedURI *url.URL `json:"-" yaml:"-"`
+	Path string `json:"path,omitempty" yaml:"path,omitempty"`
+
+	exttype string `json:"-" yaml:"-"`
+	fileext string `json:"-" yaml:"-"`
+	normalizePath bool `json:"-" yaml:"-"`
+
+	State    string   `json:"state,omitempty" yaml:"state,omitempty"`
+	config data.ConfigurationValueGetter
+	Resources data.ResourceMapper `json:"-" yaml:"-"`
+}
+
+func NewCommon() *Common {
+	return &Common{ includeQueryParamsInURI: false }
+}
+
+func (c *Common) ContentType() string {
+	if c.parsedURI.Scheme != "file" {
+		return c.parsedURI.Scheme
+	}
+	return c.exttype
+}
+
+func (c *Common) SetResourceMapper(resources data.ResourceMapper) {
+	c.Resources = resources
+}
+
+func (c *Common) Clone() *Common {
+	return &Common {
+		Uri: c.Uri,
+		parsedURI: c.parsedURI,
+		Path: c.Path,
+		exttype: c.exttype,
+		fileext: c.fileext,
+		normalizePath: c.normalizePath,
+		State: c.State,
+		config: c.config,
+		Resources: c.Resources,
+	}
+}
+
+func (c *Common) PathNormalization(flag bool) {
+	c.normalizePath = flag
+}
+
+func (c *Common) URIPath() string {
+	return c.Path
+}
+
+func (c *Common) URI() string {
+	return fmt.Sprintf("%s://%s", c.Type(), c.Path)
+}
+
+func (c *Common) SetURI(uri string) (err error) {
+	c.SetURIFromString(uri)
+	err = c.SetParsedURI(c.Uri.Parse())
+	return
+}
+
+func (c *Common) SetURIFromString(uri string) {
+	c.Uri = folio.URI(uri)
+	c.exttype, c.fileext = c.Uri.Extension()
+}
+
+func (c *Common) SetParsedURI(u *url.URL) (err error) {
+	if u != nil {
+		if c.Uri.IsEmpty()  {
+			c.SetURIFromString(u.String())
+		}
+		c.parsedURI = u
+		if c.parsedURI.Scheme == c.Type() {
+			if c.includeQueryParamsInURI {
+				c.Path = filepath.Join(c.parsedURI.Hostname(), c.parsedURI.RequestURI())
+			} else {
+				c.Path = filepath.Join(c.parsedURI.Hostname(), c.parsedURI.Path)
+			}
+			if err = c.NormalizePath(); err != nil {
+				return
+			}
+			return
+		}
+	}
+	err = fmt.Errorf("%w: %s is not a file", ErrInvalidResourceURI, c.Uri)
+	return
+}
+
+func (c *Common) UseConfig(config data.ConfigurationValueGetter) {
+	c.config = config
+}
+
+func (c *Common) ResolveId(ctx context.Context) string {
+	if e := c.NormalizePath(); e != nil {
+		panic(e)
+	}
+	return c.Path
+}
+
+func (c *Common) NormalizePath() error {
+	if c.config != nil {
+		if prefixPath, configErr := c.config.GetValue("prefix"); configErr == nil {
+			c.Path = filepath.Join(prefixPath.(string), c.Path)
+		}
+	}
+	if c.normalizePath {
+		filePath, fileAbsErr := filepath.Abs(c.Path)
+		if fileAbsErr == nil {
+			c.Path = filePath
+		}
+		return fileAbsErr
+	}
+	return nil
+}
+
+func (c *Common) Type() string { return string(c.resourceType) }

internal/resource/common_test.go

@@ -0,0 +1,27 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package resource
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"net/url"
+)
+
+func TestNewCommon(t *testing.T) {
+	c := NewCommon()
+	assert.NotNil(t, c)
+}
+
+func TestCommon(t *testing.T) {
+	for _, v := range []struct{ uri string; expected Common }{
+		{ uri: "file:///tmp/foo", expected: Common{ resourceType: "file", Uri: "file:///tmp/foo", parsedURI: &url.URL{ Scheme: "file", Path: "/tmp/foo"}, Path: "/tmp/foo" } },
+	}{
+		c := NewCommon()
+		c.resourceType = "file"
+		assert.Nil(t, c.SetURI(v.uri))
+		assert.Equal(t, v.expected.Path, c.Path)
+		assert.Equal(t, &v.expected, c)
+	}
+
+}

internal/resource/container.go

@@ -27,6 +27,11 @@ _	"os/exec"
 	"io"
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
+	"decl/internal/data"
+)
+
+const (
+	ContainerTypeName TypeName = "container"
 )
 
 type ContainerClient interface {
@@ -41,10 +46,11 @@ type ContainerClient interface {
 }
 
 type Container struct {
+	*Common																`yaml:",inline" json:",inline"`
 	stater					machine.Stater				`yaml:"-" json:"-"`
 	Id              string                `json:"ID,omitempty" yaml:"ID,omitempty"`
 	Name            string                `json:"name" yaml:"name"`
-	Path            string                `json:"path" yaml:"path"`
+//	Path            string                `json:"path" yaml:"path"`
 	Cmd             []string              `json:"cmd,omitempty" yaml:"cmd,omitempty"`
 	Entrypoint      strslice.StrSlice     `json:"entrypoint,omitempty" yaml:"entrypoint,omitempty"`
 	Args            []string              `json:"args,omitempty" yaml:"args,omitempty"`
@@ -75,15 +81,15 @@ type Container struct {
 	   NetworkSettings *NetworkSettings
 	*/
 
-	State string `yaml:"state,omitempty" json:"state,omitempty"`
+//	State string `yaml:"state,omitempty" json:"state,omitempty"`
 
-	config ConfigurationValueGetter
+//	config ConfigurationValueGetter
 	apiClient ContainerClient
-	Resources ResourceMapper							`json:"-" yaml:"-"`
+//	Resources data.ResourceMapper							`json:"-" yaml:"-"`
 }
 
 func init() {
-	ResourceTypes.Register([]string{"container"}, func(u *url.URL) Resource {
+	ResourceTypes.Register([]string{"container"}, func(u *url.URL) data.Resource {
 		c := NewContainer(nil)
 		c.Name = filepath.Join(u.Hostname(), u.Path)
 		return c
@@ -100,19 +106,20 @@ func NewContainer(containerClientApi ContainerClient) *Container {
 		}
 	}
 	return &Container{
+		Common: &Common{ resourceType: ContainerTypeName },
 		apiClient: apiClient,
 	}
 }
 
-func (c *Container) SetResourceMapper(resources ResourceMapper) {
+func (c *Container) SetResourceMapper(resources data.ResourceMapper) {
 	c.Resources = resources
 }
 
-func (c *Container) Clone() Resource {
+func (c *Container) Clone() data.Resource {
 	return &Container {
 		Id: c.Id,
 		Name: c.Name,
-		Path: c.Path,
+		Common: c.Common,
 		Cmd: c.Cmd,
 		Entrypoint: c.Entrypoint,
 		Args: c.Args,
@@ -136,7 +143,7 @@ func (c *Container) Clone() Resource {
 		SizeRw: c.SizeRw,
 		SizeRootFs: c.SizeRootFs,
 		Networks: c.Networks,
-		State: c.State,
+//		State: c.State,
 		apiClient: c.apiClient,
 	}
 }
@@ -148,6 +155,7 @@ func (c *Container) StateMachine() machine.Stater {
 	return c.stater
 }
 
+/*
 func (c *Container) URI() string {
 	return fmt.Sprintf("container://%s", c.Id)
 }
@@ -163,8 +171,9 @@ func (c *Container) SetURI(uri string) error {
 	}
 	return e
 }
+*/
 
-func (c *Container) UseConfig(config ConfigurationValueGetter) {
+func (c *Container) UseConfig(config data.ConfigurationValueGetter) {
 	c.config = config
 }
 
@@ -186,11 +195,11 @@ func (c *Container) Notify(m *machine.EventMessage) {
 			if triggerErr := c.stater.Trigger("state_read"); triggerErr == nil {
 				return
 			} else {
-				c.State = "absent"
+				c.Common.State = "absent"
 				panic(triggerErr)
 			}
 		} else {
-			c.State = "absent"
+			c.Common.State = "absent"
 			panic(readErr)
 		}
 	case "start_create":
@@ -198,11 +207,11 @@ func (c *Container) Notify(m *machine.EventMessage) {
 			if triggerErr := c.StateMachine().Trigger("created"); triggerErr == nil {
 				return
 			} else {
-				c.State = "absent"
+				c.Common.State = "absent"
 				panic(triggerErr)
 			}
 		} else {
-			c.State = "absent"
+			c.Common.State = "absent"
 			panic(createErr)
 		}
 	case "start_delete":
@@ -210,19 +219,19 @@ func (c *Container) Notify(m *machine.EventMessage) {
 			if triggerErr := c.StateMachine().Trigger("deleted"); triggerErr == nil {
 				return
 			} else {
-				c.State = "present"
+				c.Common.State = "present"
 				panic(triggerErr)
 			}
 		} else {
-			c.State = "present"
+			c.Common.State = "present"
 			panic(deleteErr)
 		}
 	case "present", "created", "read":
-		c.State = "present"
+		c.Common.State = "present"
 	case "running":
-		c.State = "running"
+		c.Common.State = "running"
 	case "absent":
-		c.State = "absent"
+		c.Common.State = "absent"
 	}
 	case machine.EXITSTATEEVENT:
 	}
@@ -230,7 +239,7 @@ func (c *Container) Notify(m *machine.EventMessage) {
 
 func (c *Container) Apply() error {
 	ctx := context.Background()
-	switch c.State {
+	switch c.Common.State {
 	case "absent":
 		return c.Delete(ctx)
 	case "present":
@@ -239,12 +248,23 @@ func (c *Container) Apply() error {
 	return nil
 }
 
-func (c *Container) Load(r io.Reader) error {
+func (c *Container) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(c)
+	err = f.StringDecoder(string(docData)).Decode(c)
+	return
+}
+
+func (c *Container) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(c)
+	return
+}
+
+func (c *Container) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(c)
+	return
 }
 
 func (c *Container) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(c)
+	return c.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
 func (c *Container) Create(ctx context.Context) error {
@@ -308,6 +328,10 @@ func (c *Container) Create(ctx context.Context) error {
 	return err
 }
 
+func (c *Container) Update(ctx context.Context) error {
+	return c.Create(ctx)
+}
+
 // produce yaml representation of any resource
 
 func (c *Container) Read(ctx context.Context) ([]byte, error) {
@@ -341,9 +365,9 @@ func (c *Container) Read(ctx context.Context) ([]byte, error) {
 func (c *Container) Inspect(ctx context.Context, containerID string) error {
 	containerJSON, err := c.apiClient.ContainerInspect(ctx, containerID)
 	if client.IsErrNotFound(err) {
-		c.State = "absent"
+		c.Common.State = "absent"
 	} else {
-		c.State = "present"
+		c.Common.State = "present"
 		c.Id = containerJSON.ID
 		if c.Name == "" {
 			if containerJSON.Name[0] == '/' {
@@ -352,7 +376,7 @@ func (c *Container) Inspect(ctx context.Context, containerID string) error {
 				c.Name = containerJSON.Name
 			}
 		}
-		c.Path = containerJSON.Path
+		c.Common.Path = containerJSON.Path
 		c.Image = containerJSON.Image
 		if containerJSON.State != nil {
 			c.ContainerState = *containerJSON.State

internal/resource/container_image.go

@@ -4,26 +4,48 @@
 package resource
 
 import (
+	"os"
 	"context"
 	"fmt"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/client"
-	"gopkg.in/yaml.v3"
-	_ "gopkg.in/yaml.v3"
 	"log/slog"
 	"net/url"
-_	"os"
-_	"os/exec"
 	"strings"
 	"encoding/json"
+	"encoding/base64"
 	"io"
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
+	"decl/internal/data"
+	"decl/internal/folio"
+	"decl/internal/transport"
+	"gopkg.in/yaml.v3"
+	"decl/internal/tempdir"
+	"archive/tar"
+_	"strconv"
 )
 
+var ContextTempDir tempdir.Path = "jx_containerimage_context"
+
+const (
+	ContainerImageTypeName TypeName = "container-image"
+)
+
+type ContainerErrorDetail struct {
+	ErrorMessage string `json:"message" yaml:"message"`
+}
+
+type ContainerError struct {
+	Detail ContainerErrorDetail `json:"errorDetail" yaml:"errorDetail"`
+	Error string `json:"error" yaml:"error"`
+}
+
 type ContainerImageClient interface {
 	ImagePull(ctx context.Context, refStr string, options image.PullOptions) (io.ReadCloser, error)
+	ImagePush(ctx context.Context, image string, options image.PushOptions) (io.ReadCloser, error)
 	ImageInspectWithRaw(ctx context.Context, imageID string) (types.ImageInspect, []byte, error)
 	ImageRemove(ctx context.Context, imageID string, options image.RemoveOptions) ([]image.DeleteResponse, error)
 	ImageBuild(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
@@ -31,6 +53,7 @@ type ContainerImageClient interface {
 }
 
 type ContainerImage struct {
+	*Common										`yaml:",inline" json:",inline"`
 	stater					machine.Stater	`yaml:"-" json:"-"`
 	Id					string					`json:"id,omitempty" yaml:"id,omitempty"`
 	Name					string					`json:"name" yaml:"name"`
@@ -42,17 +65,20 @@ type ContainerImage struct {
 	Author					string					`json:"author,omitempty" yaml:"author,omitempty"`
 	Comment					string					`json:"comment,omitempty" yaml:"comment,omitempty"`
 	Dockerfile				string					`json:"dockerfile,omitempty" yaml:"dockerfile,omitempty"`
-	ContextRef      ResourceReference     `json:"contextref,omitempty" yaml:"contextref,omitempty"`
+	DockerfileRef				folio.ResourceReference			`json:"dockerfileref,omitempty" yaml:"dockerfileref,omitempty"`
+	ContextRef				folio.ResourceReference			`json:"contextref,omitempty" yaml:"contextref,omitempty"`
 	InjectJX				bool					`json:"injectjx,omitempty" yaml:"injectjx,omitempty"`
-	State           string                `yaml:"state,omitempty" json:"state,omitempty"`
+	PushImage				bool					`json:"push,omitempty" yaml:"push,omitempty"`
+	Output					strings.Builder				`json:"output,omitempty" yaml:"output,omitempty"`
 
-	config ConfigurationValueGetter
 	apiClient ContainerImageClient
-	Resources ResourceMapper               `json:"-" yaml:"-"`
+	Resources data.ResourceMapper `json:"-" yaml:"-"`
+	contextDocument data.Document `json:"-" yaml:"-"`
+	ConverterTypes data.TypesRegistry[data.Converter] `json:"-" yaml:"-"`
 }
 
 func init() {
-	ResourceTypes.Register([]string{"container-image"}, func(u *url.URL) Resource {
+	folio.DocumentRegistry.ResourceTypes.Register([]string{"container-image"}, func(u *url.URL) data.Resource {
 		c := NewContainerImage(nil)
 		c.Name = ContainerImageNameFromURI(u)
 		slog.Info("NewContainerImage", "container", c)
@@ -70,17 +96,67 @@ func NewContainerImage(containerClientApi ContainerImageClient) *ContainerImage
 		}
 	}
 	return &ContainerImage{
+		Common: &Common{ includeQueryParamsInURI: true, resourceType: ContainerImageTypeName },
 		apiClient: apiClient,
 		InjectJX: true,
+		PushImage: false,
+		ConverterTypes: folio.DocumentRegistry.ConverterTypes,
 	}
 }
 
-func (c *ContainerImage) SetResourceMapper(resources ResourceMapper) {
+func (c *ContainerImage) RegistryAuthConfig() (authConfig registry.AuthConfig, err error) {
+	if c.config != nil {
+		var configValue any
+		if configValue, err = c.config.GetValue("repo_username"); err != nil {
+			return
+		} else {
+			authConfig.Username = configValue.(string)
+		}
+		if configValue, err = c.config.GetValue("repo_password"); err != nil {
+			return
+		} else {
+			authConfig.Password = configValue.(string)
+		}
+		if configValue, err = c.config.GetValue("repo_server"); err != nil {
+			return
+		} else {
+			authConfig.ServerAddress = configValue.(string)
+		}
+	}
+	return
+}
+
+/*
+func (c *ContainerImage) RegistryLogin(context context.Context) (token string, err error) {
+	var authConfig registry.AuthConfig
+	authConfig, err = c.RegistryAuthConfig()
+
+	if authResponse, loginErr := c.apiClient.RegistryLogin(context, authConfig); loginErr == nil {
+		return authResponse.IdentityToken, err
+	}
+	return
+}
+*/
+
+func (c *ContainerImage) RegistryAuth() (string, error) {
+	if authConfig, err := c.RegistryAuthConfig(); err == nil {
+		if encodedJSON, jsonErr := json.Marshal(authConfig); jsonErr == nil {
+			return base64.URLEncoding.EncodeToString(encodedJSON), nil
+		} else {
+			return "", jsonErr
+		}
+	} else {
+		return "", err
+	}
+}
+
+func (c *ContainerImage) SetResourceMapper(resources data.ResourceMapper) {
 	c.Resources = resources
 }
 
-func (c *ContainerImage) Clone() Resource {
+func (c *ContainerImage) Clone() data.Resource {
 	return &ContainerImage {
+		Common: c.Common,
 		Id: c.Id,
 		Name: c.Name,
 		Created: c.Created,
@@ -91,8 +167,8 @@ func (c *ContainerImage) Clone() Resource {
 		Author: c.Author,
 		Comment: c.Comment,
 		InjectJX: c.InjectJX,
-		State: c.State,
 		apiClient: c.apiClient,
+		contextDocument: c.contextDocument,
 	}
 }
 
@@ -118,9 +194,9 @@ func URIFromContainerImageName(imageName string) string {
 		repo = elements[2]
 	}
 	if namespace == "" {
-		return fmt.Sprintf("container-image://%s/%s", host, repo)
+		return fmt.Sprintf("%s://%s/%s", ContainerImageTypeName, host, repo)
 	}
-	return fmt.Sprintf("container-image://%s/%s", host, strings.Join([]string{namespace, repo}, "/"))
+	return fmt.Sprintf("%s://%s/%s", ContainerImageTypeName, host, strings.Join([]string{namespace, repo}, "/"))
 }
 
 // Reconstruct the image name from a given parsed URL
@@ -148,6 +224,7 @@ func (c *ContainerImage) URI() string {
 	return URIFromContainerImageName(c.Name)
 }
 
+/*
 func (c *ContainerImage) SetURI(uri string) error {
 	resourceUri, e := url.Parse(uri)
 	if e == nil {
@@ -160,9 +237,10 @@ func (c *ContainerImage) SetURI(uri string) error {
 	return e
 }
 
-func (c *ContainerImage) UseConfig(config ConfigurationValueGetter) {
+func (c *ContainerImage) UseConfig(config data.ConfigurationValueGetter) {
 	c.config = config
 }
+*/
 
 func (c *ContainerImage) JSON() ([]byte, error) {
 	return json.Marshal(c)
@@ -193,8 +271,10 @@ func (c *ContainerImage) Notify(m *machine.EventMessage) {
 	case "start_create":
 		if createErr := c.Create(ctx); createErr == nil {
 			if triggerErr := c.stater.Trigger("created"); triggerErr == nil {
+				slog.Info("ContainerImage.Notify()", "created", c, "error", triggerErr)
 				return
 			} else {
+				slog.Info("ContainerImage.Notify()", "created", c, "error", triggerErr)
 				c.State = "absent"
 				panic(triggerErr)
 			}
@@ -245,39 +325,281 @@ func (c *ContainerImage) Apply() error {
 	return nil
 }
 
-func (c *ContainerImage) Load(r io.Reader) error {
+func (c *ContainerImage) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(c)
+	err = f.StringDecoder(string(docData)).Decode(c)
+	return
+}
+
+func (c *ContainerImage) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(c)
+	return
+}
+
+func (c *ContainerImage) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(c)
+	return
 }
 
 func (c *ContainerImage) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(c)
+	return c.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
-func (c *ContainerImage) Create(ctx context.Context) error {
+func (c *ContainerImage) SetContextDocument(document data.Document) {
-	buildOptions := types.ImageBuildOptions{
+	c.contextDocument = document
-		Dockerfile: c.Dockerfile,
+}
-		Tags:       []string{c.Name},
+
+func (c *ContainerImage) ContextDocument() (document data.Document, err error) {
+	var sourceRef data.Resource
+
+	if v, ok := folio.DocumentRegistry.GetDocument(folio.URI(c.ContextRef)); ok {
+		return v, nil
 	}
 
+	slog.Info("ContainerImage.ContextDocument()", "contextref", c.ContextRef, "resources", c.Resources)
+	if sourceRef = c.ContextRef.Dereference(c.Resources); sourceRef == nil {
+		if sourceRef, err = folio.URI(c.ContextRef).NewResource(nil); err != nil {
+			return
+		}
+	}
+
+	if sourceRef != nil {
+		slog.Info("ContainerImage.ContextDocument() - Dereference", "contextref", c.ContextRef, "ref", sourceRef, "uri", sourceRef.URI())
+		var extractor data.Converter
+		if extractor, err = c.ConverterTypes.New(sourceRef.URI()); err == nil {
+			if v, ok := extractor.(data.DirectoryConverter); ok {
+				v.SetRelative(true)
+			}
+			slog.Info("ContainerImage.ContextDocument() - Converter", "extractor", extractor, "sourceref", sourceRef.URI(), "type", extractor.Type())
+			if document, err = extractor.Extract(sourceRef, nil); err != nil {
+				return
+			}
+		}
+	} else {
+		err = ErrUnableToFindResource
+		return
+	}
+
+	if err = ContextTempDir.Create(); err != nil {
+		return
+	}
+	if ! ContextTempDir.ValidPath() {
+		err = fmt.Errorf("Invalid temp dir path: %s", ContextTempDir)
+		return
+	}
+	//defer ContextTempDir.Remove()
+
+	var dockerfileResource data.Resource
+	var tarDockerfile folio.URI = folio.URI("file://Dockerfile")
+	if dockerfileDecl, ok := document.Get(string(tarDockerfile)); ok {
+		dockerfileResource = dockerfileDecl.(data.Declaration).Resource()
+	} else {
+		if dockerfileResource, err = document.(*folio.Document).NewResourceFromURI(tarDockerfile); err != nil {
+			return
+		}
+	}
+	if len(c.Dockerfile) > 0 {
+		dockerfileResource.(data.FileResource).SetContentSourceRef("")
+		err = dockerfileResource.(data.FileResource).SetContent(strings.NewReader(c.Dockerfile))
+		slog.Info("ContainerImage.ContextDocument()", "dockerfile", dockerfileResource)
+	} else if len(c.DockerfileRef) > 0 {
+		dockerfileResource.(data.FileResource).SetContentSourceRef(string(c.DockerfileRef))
+	}
+	if c.InjectJX {
+		var jxResource data.Resource
+		var jxURI folio.URI
+		jxURI, err = JXPath()
+		slog.Info("ContainerImage.ContextDocument()", "jx", jxURI, "error", err)
+		if jxResource, err = document.(*folio.Document).NewResource("file://jx"); err != nil {
+			return
+		}
+		jxResource.(data.FileResource).SetContentSourceRef(string(jxURI))
+		slog.Info("ContainerImage.ContextDocument()", "jxResource", jxResource)
+/*
+		fi, fiErr := data.FileInfoGetter(jxReader).Stat()
+		if fiErr != nil {
+			err = fiErr
+			return
+		}
+		jxResource.SetFileInfo(fi)
+*/
+	}
+	return
+}
+
+// creates tmp context archive file from source context archive reader
+func (c *ContainerImage) CreateContextArchive(reader io.ReadCloser) (contextTempFile folio.URI, err error) {
+	if err = ContextTempDir.Create(); err != nil {
+		return
+	}
+	if ! ContextTempDir.ValidPath() {
+		err = fmt.Errorf("Invalid temp dir path: %s", ContextTempDir)
+		return
+	}
+	//defer ContextTempDir.Remove()
+	contextTempFile = folio.URI(fmt.Sprintf("tar://%s/%s", ContextTempDir, "context.tar"))
+	writer, e := contextTempFile.ContentWriterStream()
+	if e != nil {
+		return contextTempFile, e
+	}
+
+	var header *tar.Header
+	tarReader := tar.NewReader(reader)
+	tarWriter := tar.NewWriter(writer)
+	defer tarWriter.Close()
+	for {
+		header, err = tarReader.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return
+		}
+
+		if err = tarWriter.WriteHeader(header); err != nil {
+			return
+		}
+
+		if _, err = io.Copy(tarWriter, tarReader); err != nil {
+			return
+		}
+	}
+
+	header = &tar.Header{
+		Name: "Dockerfile",
+		Mode: 0644,
+	}
+	if err = tarWriter.WriteHeader(header); err != nil {
+		return
+	}
+	var dockerfileReader io.Reader
+	if len(c.Dockerfile) > 0 {
+		dockerfileReader = strings.NewReader(c.Dockerfile)
+	} else {
+		if dockerfileReader, err = c.DockerfileRef.ContentReaderStream(); err != nil {
+			return
+		}
+	}
+	if _, err = io.Copy(tarWriter, dockerfileReader); err != nil {
+		return
+	}
+	if c.InjectJX {
+		var jxURI folio.URI
+		jxURI, err = JXPath()
+		var jxReader *transport.Reader
+		if jxReader, err = jxURI.ContentReaderStream(); err != nil {
+			return
+		}
+		fi, fiErr := data.FileInfoGetter(jxReader).Stat()
+		if fiErr != nil {
+			err = fiErr
+			return
+		}
+
+		slog.Info("ContainerImage.CreateContextArchive()", "jx", jxURI, "error", err)
+		header = &tar.Header{
+			Name: "jx",
+			Mode: 0755,
+			Size: fi.Size(),
+		}
+		if err = tarWriter.WriteHeader(header); err != nil {
+			return
+		}
+		if _, err = io.Copy(tarWriter, jxReader); err != nil {
+			return
+		}
+	}
+	return
+}
+
+func JXPath() (jxPath folio.URI, err error) {
+	var path string
+	path, err = os.Executable()
+	if err == nil {
+		jxPath = folio.URI(path)
+		if jxPath.Exists() {
+			return
+		} else {
+			err = os.ErrNotExist
+		}
+	}
+	jxPath = ""
+	return
+}
+
+// The contextref can be a tar file or a directory or maybe a loaded document
+func (c *ContainerImage) Create(ctx context.Context) (err error) {
+	dockerfileURI := c.DockerfileRef.Parse()
+	buildOptions := types.ImageBuildOptions{
+		Dockerfile: dockerfileURI.Path,
+		Tags:       []string{c.Name},
+	}
+	var reader io.ReadCloser
+
 	if c.ContextRef.Exists() {
-		if c.ContextRef.ContentType() == "tar" {
+
-			ref := c.ContextRef.Lookup(c.Resources)
+		contentType := folio.URI(c.ContextRef).ContentType()
-			reader, readerErr := ref.ContentReaderStream()
+
-			if readerErr != nil {
+		switch contentType {
-				return readerErr
+		case "tar", "tar.gz", "tgz":
+			var ctxArchiveURI folio.URI
+			r, refStreamErr := c.ContextRef.ContentReaderStream()
+			if refStreamErr != nil {
+				return refStreamErr
+			}
+			if ctxArchiveURI, err = c.CreateContextArchive(r); err != nil {
+				return err
+			}
+			reader, _ = ctxArchiveURI.ContentReaderStream()
+		default:
+			doc, ctErr := c.ContextDocument()
+			if ctErr != nil {
+				return ctErr
+			}
+			
+			emitTar, tarErr := c.ConverterTypes.New(fmt.Sprintf("tar://%s/%s", ContextTempDir, "context.tar"))
+			if tarErr != nil {
+				return tarErr
+			}
+
+			slog.Info("ContainerImage.Create()", "document", doc, "error", err)
+			tarResource, emitErr := emitTar.Emit(doc, nil)
+			if emitErr != nil {
+				slog.Info("ContainerImage.Create() Emit", "document", doc, "error", emitErr)
+				return emitErr
+			}
+			emitTar.Close()
+			slog.Info("ContainerImage.Create()", "tar", tarResource, "error", err)
+
+			reader, _ = tarResource.(*File).GetContent(nil)
 		}
 
 		buildResponse, buildErr := c.apiClient.ImageBuild(ctx, reader, buildOptions)
+		slog.Info("ContainerImage.Create() - ImageBuild()", "buildResponse", buildResponse, "error", buildErr)
 		if buildErr != nil {
 			return buildErr
 		}
+
 		defer buildResponse.Body.Close()
-			if _, outputErr := io.ReadAll(buildResponse.Body); outputErr != nil {
+		if output, outputErr := io.ReadAll(buildResponse.Body); outputErr != nil {
+			slog.Info("ContainerImage.Create() - ImageBuild()", "output", output, "error", outputErr)
 			return fmt.Errorf("%w %s %s", outputErr, c.Type(), c.Name)
+		} else {
+			slog.Info("ContainerImage.Create() - ImageBuild()", "output", output, "error", outputErr)
+			var containerErr ContainerError
+			for _, jsonBody := range strings.Split(string(output), "\r\n") {
+				decoder := codec.NewJSONStringDecoder(jsonBody)
+				decodeErr := decoder.Decode(&containerErr)
+				slog.Info("ContainerImage.Create() - ImageBuild()", "output", jsonBody, "error", containerErr, "decodeErr", decodeErr)
+				if len(containerErr.Error) > 0 {
+					return fmt.Errorf("%s", containerErr.Error)
 				}
 			}
 		}
 
+		if c.PushImage {
+			err = c.Push(ctx)
+		}
+	}
 	return nil
 }
 
@@ -285,6 +607,32 @@ func (c *ContainerImage) Update(ctx context.Context) error {
 	return c.Create(ctx)
 }
 
+func (c *ContainerImage) Push(ctx context.Context) (err error) {
+	var AuthToken string
+	AuthToken, err = c.RegistryAuth()
+	if err != nil {
+		return
+	}
+/*
+	if err = c.apiClient.ImageTag(ctx, imageName, targetImage); err != nil {
+		return
+	}
+*/
+	var response io.ReadCloser
+
+	if response, err = c.apiClient.ImagePush(context.Background(), c.Name, image.PushOptions{
+		RegistryAuth: AuthToken,
+	}); err != nil {
+		return
+	}
+	defer response.Close()
+
+	copyBuffer := make([]byte, 32 * 1024)
+	_, err = io.CopyBuffer(&c.Output, response, copyBuffer)
+
+	return
+}
+
 func (c *ContainerImage) Pull(ctx context.Context) error {
 	out, err := c.apiClient.ImagePull(ctx, c.Name, image.PullOptions{})
 	slog.Info("ContainerImage.Pull()", "name", c.Name, "error", err)

internal/resource/container_image_test.go

@@ -14,9 +14,15 @@ _ "encoding/json"
 _ "net/http"
 _ "net/http/httptest"
 	"net/url"
-_ "os"
+	"os"
 	"strings"
 	"testing"
+	"path/filepath"
+	"decl/internal/data"
+	"decl/internal/folio"
+	"decl/internal/codec"
+//_	"decl/internal/fan"
+	"strconv"
 )
 
 func TestNewContainerImageResource(t *testing.T) {
@@ -37,7 +43,7 @@ func TestContainerImageURI(t *testing.T) {
 
 func TestLoadFromContainerImageURI(t *testing.T) {
 	testURI := URIFromContainerImageName("myhost/quuz/foo:bar")
-	newResource, resourceErr := ResourceTypes.New(testURI)
+	newResource, resourceErr := folio.DocumentRegistry.ResourceTypes.New(testURI)
 	assert.Nil(t, resourceErr)
 	assert.NotNil(t, newResource)
 	assert.IsType(t, &ContainerImage{}, newResource)
@@ -112,8 +118,111 @@ func TestCreateContainerImage(t *testing.T) {
 	stater := c.StateMachine()
 
 	e := c.LoadDecl(decl)
-	assert.Equal(t, nil, e)
+	assert.Nil(t, e)
 	assert.Equal(t, "testcontainerimage", c.Name)
 
 	assert.Nil(t, stater.Trigger("create"))
 }
+
+func TestCreateContainerImagePush(t *testing.T) {
+	m := &mocks.MockContainerClient{
+		InjectImageBuild: func(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) {
+			return types.ImageBuildResponse{Body: io.NopCloser(strings.NewReader("image built")) }, nil
+		},
+		InjectImagePush: func(ctx context.Context, image string, options image.PushOptions) (io.ReadCloser, error) {
+			return io.NopCloser(strings.NewReader("foo")), nil
+		},
+	}
+
+	decl := fmt.Sprintf(`
+  name: "testcontainerimage"
+  image: "alpine"
+  push: true
+  contextref: file://%s
+`, "")
+	c := NewContainerImage(m)
+	stater := c.StateMachine()
+
+	e := c.LoadDecl(decl)
+	assert.Nil(t, e)
+	assert.Equal(t, "testcontainerimage", c.Name)
+
+	assert.Nil(t, stater.Trigger("create"))
+
+	c.Push(context.Background())
+	assert.True(t, c.PushImage)
+	assert.Equal(t, "foo", c.Output.String())
+}
+
+func TestContainerImageContextDocument(t *testing.T) {
+	contextDir, _ := filepath.Abs(filepath.Join(TempDir, "context"))
+	etcDir, _ := filepath.Abs(filepath.Join(contextDir, "etc"))
+	binDir, _ := filepath.Abs(filepath.Join(contextDir, "bin"))
+
+	assert.Nil(t, os.Mkdir(contextDir, os.ModePerm))
+	assert.Nil(t, os.Mkdir(etcDir, os.ModePerm))
+	assert.Nil(t, os.Mkdir(binDir, os.ModePerm))
+
+	m := &mocks.MockContainerClient{
+		InjectImageBuild: func(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) {
+			return types.ImageBuildResponse{Body: io.NopCloser(strings.NewReader("image built")) }, nil
+		},
+	}
+
+	contextDirUri := fmt.Sprintf("file://%s", contextDir)
+	
+	contextDirDecl := fmt.Sprintf(`
+type: file
+attributes:
+  path: %s
+`, contextDir)
+
+	decl := fmt.Sprintf(`
+  name: "testcontainerimage"
+  image: "alpine"
+  contextref: %s
+`, contextDirUri)
+
+	c := NewContainerImage(m)
+	c.ConverterTypes = TestConverterTypes
+	//stater := c.StateMachine()
+	e := c.LoadDecl(decl)
+	assert.Nil(t, e)
+
+	c.ContextRef = folio.ResourceReference(contextDirUri)
+	contextFile := folio.NewDeclaration()
+	assert.Nil(t, contextFile.NewResource(&contextDirUri))
+	assert.Nil(t, contextFile.LoadString(contextDirDecl, codec.FormatYaml))
+	_, readErr := contextFile.Resource().Read(context.Background())
+	assert.Nil(t, readErr)
+
+	c.Resources = data.NewResourceMapper()
+	c.Resources.Set(contextDirUri, contextFile)
+
+	d, contextErr := c.ContextDocument()
+	assert.Nil(t, contextErr)
+	assert.NotNil(t, d)
+	assert.Greater(t, 3, d.Len())
+
+}
+
+func TestContainerError(t *testing.T) {
+	var expected, err, boErr  ContainerError
+	expected.Detail.ErrorMessage = "invalid reference format"
+	expected.Error = "invalid reference format"
+	buildOutput := `"{\"stream\":\"Step 1/6 : ARG DIST\"}\r\n{\"stream\":\"\\n\"}\r\n{\"stream\":\"Step 2/6 : FROM golang:${DIST}\"}\r\n{\"stream\":\"\\n\"}\r\n{\"errorDetail\":{\"message\":\"invalid reference format\"},\"error\":\"invalid reference format\"}\r\n"`
+	unqBuildOutput, buildOutputErr := strconv.Unquote(buildOutput)
+	assert.Nil(t, buildOutputErr)
+
+	dec := codec.NewJSONStringDecoder(strings.Split(unqBuildOutput, "\r\n")[4])
+	assert.Nil(t, dec.Decode(&boErr))
+	assert.Equal(t, expected, boErr)
+
+	msg := `"{\"errorDetail\":{\"message\":\"invalid reference format\"},\"error\":\"invalid reference format\"}"`
+	unquotedmsg := `{"errorDetail":{"message":"invalid reference format"},"error":"invalid reference format"}`
+	c,_ := strconv.Unquote(msg)
+	assert.Equal(t, unquotedmsg, c)
+	decoder := codec.NewJSONStringDecoder(c)
+	assert.Nil(t, decoder.Decode(&err))
+	assert.Equal(t, expected, err)
+}

internal/resource/container_network.go

@@ -21,10 +21,16 @@ _	"strings"
 	"io"
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
+	"decl/internal/data"
+	"decl/internal/folio"
 	"log/slog"
 	"time"
 )
 
+const (
+	ContainerNetworkTypeName TypeName = "container-network"
+)
+
 type ContainerNetworkClient interface {
 	ContainerClient
 	NetworkCreate(ctx context.Context, name string, options network.CreateOptions) (network.CreateResponse, error)
@@ -33,6 +39,7 @@ type ContainerNetworkClient interface {
 }
 
 type ContainerNetwork struct {
+	*Common									`json:",inline" yaml:",inline"`
 	stater	machine.Stater	`json:"-" yaml:"-"`
 	Id              string  `json:"ID,omitempty" yaml:"ID,omitempty"`
 	Name            string  `json:"name" yaml:"name"`
@@ -41,15 +48,14 @@ type ContainerNetwork struct {
 	Internal				bool		`json:"internal,omitempty" yaml:"internal,omitempty"`
 	Labels					map[string]string	`json:"labels,omitempty" yaml:"labels,omitempty"`
 	Created					time.Time	`json:"created" yaml:"created"`
-	State string `yaml:"state"`
+	//State string `yaml:"state"`
 
-	config ConfigurationValueGetter
 	apiClient ContainerNetworkClient
-	Resources ResourceMapper `json:"-" yaml:"-"`
+	Resources data.ResourceMapper `json:"-" yaml:"-"`
 }
 
 func init() {
-	ResourceTypes.Register([]string{"container-network"}, func(u *url.URL) Resource {
+	folio.DocumentRegistry.ResourceTypes.Register([]string{"container-network"}, func(u *url.URL) data.Resource {
 		n := NewContainerNetwork(nil)
 		n.Name = filepath.Join(u.Hostname(), u.Path)
 		return n
@@ -66,19 +72,21 @@ func NewContainerNetwork(containerClientApi ContainerNetworkClient) *ContainerNe
 		}
 	}
 	return &ContainerNetwork{
+		Common: &Common{ includeQueryParamsInURI: true, resourceType: ContainerNetworkTypeName },
 		apiClient: apiClient,
 	}
 }
 
-func (n *ContainerNetwork) SetResourceMapper(resources ResourceMapper) {
+func (n *ContainerNetwork) SetResourceMapper(resources data.ResourceMapper) {
 	n.Resources = resources
 }
 
-func (n *ContainerNetwork) Clone() Resource {
+func (n *ContainerNetwork) Clone() data.Resource {
 	return &ContainerNetwork {
+		Common: n.Common,
 		Id: n.Id,
 		Name: n.Name,
-		State: n.State,
+		//State: n.State,
 		apiClient: n.apiClient,
 	}
 }
@@ -101,11 +109,11 @@ func (n *ContainerNetwork) Notify(m *machine.EventMessage) {
 				if triggerErr := n.StateMachine().Trigger("state_read"); triggerErr == nil {
 					return
 				} else {
-					n.State = "absent"
+					n.Common.State = "absent"
 					panic(triggerErr)
 				}
 			} else {
-				n.State = "absent"
+				n.Common.State = "absent"
 				panic(readErr)
 			}
 		case "start_delete":
@@ -113,11 +121,11 @@ func (n *ContainerNetwork) Notify(m *machine.EventMessage) {
 				if triggerErr := n.StateMachine().Trigger("deleted"); triggerErr == nil {
 					return
 				} else {
-					n.State = "present"
+					n.Common.State = "present"
 					panic(triggerErr)
 				}
 			} else {
-				n.State = "present"
+				n.Common.State = "present"
 				panic(deleteErr)
 			}
     case "start_create":
@@ -126,11 +134,11 @@ func (n *ContainerNetwork) Notify(m *machine.EventMessage) {
 					return
 				}
 			}
-			n.State = "absent"
+			n.Common.State = "absent"
 		case "absent":
-			n.State = "absent"
+			n.Common.State = "absent"
 		case "present", "created", "read":
-      n.State = "present"
+      n.Common.State = "present"
     }
   case machine.EXITSTATEEVENT:
   }
@@ -140,6 +148,7 @@ func (n *ContainerNetwork) URI() string {
 	return fmt.Sprintf("container-network://%s", n.Name)
 }
 
+/*
 func (n *ContainerNetwork) SetURI(uri string) error {
 	resourceUri, e := url.Parse(uri)
 	if e == nil {
@@ -152,9 +161,10 @@ func (n *ContainerNetwork) SetURI(uri string) error {
 	return e
 }
 
-func (n *ContainerNetwork) UseConfig(config ConfigurationValueGetter) {
+func (n *ContainerNetwork) UseConfig(config data.ConfigurationValueGetter) {
 	n.config = config
 }
+*/
 
 func (n *ContainerNetwork) JSON() ([]byte, error) {
 	return json.Marshal(n)
@@ -166,7 +176,7 @@ func (n *ContainerNetwork) Validate() error {
 
 func (n *ContainerNetwork) Apply() error {
 	ctx := context.Background()
-	switch n.State {
+	switch n.Common.State {
 	case "absent":
 		return n.Delete(ctx)
 	case "present":
@@ -175,12 +185,23 @@ func (n *ContainerNetwork) Apply() error {
 	return nil
 }
 
-func (n *ContainerNetwork) Load(r io.Reader) error {
+func (n *ContainerNetwork) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(n)
+	err = f.StringDecoder(string(docData)).Decode(n)
+	return
+}
+
+func (n *ContainerNetwork) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(n)
+	return
+}
+
+func (n *ContainerNetwork) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(n)
+	return
 }
 
 func (n *ContainerNetwork) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(n)
+	return n.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
 func (n *ContainerNetwork) Create(ctx context.Context) error {
@@ -198,9 +219,9 @@ func (n *ContainerNetwork) Create(ctx context.Context) error {
 func (n *ContainerNetwork) Inspect(ctx context.Context, networkID string) error {
 	networkInspect, err := n.apiClient.NetworkInspect(ctx, networkID, network.InspectOptions{})
 	if client.IsErrNotFound(err) {
-		n.State = "absent"
+		n.Common.State = "absent"
 	} else {
-		n.State = "present"
+		n.Common.State = "present"
 		n.Id = networkInspect.ID
 		if n.Name == "" {
 			if networkInspect.Name[0] == '/' {
@@ -244,6 +265,10 @@ func (n *ContainerNetwork) Read(ctx context.Context) ([]byte, error) {
 	return yaml.Marshal(n)
 }
 
+func (n *ContainerNetwork) Update(ctx context.Context) error {
+	return n.Create(ctx)
+}
+
 func (n *ContainerNetwork) Delete(ctx context.Context) error {
 	return nil
 }

internal/resource/declaration.go

@@ -1,274 +0,0 @@
-// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
-
-package resource
-
-import (
-_	"errors"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"gopkg.in/yaml.v3"
-	"log/slog"
-_	"gitea.rosskeen.house/rosskeen.house/machine"
-//_	"gitea.rosskeen.house/pylon/luaruntime"
-	"decl/internal/codec"
-	"decl/internal/config"
-)
-
-type ConfigName string
-
-type DeclarationType struct {
-	Type TypeName `json:"type" yaml:"type"`
-	Transition string `json:"transition,omitempty" yaml:"transition,omitempty"`
-	Config ConfigName `json:"config,omitempty" yaml:"config,omitempty"`
-}
-
-type Declaration struct {
-	Type TypeName `json:"type" yaml:"type"`
-	Transition string `json:"transition,omitempty" yaml:"transition,omitempty"`
-	Attributes Resource `json:"attributes" yaml:"attributes"`
-	Config ConfigName `json:"config,omitempty" yaml:"config,omitempty"`
-//	runtime luaruntime.LuaRunner
-	document *Document
-	configBlock *config.Block
-}
-
-type ResourceLoader interface {
-	LoadDecl(string) error
-}
-
-type StateTransformer interface {
-	Apply() error
-}
-
-func NewDeclaration() *Declaration {
-	return &Declaration{}
-}
-
-func NewDeclarationFromDocument(document *Document) *Declaration {
-	return &Declaration{ document: document }
-}
-
-func (d *Declaration) SetDocument(newDocument *Document) {
-	slog.Info("Declaration.SetDocument()")
-	d.document = newDocument
-	d.SetConfig(d.document.config)
-	d.Attributes.SetResourceMapper(d.document.uris)
-}
-
-func (d *Declaration) ResolveId(ctx context.Context) string {
-	defer func() {
-		if r := recover(); r != nil {
-			slog.Info("Declaration.ResolveId() - panic", "recover", r, "state", d.Attributes.StateMachine())
-			if triggerErr := d.Attributes.StateMachine().Trigger("notexists"); triggerErr != nil {
-				panic(triggerErr)
-			}
-		}
-	}()
-	slog.Info("Declaration.ResolveId()")
-	id := d.Attributes.ResolveId(ctx)
-	return id
-}
-
-func (d *Declaration) Clone() *Declaration {
-	return &Declaration {
-		Type: d.Type,
-    Transition: d.Transition,
-		Attributes: d.Attributes.Clone(),
-		//runtime: luaruntime.New(),
-		Config: d.Config,
-	}
-}
-
-func (d *Declaration) Load(r io.Reader) error {
-	return codec.NewYAMLDecoder(r).Decode(d)
-}
-
-func (d *Declaration) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(d)
-}
-
-func (d *Declaration) JSON() ([]byte, error) {
-	return json.Marshal(d)
-}
-
-func (d *Declaration) Validate() (err error) {
-	var declarationJson []byte
-	if declarationJson, err = d.JSON(); err == nil {
-		s := NewSchema(fmt.Sprintf("%s-declaration", d.Type))
-		err = s.Validate(string(declarationJson))
-	}
-	return err
-}
-
-func (d *Declaration) NewResource() error {
-	uri := fmt.Sprintf("%s://", d.Type)
-	newResource, err := ResourceTypes.New(uri)
-	d.Attributes = newResource
-	return err
-}
-
-func (d *Declaration) Resource() Resource {
-	return d.Attributes
-}
-
-func (d *Declaration) Apply() (result error) {
-	defer func() {
-		if r := recover(); r != nil {
-			result = fmt.Errorf("%s", r)
-		}
-	}()
-
-	stater := d.Attributes.StateMachine()
-	slog.Info("Declaration.Apply()", "machine", stater, "machine.state", stater.CurrentState(), "uri", d.Attributes.URI())
-	switch d.Transition {
-	case "read":
-		result = stater.Trigger("read")
-	case "delete", "absent":
-		if stater.CurrentState() == "present" {
-			result = stater.Trigger("delete")
-		}
-	case "update":
-		if result = stater.Trigger("update"); result != nil {
-			return result
-		}
-		result = stater.Trigger("read")
-	default:
-		fallthrough
-	case "create", "present":
-		if stater.CurrentState() == "absent" || stater.CurrentState() == "unknown" {
-			if result = stater.Trigger("create"); result != nil {
-				return result
-			}
-		}
-		result = stater.Trigger("read")
-	}
-	return result
-}
-
-func (d *Declaration) SetConfig(configDoc *config.Document) {
-	if configDoc != nil {
-		if configDoc.Has(string(d.Config)) {
-			d.configBlock = configDoc.Get(string(d.Config))
-			d.Attributes.UseConfig(d.configBlock)
-		}
-	}
-}
-
-func (d *Declaration) SetURI(uri string) error {
-	slog.Info("Declaration.SetURI()", "uri", uri, "declaration", d)
-	d.Attributes = NewResource(uri)
-	if d.Attributes == nil {
-		return ErrUnknownResourceType
-	}
-	d.Type = TypeName(d.Attributes.Type())
-	_,e := d.Attributes.Read(context.Background()) // fix context
-	return e
-}
-
-
-func (d *Declaration) UnmarshalValue(value *DeclarationType) error {
-	d.Type = value.Type
-	d.Transition = value.Transition
-	d.Config = value.Config
-	newResource, resourceErr := ResourceTypes.New(fmt.Sprintf("%s://", value.Type))
-	if resourceErr != nil {
-		slog.Info("Declaration.UnmarshalValue", "value", value, "error", resourceErr)
-		return resourceErr
-	}
-	d.Attributes = newResource
-	return nil
-}
-
-func (d *Declaration) UnmarshalYAML(value *yaml.Node) error {
-	t := &DeclarationType{}
-	if unmarshalResourceTypeErr := value.Decode(t); unmarshalResourceTypeErr != nil {
-		return unmarshalResourceTypeErr
-	}
-
-	if err := d.UnmarshalValue(t); err != nil {
-		return err
-	}
-
-	resourceAttrs := struct {
-		Attributes yaml.Node `json:"attributes"`
-	}{}
-	if unmarshalAttributesErr := value.Decode(&resourceAttrs); unmarshalAttributesErr != nil {
-		return unmarshalAttributesErr
-	}
-	if unmarshalResourceErr := resourceAttrs.Attributes.Decode(d.Attributes); unmarshalResourceErr != nil {
-		return unmarshalResourceErr
-	}
-	return nil
-}
-
-func (d *Declaration) UnmarshalJSON(data []byte) error {
-	t := &DeclarationType{}
-	if unmarshalResourceTypeErr := json.Unmarshal(data, t); unmarshalResourceTypeErr != nil {
-		return unmarshalResourceTypeErr
-	}
-
-	if err := d.UnmarshalValue(t); err != nil {
-		return err
-	}
-
-	resourceAttrs := struct {
-		Attributes Resource `json:"attributes"`
-	}{Attributes: d.Attributes}
-	if unmarshalAttributesErr := json.Unmarshal(data, &resourceAttrs); unmarshalAttributesErr != nil {
-		return unmarshalAttributesErr
-	}
-
-	return nil
-}
-
-/*
-func (d *Declaration) MarshalJSON() ([]byte, error) {
-  buf := new(bytes.Buffer)
-  buf.WriteByte('"')
-  buf.WriteString("value"))
-  buf.WriteByte('"')
-  return buf.Bytes(), nil
-}
-*/
-
-func (d *Declaration) MarshalYAML() (any, error) {
-	return d, nil
-}
-
-/*
-func (l *LuaWorker) Receive(m message.Envelope) {
-  s := m.Sender()
-  switch b := m.Body().(type) {
-  case *message.Error:
-  //  case *worker.Terminated:
-  case *CodeExecute:
-    stackSize := l.runtime.Api().GetTop()
-    if e := l.runtime.LoadScriptFromString(b.Code); e != nil {
-      s.Send(message.New(&message.Error{ E: e }, l))
-    }
-    returnsCount := l.runtime.Api().GetTop() - stackSize
-    if len(b.Entrypoint) == 0 {
-      if ! l.runtime.Api().IsNil(-1) {
-        if returnsCount == 0 {
-            s.Send(message.New(&CodeResult{ Result: []interface{}{ 0 } }, l))
-        } else {
-          lr,le := l.runtime.CopyReturnValuesFromCall(int(returnsCount))
-          if le != nil {
-            s.Send(message.New(&message.Error{ E: le }, l))
-          } else {
-            s.Send(message.New(&CodeResult{ Result: lr }, l))
-          }
-        }
-      }
-    } else {
-      r,ce := l.runtime.CallFunction(b.Entrypoint, b.Args)
-      if ce != nil {
-        s.Send(message.New(&message.Error{ E: ce }, l))
-      }
-      s.Send(message.New(&CodeResult{ Result: r }, l))
-    }
-  }
-}
-*/

internal/resource/declaration_test.go

@@ -1,136 +0,0 @@
-// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
-
-
-package resource
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/stretchr/testify/assert"
-	_ "log"
-	_ "os"
-	"path/filepath"
-	"decl/internal/types"
-	"testing"
-)
-
-/*
-func TestYamlLoadDecl(t *testing.T) {
-
-	file := filepath.Join(TempDir, "fooread.txt")
-
-	resourceAttributes := make(map[string]any)
-	decl := fmt.Sprintf(`
-  path: "%s"
-  owner: "nobody"
-  group: "nobody"
-  mode: "0600"
-  content: |-
-    test line 1
-    test line 2
-`, file)
-
-	e := YamlLoadDecl(decl, &resourceAttributes)
-	assert.Equal(t, nil, e)
-
-	assert.Equal(t, "nobody", resourceAttributes["group"])
-}
-*/
-
-func TestNewResourceDeclaration(t *testing.T) {
-	resourceDeclaration := NewDeclaration()
-	assert.NotEqual(t, nil, resourceDeclaration)
-}
-
-func TestNewResourceDeclarationType(t *testing.T) {
-	file := filepath.Join(TempDir, "fooread.txt")
-
-	decl := fmt.Sprintf(`
-    type: file
-    attributes:
-      path: "%s"
-      owner: "nobody"
-      group: "nobody"
-      mode: "0600"
-      content: |-
-        test line 1
-        test line 2
-`, file)
-
-	resourceDeclaration := NewDeclaration()
-	assert.NotNil(t, resourceDeclaration)
-
-	e := resourceDeclaration.LoadDecl(decl)
-	assert.Nil(t, e)
-	assert.Equal(t, TypeName("file"), resourceDeclaration.Type)
-	assert.NotNil(t, resourceDeclaration.Attributes)
-}
-
-func TestDeclarationNewResource(t *testing.T) {
-	resourceDeclaration := NewDeclaration()
-	assert.NotNil(t, resourceDeclaration)
-
-	errNewUnknownResource := resourceDeclaration.NewResource()
-	assert.ErrorIs(t, errNewUnknownResource, types.ErrUnknownType)
-
-	resourceDeclaration.Type = "file"
-	errNewFileResource := resourceDeclaration.NewResource()
-	assert.Nil(t, errNewFileResource)
-
-	assert.NotNil(t, resourceDeclaration.Attributes)
-}
-
-func TestDeclarationJson(t *testing.T) {
-	fileDeclJson := `
-{
-  "type": "file",
-  "attributes": {
-    "path": "foo"
-  }
-}
-`
-	resourceDeclaration := NewDeclaration()
-	e := json.Unmarshal([]byte(fileDeclJson), resourceDeclaration)
-	assert.Nil(t, e)
-	assert.Equal(t, TypeName("file"), resourceDeclaration.Type)
-	assert.Equal(t, "foo", resourceDeclaration.Attributes.(*File).Path)
-
-	userDeclJson := `
-{
-  "type": "user",
-  "attributes": {
-    "name": "testuser",
-		"uid": "10012"
-  }
-}
-`
-	userResourceDeclaration := NewDeclaration()
-	ue := json.Unmarshal([]byte(userDeclJson), userResourceDeclaration)
-	assert.Nil(t, ue)
-	assert.Equal(t, TypeName("user"), userResourceDeclaration.Type)
-	assert.Equal(t, "testuser", userResourceDeclaration.Attributes.(*User).Name)
-	assert.Equal(t, "10012", userResourceDeclaration.Attributes.(*User).UID)
-
-}
-
-func TestDeclarationTransition(t *testing.T) {
-	fileName := filepath.Join(TempDir, "testdecl.txt")
-	fileDeclJson := fmt.Sprintf(`
-{
-  "type": "file",
-  "transition": "present",
-  "attributes": {
-    "path": "%s"
-  }
-}
-`, fileName)
-
-	resourceDeclaration := NewDeclaration()
-	e := json.Unmarshal([]byte(fileDeclJson), resourceDeclaration)
-	assert.Nil(t, e)
-	assert.Equal(t, TypeName("file"), resourceDeclaration.Type)
-	assert.Equal(t, fileName, resourceDeclaration.Attributes.(*File).Path)
-	err := resourceDeclaration.Apply()
-	assert.Nil(t, err)
-	assert.FileExists(t, fileName)
-}

internal/resource/document.go

@@ -1,268 +0,0 @@
-// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
-
-package resource
-
-import (
-	"encoding/json"
-	"fmt"
-	"gopkg.in/yaml.v3"
-	"io"
-	"log/slog"
-_	"net/url"
-	"github.com/sters/yaml-diff/yamldiff"
-	"strings"
-	"decl/internal/codec"
-	"decl/internal/types"
-	"decl/internal/config"
-	"context"
-)
-
-type ResourceMap[Value any] map[string]Value
-
-func (rm ResourceMap[Value]) Get(key string) (any, bool) {
-	v, ok := rm[key]
-	return v, ok
-}
-
-type ResourceMapper interface {
-	Get(key string) (any, bool)
-}
-
-type Document struct {
-  uris ResourceMap[*Declaration]
-	ResourceDecls []Declaration `json:"resources" yaml:"resources"`
-	config *config.Document
-}
-
-func NewDocument() *Document {
-	return &Document{ uris: make(ResourceMap[*Declaration]) }
-}
-
-func (d *Document) Types() *types.Types[Resource] {
-	return ResourceTypes
-}
-
-func (d *Document) Filter(filter ResourceSelector) []*Declaration {
-	resources := make([]*Declaration, 0, len(d.ResourceDecls))
-	for i := range d.ResourceDecls {
-		filterResource := &d.ResourceDecls[i]
-		if filter == nil || filter(filterResource) {
-			resources = append(resources, &d.ResourceDecls[i])
-		}
-	}
-	return resources
-}
-
-func (d *Document) GetResource(uri string) *Declaration {
-	if decl, ok := d.uris[uri]; ok {
-		return decl
-	}
-	return nil
-}
-
-func (d *Document) Clone() *Document {
-	clone := NewDocument()
-	clone.config = d.config
-	clone.ResourceDecls = make([]Declaration, len(d.ResourceDecls))
-	for i, res := range d.ResourceDecls {
-		clone.ResourceDecls[i] = *res.Clone()
-		clone.ResourceDecls[i].SetDocument(clone)
-		clone.ResourceDecls[i].SetConfig(d.config)
-	}
-	return clone
-}
-
-func (d *Document) Load(r io.Reader) (err error) {
-	c := codec.NewYAMLDecoder(r)
-	err = c.Decode(d)
-	slog.Info("Document.Load()", "error", err)
-	if err == nil {
-		for i := range d.ResourceDecls {
-			d.ResourceDecls[i].SetDocument(d)
-		}
-	}
-	return
-}
-
-func (d *Document) Validate() error {
-	jsonDocument, jsonErr := d.JSON()
-	slog.Info("document.Validate() json", "json", jsonDocument, "err", jsonErr)
-	if jsonErr == nil {
-		s := NewSchema("document")
-		err := s.Validate(string(jsonDocument))
-		if err != nil {
-			return err
-		}
-/*
-		for i := range d.ResourceDecls {
-			if e := d.ResourceDecls[i].Resource().Validate(); e != nil {
-				return fmt.Errorf("failed to validate resource %s; %w", d.ResourceDecls[i].Resource().URI(), e)
-			}
-		}
-*/
-	}
-	return nil
-}
-
-func (d *Document) SetConfig(config *config.Document) {
-	d.config = config
-}
-
-func (d *Document) ConfigDoc() *config.Document {
-	return d.config
-}
-
-func (d *Document) Resources() []Declaration {
-	return d.ResourceDecls
-}
-
-func (d *Document) ResolveIds(ctx context.Context) {
-	for i := range d.ResourceDecls {
-		d.ResourceDecls[i].ResolveId(ctx)
-	}
-}
-
-func (d *Document) Apply(state string) error {
-	if d == nil {
-		panic("Undefined Document")
-	}
-	slog.Info("Document.Apply()", "declarations", d, "override", state)
-	var start, i int = 0, 0
-	if state == "delete" {
-		start = len(d.ResourceDecls) - 1
-	}
-	for {
-		idx := i - start
-		if idx < 0 { idx = - idx }
-
-		slog.Info("Document.Apply() applying resource", "index", idx, "uri", d.ResourceDecls[idx].Resource().URI(), "resource", d.ResourceDecls[idx].Resource())
-		if state != "" {
-			d.ResourceDecls[idx].Transition = state
-		}
-		d.ResourceDecls[idx].SetConfig(d.config)
-		if e := d.ResourceDecls[idx].Apply(); e != nil {
-			slog.Error("Document.Apply() error applying resource", "index", idx, "uri", d.ResourceDecls[idx].Resource().URI(), "resource", d.ResourceDecls[idx].Resource(), "error", e)
-			return e
-		}
-		if i >= len(d.ResourceDecls) - 1 {
-			break
-		}
-		i++
-	}
-	return nil
-}
-
-func (d *Document) Generate(w io.Writer) error {
-	e := codec.NewYAMLEncoder(w)
-	err := e.Encode(d);
-	if err == nil {
-		return e.Close()
-	}
-	e.Close()
-	return err
-}
-
-func (d *Document) MapResourceURI(uri string, declaration *Declaration) {
-	d.uris[uri] = declaration
-}
-
-func (d *Document) AddResourceDeclaration(resourceType string, resourceDeclaration Resource) {
-	slog.Info("Document.AddResourceDeclaration()", "type", resourceType, "resource", resourceDeclaration)
-	decl := NewDeclarationFromDocument(d)
-	decl.Type = TypeName(resourceType)
-	decl.Attributes = resourceDeclaration
-	d.ResourceDecls = append(d.ResourceDecls, *decl)
-	d.MapResourceURI(decl.Attributes.URI(), decl)
-	decl.SetDocument(d)
-}
-
-func (d *Document) AddResource(uri string) error {
-	decl := NewDeclarationFromDocument(d)
-	if e := decl.SetURI(uri); e != nil {
-		return e
-	}
-	d.ResourceDecls = append(d.ResourceDecls, *decl)
-	d.MapResourceURI(decl.Attributes.URI(), decl)
-	decl.SetDocument(d)
-	return nil
-}
-
-func (d *Document) JSON() ([]byte, error) {
-	return json.Marshal(d)
-}
-
-func (d *Document) YAML() ([]byte, error) {
-	return yaml.Marshal(d)
-}
-
-func (d *Document) Diff(with *Document, output io.Writer) (returnOutput string, diffErr error) {
-	defer func() {
-		if r := recover(); r != nil {
-			returnOutput = ""
-			diffErr = fmt.Errorf("%s", r)
-		}
-	}()
-	slog.Info("Document.Diff()")
-	opts := []yamldiff.DoOptionFunc{}
-	if output == nil {
-		output = &strings.Builder{}
-	}
-	ydata, yerr := d.YAML()
-	if yerr != nil {
-		return "", yerr
-	}
-	yamlDiff,yamlDiffErr := yamldiff.Load(string(ydata))
-	if yamlDiffErr != nil {
-		return "", yamlDiffErr
-	}
-	
-	wdata,werr := with.YAML()
-	if werr != nil {
-		return "", werr
-	}
-	withDiff,withDiffErr := yamldiff.Load(string(wdata))
-	if withDiffErr != nil {
-		return "", withDiffErr
-	}
-	
-	for _,docDiffResults := range yamldiff.Do(yamlDiff, withDiff, opts...) {
-		slog.Info("Diff()", "diff", docDiffResults, "dump", docDiffResults.Dump())
-		_,e := output.Write([]byte(docDiffResults.Dump()))
-		if e != nil {
-			return "", e
-		}
-	}
-	slog.Info("Document.Diff() ", "document.yaml", ydata, "with.yaml", wdata)
-	if stringOutput, ok := output.(*strings.Builder); ok {
-		return stringOutput.String(), nil
-	}
-	return "", nil
-}
-
-
-func (d *Document) UnmarshalYAML(value *yaml.Node) error {
-	type decodeDocument Document
-	t := (*decodeDocument)(d)
-	if unmarshalDocumentErr := value.Decode(t); unmarshalDocumentErr != nil {
-		return unmarshalDocumentErr
-	}
-	for i := range d.ResourceDecls {
-		d.ResourceDecls[i].SetDocument(d)
-		d.MapResourceURI(d.ResourceDecls[i].Attributes.URI(), &d.ResourceDecls[i])
-	}
-	return nil
-}
-
-func (d *Document) UnmarshalJSON(data []byte) error {
-	type decodeDocument Document
-	t := (*decodeDocument)(d)
-	if unmarshalDocumentErr := json.Unmarshal(data, t); unmarshalDocumentErr != nil {
-		return unmarshalDocumentErr
-	}
-	for i := range d.ResourceDecls {
-		d.ResourceDecls[i].SetDocument(d)
-		d.MapResourceURI(d.ResourceDecls[i].Attributes.URI(), &d.ResourceDecls[i])
-	}
-	return nil
-}
-

internal/resource/document_test.go

@@ -1,229 +0,0 @@
-// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
-
-package resource
-
-import (
-	"context"
-	"fmt"
-	"github.com/stretchr/testify/assert"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-	"syscall"
-	"testing"
-	"time"
-	"os/user"
-)
-
-func TestNewDocumentLoader(t *testing.T) {
-	d := NewDocument()
-	assert.NotNil(t, d)
-}
-
-func TestDocumentLoader(t *testing.T) {
-	dir, err := os.MkdirTemp("", "testdocumentloader")
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-
-	file, _ := filepath.Abs(filepath.Join(dir, "foo.txt"))
-
-	document := fmt.Sprintf(`
----
-resources:
-- type: file
-  attributes:
-    path: "%s"
-    owner: "%s"
-    group: "%s"
-    mode: "0600"
-    content: |-
-      test line 1
-      test line 2
-    state: present
-- type: user
-  attributes:
-    name: "testuser"
-    uid: "10022"
-    group: "10022"
-    home: "/home/testuser"
-    createhome: true
-    state: present
-`, file, ProcessTestUserName, ProcessTestGroupName)
-	d := NewDocument()
-	assert.NotNil(t, d)
-
-	docReader := strings.NewReader(document)
-
-	e := d.Load(docReader)
-	assert.Nil(t, e)
-
-	resources := d.Resources()
-	assert.Equal(t, 2, len(resources))
-}
-
-func TestDocumentGenerator(t *testing.T) {
-	ctx := context.Background()
-
-	fileContent := `// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
-`
-
-	file, _ := filepath.Abs(filepath.Join(TempDir, "foo.txt"))
-
-	err := os.WriteFile(file, []byte(fileContent), 0644)
-	assert.Nil(t, err)
-
-	info, statErr := os.Stat(file)
-	assert.Nil(t, statErr)
-	mTime := info.ModTime()
-	stat, ok := info.Sys().(*syscall.Stat_t)
-	assert.True(t, ok)
-
-	aTime := time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec))
-	cTime := time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec))
-	processUser, userErr := user.Current()
-	assert.Nil(t, userErr)
-	processGroup, groupErr := user.LookupGroupId(processUser.Gid)
-	assert.Nil(t, groupErr)
-
-	expected := fmt.Sprintf(`
-resources:
-  - type: file
-    attributes:
-      path: %s
-      owner: "%s"
-      group: "%s"
-      mode: "0644"
-      content: |
-        %s
-      atime: %s
-      ctime: %s
-      mtime: %s
-      sha256: ea33e2082ca777f82dc9571b08df95d81925eed04e1bdbac7cdc6dc52d330eca
-      size: 82
-      filetype: "regular"
-      state: present
-`, file, processUser.Username, processGroup.Name, fileContent, aTime.Format(time.RFC3339Nano), cTime.Format(time.RFC3339Nano), mTime.Format(time.RFC3339Nano))
-
-	var documentYaml strings.Builder
-	d := NewDocument()
-	assert.NotNil(t, d)
-
-	f, e := ResourceTypes.New("file://")
-	assert.Nil(t, e)
-	assert.NotNil(t, f)
-
-	f.(*File).Path = filepath.Join(TempDir, "foo.txt")
-	_,readErr := f.(*File).Read(ctx)
-	assert.Nil(t, readErr)
-	d.AddResourceDeclaration("file", f)
-	ey := d.Generate(&documentYaml)
-	assert.Nil(t, ey)
-
-	assert.Greater(t, documentYaml.Len(), 0)
-	assert.YAMLEq(t, expected, documentYaml.String())
-}
-
-func TestDocumentAddResource(t *testing.T) {
-	file, _ := filepath.Abs(filepath.Join(TempDir, "foo.txt"))
-	err := os.WriteFile(file, []byte(""), 0644)
-	assert.Nil(t, err)
-
-	d := NewDocument()
-	assert.NotNil(t, d)
-	e := d.AddResource(fmt.Sprintf("file://%s", file))
-	assert.Nil(t, e)
-}
-
-func TestDocumentJSON(t *testing.T) {
-	document := `
----
-resources:
-- type: user
-  attributes:
-    name: "testuser"
-    uid: "10022"
-    group: "10022"
-    home: "/home/testuser"
-    createhome: true
-    state: present
-`
-	d := NewDocument()
-	assert.NotNil(t, d)
-	docReader := strings.NewReader(document)
-
-	e := d.Load(docReader)
-	assert.Nil(t, e)
-
-	marshalledJSON, jsonErr := d.JSON()
-	assert.Nil(t, jsonErr)
-	assert.Greater(t, len(marshalledJSON), 0)
-}
-
-func TestDocumentJSONSchema(t *testing.T) {
-	document := NewDocument()
-	document.ResourceDecls = []Declaration{}
-	e := document.Validate()
-	assert.Nil(t, e)
-}
-
-func TestDocumentYAML(t *testing.T) {
-	document := `
----
-resources:
-- type: user
-  attributes:
-    name: "testuser"
-    uid: "10022"
-    group: "10022"
-    home: "/home/testuser"
-    createhome: true
-    state: present
-`
-	d := NewDocument()
-	assert.NotNil(t, d)
-	docReader := strings.NewReader(document)
-
-	e := d.Load(docReader)
-	assert.Nil(t, e)
-
-	marshalledYAML, yamlErr := d.YAML()
-	assert.Nil(t, yamlErr)
-	assert.YAMLEq(t, string(document), string(marshalledYAML))
-
-}
-
-func TestDocumentResourceFilter(t *testing.T) {
-	document := `
----
-resources:
-- type: user
-  attributes:
-    name: "testuser"
-    uid: "10022"
-    home: "/home/testuser"
-    state: present
-- type: file
-  attributes:
-    path: "foo.txt"
-    state: present
-- type: file
-  attributes:
-    path: "bar.txt"
-    state: present
-`
-
-	d := NewDocument()
-	assert.NotNil(t, d)
-	docReader := strings.NewReader(document)
-
-	e := d.Load(docReader)
-	assert.Nil(t, e)
-	
-	resources := d.Filter(func(d *Declaration) bool {
-		return d.Type == "file"
-	})
-	assert.Equal(t, 2, len(resources))
-}

internal/resource/group.go

@@ -18,6 +18,8 @@ _	"os"
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
 	"decl/internal/command"
+	"decl/internal/data"
+	"decl/internal/folio"
 )
 
 type decodeGroup Group
@@ -35,6 +37,7 @@ var ErrInvalidGroupType error = errors.New("invalid GroupType value")
 var SystemGroupType GroupType = FindSystemGroupType()
 
 type Group struct {
+	*Common	`json:"-" yaml:"-"`
 	stater		machine.Stater	`json:"-" yaml:"-"`
 	Name       string   `json:"name" yaml:"name"`
 	GID        string   `json:"gid,omitempty" yaml:"gid,omitempty"`
@@ -45,8 +48,8 @@ type Group struct {
 	UpdateCommand *command.Command `json:"-" yaml:"-"`
 	DeleteCommand *command.Command `json:"-" yaml:"-"`
 	State string           `json:"state,omitempty" yaml:"state,omitempty"`
-	config ConfigurationValueGetter
+	config data.ConfigurationValueGetter
-	Resources ResourceMapper `json:"-" yaml:"-"`
+	Resources data.ResourceMapper `json:"-" yaml:"-"`
 }
 
 func NewGroup() *Group {
@@ -54,7 +57,7 @@ func NewGroup() *Group {
 }
 
 func init() {
-	ResourceTypes.Register([]string{"group"}, func(u *url.URL) Resource {
+	folio.DocumentRegistry.ResourceTypes.Register([]string{"group"}, func(u *url.URL) data.Resource {
 		group := NewGroup()
 		group.Name = u.Hostname()
 		group.GID = LookupGIDString(u.Hostname())
@@ -79,11 +82,11 @@ func FindSystemGroupType() GroupType {
 	return GroupTypeAddGroup
 }
 
-func (g *Group) SetResourceMapper(resources ResourceMapper) {
+func (g *Group) SetResourceMapper(resources data.ResourceMapper) {
 	g.Resources = resources
 }
 
-func (g *Group) Clone() Resource {
+func (g *Group) Clone() data.Resource {
 	newg := &Group {
 		Name: g.Name,
 		GID: g.GID,
@@ -136,7 +139,7 @@ func (g *Group) URI() string {
 	return fmt.Sprintf("group://%s", g.Name)
 }
 
-func (g *Group) UseConfig(config ConfigurationValueGetter) {
+func (g *Group) UseConfig(config data.ConfigurationValueGetter) {
 	g.config = config
 }
 
@@ -149,26 +152,38 @@ func (g *Group) Validate() error {
 }
 
 func (g *Group) Apply() error {
+	ctx := context.Background()
 	switch g.State {
 	case "present":
 		_, NoGroupExists := LookupGID(g.Name)
 		if NoGroupExists != nil {
-			cmdErr := g.Create(context.Background())
+			cmdErr := g.Create(ctx)
 			return cmdErr
 		}
 	case "absent":
-		cmdErr := g.Delete()
+		cmdErr := g.Delete(ctx)
 		return cmdErr
 	}
 	return nil
 }
 
-func (g *Group) Load(r io.Reader) error {
+func (g *Group) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(g)
+	err = f.StringDecoder(string(docData)).Decode(g)
+	return
+}
+
+func (g *Group) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(g)
+	return
+}
+
+func (g *Group) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(g)
+	return
 }
 
 func (g *Group) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(g)
+	return g.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
 func (g *Group) Type() string { return "group" }
@@ -195,7 +210,11 @@ func (g *Group) Read(ctx context.Context) ([]byte, error) {
 	}
 }
 
-func (g *Group) Delete() (error) {
+func (g *Group) Update(ctx context.Context) (error) {
+	return g.Create(ctx)
+}
+
+func (g *Group) Delete(ctx context.Context) (error) {
 	_, err := g.DeleteCommand.Execute(g)
 	if err != nil {
 		return err

internal/resource/group_test.go

@@ -23,18 +23,18 @@ func TestNewGroupResource(t *testing.T) {
 func TestReadGroup(t *testing.T) {
 	ctx := context.Background()
   decl := `
-name: "syslog"
+name: "sys"
 `
 
 	g := NewGroup()
 	e := g.LoadDecl(decl)
 	assert.Nil(t, e)
-	assert.Equal(t, "syslog", g.Name)
+	assert.Equal(t, "sys", g.Name)
 
 	_, readErr := g.Read(ctx)
 	assert.Nil(t, readErr)
 
-	assert.Equal(t, "111", g.GID)
+	assert.Equal(t, "3", g.GID)
 
 }
 

internal/resource/http.go

@@ -16,15 +16,33 @@ _	"os"
 	"log/slog"
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
+	"decl/internal/data"
+	"decl/internal/transport"
+	"decl/internal/folio"
+	"decl/internal/iofilter"
+	"crypto/sha256"
+	"encoding/hex"
+)
+
+const (
+	HTTPTypeName TypeName = "http"
 )
 
 func init() {
 	ResourceTypes.Register([]string{"http", "https"}, HTTPFactory)
 }
 
-func HTTPFactory(u *url.URL) Resource {
+func HTTPFactory(u *url.URL) data.Resource {
+	var err error
 	h := NewHTTP()
-	h.Endpoint = u.String()
+	(&h.Endpoint).SetURL(u)
+	h.parsedURI = u
+	if h.reader, err = transport.NewReader(u); err != nil {
+		panic(err)
+	}
+	if h.writer, err = transport.NewWriter(u); err != nil {
+		panic(err)
+	}
 	return h
 }
 
@@ -35,33 +53,44 @@ type HTTPHeader struct {
 
 // Manage the state of an HTTP endpoint
 type HTTP struct {
+	*Common `yaml:",inline" json:",inline"`
+	parsedURI *url.URL `yaml:"-" json:"-"`
 	stater machine.Stater `yaml:"-" json:"-"`
 	client *http.Client `yaml:"-" json:"-"`
-	Endpoint string `yaml:"endpoint" json:"endpoint"`
+	Endpoint folio.URI `yaml:"endpoint" json:"endpoint"`
+	
 	Headers []HTTPHeader `yaml:"headers,omitempty" json:"headers,omitempty"`
-	Body  string `yaml:"body,omitempty" json:"body,omitempty"`
+	Content  string `yaml:"body,omitempty" json:"body,omitempty"`
+	ContentSourceRef  folio.ResourceReference   `json:"sourceref,omitempty" yaml:"sourceref,omitempty"`
 	Status string `yaml:"status,omitempty" json:"status,omitempty"`
 	StatusCode int `yaml:"statuscode,omitempty" json:"statuscode,omitempty"`
-	State string `yaml:"state,omitempty" json:"state,omitempty"`
+	Sha256 string `yaml:"sha256,omitempty" json:"sha256,omitempty"`
-	config ConfigurationValueGetter
+	SerializeContent bool `json:"serializecontent,omitempty" yaml:"serializecontent,omitempty"`
-	Resources ResourceMapper `yaml:"-" json:"-"`
+	Size int64 `yaml:"size,omitempty" json:"size,omitempty"`
+	SignatureValue string `yaml:"signature,omitempty" json:"signature,omitempty"`
+	config data.ConfigurationValueGetter
+	Resources data.ResourceMapper `yaml:"-" json:"-"`
+	reader *transport.Reader `yaml:"-" json:"-"`
+	writer *transport.Writer `yaml:"-" json:"-"`
 }
 
 func NewHTTP() *HTTP {
 	return &HTTP{ client: &http.Client{} }
 }
 
-func (h *HTTP) SetResourceMapper(resources ResourceMapper) {
+func (h *HTTP) SetResourceMapper(resources data.ResourceMapper) {
 	h.Resources = resources
 }
 
-func (h *HTTP) Clone() Resource {
+func (h *HTTP) Clone() data.Resource {
 	return &HTTP {
+		Common: &Common{ includeQueryParamsInURI: true, resourceType: HTTPTypeName },
 		client: h.client,
 		Endpoint: h.Endpoint,
 		Headers: h.Headers,
-		Body: h.Body,
+		Content: h.Content,
-		State: h.State,
+		reader: h.reader,
+		writer: h.writer,
 	}
 }
 
@@ -83,11 +112,11 @@ func (h *HTTP) Notify(m *machine.EventMessage) {
 				if triggerErr := h.StateMachine().Trigger("state_read"); triggerErr == nil {
 					return
 				} else {
-					h.State = "absent"
+					h.Common.State = "absent"
 					panic(triggerErr)
 				}
 			} else {
-				h.State = "absent"
+				h.Common.State = "absent"
 				panic(readErr)
 			}
     case "start_create":
@@ -96,41 +125,50 @@ func (h *HTTP) Notify(m *machine.EventMessage) {
 					return
 				}
       }
-			h.State = "absent"
+			h.Common.State = "absent"
 		case "start_delete":
 			if deleteErr := h.Delete(ctx); deleteErr == nil {
 				if triggerErr := h.StateMachine().Trigger("deleted"); triggerErr == nil {
 					return
 				} else {
-					h.State = "present"
+					h.Common.State = "present"
 					panic(triggerErr)
 				}
 			} else {
-				h.State = "present"
+				h.Common.State = "present"
 				panic(deleteErr)
 			}
 		case "absent":
-			h.State = "absent"
+			h.Common.State = "absent"
     case "present", "created", "read":
-      h.State = "present"
+      h.Common.State = "present"
     }
   case machine.EXITSTATEEVENT:
   }
 }
 
 func (h *HTTP) URI() string {
-	return h.Endpoint
+	return string(h.Endpoint)
 }
 
-func (h *HTTP) SetURI(uri string) error {
+func (h *HTTP) setParsedURI(uri folio.URI) error {
-	if _, e := url.Parse(uri); e != nil {
+	if parsed := uri.Parse(); parsed == nil {
-		return fmt.Errorf("%w: %s is not a file", ErrInvalidResourceURI, uri)
+		return folio.ErrInvalidURI
+	} else {
+		h.parsedURI = parsed
 	}
-	h.Endpoint = uri
 	return nil
 }
 
-func (h *HTTP) UseConfig(config ConfigurationValueGetter) {
+func (h *HTTP) SetURI(uri string) (err error) {
+	v := folio.URI(uri)
+	if err = h.setParsedURI(v); err == nil {
+		h.Endpoint = v
+	}
+	return
+}
+
+func (h *HTTP) UseConfig(config data.ConfigurationValueGetter) {
 	h.config = config
 }
 
@@ -148,32 +186,114 @@ func (h *HTTP) Validate() error {
 }
 
 func (h *HTTP) Apply() error {
-	switch h.State {
+	switch h.Common.State {
 	case "absent":
 	case "present":
 	}
 	_,e := h.Read(context.Background())
 	if e == nil {
-		h.State = "present"
+		h.Common.State = "present"
 	}
 	return e
 }
 
-func (h *HTTP) Load(r io.Reader) error {
+func (h *HTTP) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(h)
+	if err = f.StringDecoder(string(docData)).Decode(h); err == nil {
+		err = h.setParsedURI(h.Endpoint)
+	}
+	return
+}
+
+func (h *HTTP) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	if err = f.Decoder(r).Decode(h); err == nil {
+		err = h.setParsedURI(h.Endpoint)
+	}
+	return
+}
+
+func (h *HTTP) LoadString(docData string, f codec.Format) (err error) {
+	if err = f.StringDecoder(docData).Decode(h); err == nil {
+		err = h.setParsedURI(h.Endpoint)
+	}
+	return
 }
 
 func (h *HTTP) LoadDecl(yamlResourceDeclaration string) error {
-	slog.Info("LoadDecl()", "yaml", yamlResourceDeclaration)
+	return h.LoadString(yamlResourceDeclaration, codec.FormatYaml)
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(h)
 }
 
 func (h *HTTP) ResolveId(ctx context.Context) string {
-	return h.Endpoint
+	return h.Endpoint.String()
 }
 
-func (h *HTTP) Create(ctx context.Context) error {
+func (h *HTTP) Signature() folio.Signature {
-	body := strings.NewReader(h.Body)
+	var s folio.Signature
+	if e := (&s).SetHexString(h.SignatureValue); e != nil {
+		panic(e)
+	}
+	return s
+}
+
+func (h *HTTP) Hash() (shaBytes []byte) {
+	shaBytes, _ = hex.DecodeString(h.Sha256)
+	return
+}
+
+func (h *HTTP) HashHexString() string {
+	return h.Sha256
+}
+
+func (h *HTTP) Create(ctx context.Context) (err error) {
+	slog.Error("HTTP.Create()", "http", h)
+
+	var contentReader io.ReadCloser
+	h.writer, err = transport.NewWriterWithContext(h.parsedURI, ctx)
+	if err != nil {
+		slog.Error("HTTP.Create()", "http", h, "error", err)
+		//panic(err)
+		return
+	}
+	
+	slog.Error("HTTP.Create() content", "http", h)
+
+	contentReader, err = h.contentSourceReader()
+	if err != nil {
+		return
+	}
+	contentReader = h.UpdateContentAttributesFromReader(contentReader)
+
+	if err != nil {
+		return
+	}
+
+	defer func () {
+		h.writer.Close()
+		contentReader.Close()
+	}()
+
+	err = h.AddAuthorizationTokenFromConfigToTransport()
+	if err != nil {
+		return
+	}
+	
+	if len(h.Headers) > 0 {
+		for _, header := range h.Headers {
+			h.writer.AddHeader(header.Name, header.Value)
+		}
+	}
+
+	slog.Error("HTTP.Create()", "http", h)
+
+	copyBuffer := make([]byte, 32 * 1024)
+	_, writeErr := io.CopyBuffer(h.writer, contentReader, copyBuffer)
+	if writeErr != nil {
+		return fmt.Errorf("Http.Create(): CopyBuffer failed %v %v: %w", h.writer, contentReader, writeErr)
+	}
+	h.Status = h.writer.Status()
+	h.StatusCode = h.writer.StatusCode()
+	return
+/*
+	body := strings.NewReader(h.Content)
 	req, reqErr := http.NewRequest("POST", h.Endpoint, body)
 	if reqErr != nil {
 		return reqErr
@@ -195,6 +315,11 @@ func (h *HTTP) Create(ctx context.Context) error {
 	}
 	defer resp.Body.Close()
 	return err
+*/
+}
+
+func (h *HTTP) Update(ctx context.Context) error {
+	return h.Create(ctx)
 }
 
 func (h *HTTP) ReadAuthorizationTokenFromConfig(req *http.Request) error {
@@ -209,7 +334,153 @@ func (h *HTTP) ReadAuthorizationTokenFromConfig(req *http.Request) error {
 	return nil
 }
 
-func (h *HTTP) Read(ctx context.Context) ([]byte, error) {
+func (h *HTTP) AddAuthorizationTokenFromConfigToTransport() (err error) {
+	if h.config != nil {
+		token, tokenErr := h.config.GetValue("authorization_token")
+		if tokenErr == nil {
+			if h.reader != nil {
+				h.reader.AddHeader("Authorization", fmt.Sprintf("Bearer %s", token))
+			}
+			if h.writer != nil {
+				h.writer.AddHeader("Authorization", fmt.Sprintf("Bearer %s", token))
+			}
+		} else {
+			err = tokenErr
+		}
+	}
+	return
+}
+
+func (h *HTTP) UpdateContentAttributes() {
+	h.Size = int64(len(h.Content))
+	h.Sha256 = fmt.Sprintf("%x", sha256.Sum256([]byte(h.Content)))
+}
+
+func (h *HTTP) UpdateContentAttributesFromReader(reader io.ReadCloser) io.ReadCloser {
+	var content strings.Builder
+	hash := sha256.New()
+	h.Size = 0
+	h.Content = ""
+	h.Sha256 = ""
+	return iofilter.NewReader(reader, func(p []byte, readn int, readerr error) (n int, err error) {
+		hash.Write(p[:readn])
+		h.Sha256 = fmt.Sprintf("%x", hash.Sum(nil))
+		h.Size += int64(readn)
+		if len(h.ContentSourceRef) == 0 || h.SerializeContent {
+			content.Write(p[:readn])
+			h.Content = content.String()
+		}
+		return readn, readerr
+	})
+}
+
+func (h *HTTP) SetContent(r io.Reader) error {
+	fileContent, ioErr := io.ReadAll(r)
+	h.Content = string(fileContent)
+	h.UpdateContentAttributes()
+	return ioErr
+}
+
+func (h *HTTP) GetContent(w io.Writer) (contentReader io.ReadCloser, err error) {
+	slog.Info("Http.GetContent()", "content", len(h.Content), "sourceref", h.ContentSourceRef)
+	contentReader, err = h.readThru(context.Background())
+
+	if w != nil {
+		copyBuffer := make([]byte, 32 * 1024)
+		_, writeErr := io.CopyBuffer(w, contentReader, copyBuffer)
+		if writeErr != nil {
+			return nil, fmt.Errorf("Http.GetContent(): CopyBuffer failed %v %v: %w", w, contentReader, writeErr)
+		}
+		return nil, nil
+	}
+	if v, ok := contentReader.(*transport.Reader); ok {
+		h.SignatureValue = v.Signature()
+	} 
+	return
+}
+
+/*
+func (h *HTTP) writeThru(ctx context.Context) (contentWriter io.WriteCloser, err error) {
+	if len(h.ContentSourceRef) != 0 {
+		contentReader, err = h.ContentSourceRef.Lookup(nil).ContentReaderStream()
+		contentReader.(*transport.Reader).SetGzip(false)
+	} else {
+		if len(h.Content) != 0 {
+			contentReader = io.NopCloser(strings.NewReader(h.Content))
+		} else {
+			//contentReader, err = os.Open(f.Path)
+			contentReader = transport.NewReaderWithContext(u, ctx)
+			contentReader.(*transport.Reader).SetGzip(false)
+		}
+	}
+	contentReader = h.UpdateContentAttributesFromReader(contentReader)
+	return
+}
+*/
+
+func (h *HTTP) contentSourceReader() (contentReader io.ReadCloser, err error) {
+	if len(h.ContentSourceRef) != 0 {
+		contentReader, err = h.ContentSourceRef.Lookup(nil).ContentReaderStream()
+		contentReader.(*transport.Reader).SetGzip(false)
+	} else {
+		if len(h.Content) != 0 {
+			contentReader = io.NopCloser(strings.NewReader(h.Content))
+		} else {
+			err = fmt.Errorf("Cannot create reader: no content defined")
+		}
+	}
+	return
+}
+
+// set up reader for source content
+func (h *HTTP) readThru(ctx context.Context) (contentReader io.ReadCloser, err error) {
+	if contentReader, err = h.contentSourceReader(); err != nil {
+		if h.reader == nil {
+			h.reader, err = transport.NewReaderWithContext(h.parsedURI, ctx)
+			h.reader.SetGzip(false)
+		}
+		contentReader = h.reader
+	}
+	contentReader = h.UpdateContentAttributesFromReader(contentReader)
+	return
+}
+
+func (h *HTTP) Read(ctx context.Context) (yamlData []byte, err error) {
+	var contentReader io.ReadCloser
+	contentReader, err = h.readThru(ctx)
+	if err != nil {
+		return
+	}
+
+	defer contentReader.Close()
+	err = h.AddAuthorizationTokenFromConfigToTransport()
+	if err != nil {
+		return
+	}
+	
+	if len(h.Headers) > 0 {
+		for _, header := range h.Headers {
+			h.reader.AddHeader(header.Name, header.Value)
+		}
+	}
+
+	slog.Info("HTTP.Read()", "reader", h.reader)
+/*
+	copyBuffer := make([]byte, 32 * 1024)
+	_, writeErr := io.CopyBuffer(w, h.reader, copyBuffer)
+	if writeErr != nil {
+		return nil, fmt.Errorf("Http.GetContent(): CopyBuffer failed %v %v: %w", w, contentReader, writeErr)
+	}
+*/
+	if err = h.SetContent(contentReader); err != nil {
+		return
+	}
+	
+	h.Status = h.reader.Status()
+	h.StatusCode = h.reader.StatusCode()
+	return yaml.Marshal(h)
+
+/*	
 	req, reqErr := http.NewRequestWithContext(ctx, "GET", h.Endpoint, nil)
 	if reqErr != nil {
 		return nil, reqErr
@@ -241,6 +512,7 @@ func (h *HTTP) Read(ctx context.Context) ([]byte, error) {
 	}
 	h.Body = string(body)
 	return yaml.Marshal(h)
+*/
 }
 
 func (h *HTTP) Delete(ctx context.Context) error {

internal/resource/http_test.go

@@ -4,18 +4,18 @@ package resource
 
 import (
 	"context"
-	_ "encoding/json"
+_ "encoding/json"
 	"fmt"
 	"github.com/stretchr/testify/assert"
-	_ "gopkg.in/yaml.v3"
+_ "gopkg.in/yaml.v3"
 	"io"
-	_ "log"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
-	_ "net/url"
+_ "net/url"
-	_ "os"
+_ "os"
-	_ "path/filepath"
+_ "path/filepath"
-	_ "strings"
+_ "strings"
 	"testing"
 	"regexp"
 )
@@ -35,7 +35,7 @@ body: |-
 `
 
 	assert.Nil(t, h.LoadDecl(decl))
-	assert.Equal(t, "test body", h.Body)
+	assert.Equal(t, "test body", h.Content)
 }
 
 func TestHTTPRead(t *testing.T) {
@@ -62,7 +62,7 @@ endpoint: "%s/resource/user/foo"
 	assert.Nil(t, h.LoadDecl(decl))
 	_,e := h.Read(context.Background())
 	assert.Nil(t, e)
-	assert.Greater(t, len(h.Body), 0)
+	assert.Greater(t, len(h.Content), 0)
 	assert.Nil(t, h.Validate())
 }
 
@@ -83,6 +83,7 @@ attributes:
 		body, err := io.ReadAll(req.Body)
 		assert.Nil(t, err)
 		assert.Equal(t, userdecl, string(body))
+		assert.Equal(t, "application/yaml", req.Header.Get("content-type"))
 	}))
 	defer server.Close()
 
@@ -96,7 +97,10 @@ body: |
 %s
 `, server.URL, re.ReplaceAllString(userdecl, "  $1"))
 	assert.Nil(t, h.LoadDecl(decl))
-	assert.Greater(t, len(h.Body), 0)
+	assert.Greater(t, len(h.Content), 0)
+
+	slog.Info("TestHTTPCreate()", "resource", h, "decl", decl)
 	e := h.Create(ctx)
 	assert.Nil(t, e)
+	
 }

internal/resource/iptables.go

@@ -19,23 +19,31 @@ _	"os/exec"
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
 	"decl/internal/command"
+	"decl/internal/data"
+	"decl/internal/folio"
+)
+
+const (
+	IptableTypeName TypeName = "iptable"
 )
 
 func init() {
-	ResourceTypes.Register([]string{"iptable"}, func(u *url.URL) Resource {
+	folio.DocumentRegistry.ResourceTypes.Register([]string{"iptable"}, func(u *url.URL) data.Resource {
 		i := NewIptable()
 		i.Table = IptableName(u.Hostname())
 		if len(u.Path) > 0 {
-			fields := strings.Split(u.Path, "/")
+			fields := strings.FieldsFunc(u.Path, func(c rune) bool { return c == '/' })
 			slog.Info("iptables factory", "iptable", i, "uri", u, "fields", fields, "number_fields", len(fields))
-			i.Chain = IptableChain(fields[1])
+			if len(fields) > 0 {
+				i.Chain = IptableChain(fields[0])
 				if len(fields) < 3 {
 					i.ResourceType = IptableTypeChain
 				} else {
 					i.ResourceType = IptableTypeRule
-				id, _ := strconv.ParseUint(fields[2], 10, 32)
+					id, _ := strconv.ParseUint(fields[1], 10, 32)
 					i.Id = uint(id)
 				}
+			}
 			i.CreateCommand, i.ReadCommand, i.UpdateCommand, i.DeleteCommand = i.ResourceType.NewCRUD()
 		}
 		return i
@@ -52,11 +60,11 @@ const (
 type IptableName string
 
 const (
-	IptableNameFilter     = "filter"
+	IptableNameFilter IptableName     = "filter"
-	IptableNameNat        = "nat"
+	IptableNameNat IptableName        = "nat"
-	IptableNameMangel     = "mangle"
+	IptableNameMangel IptableName     = "mangle"
-	IptableNameRaw        = "raw"
+	IptableNameRaw IptableName        = "raw"
-	IptableNameSecurity   = "security"
+	IptableNameSecurity IptableName   = "security"
 )
 
 var IptableNumber = regexp.MustCompile(`^[0-9]+$`)
@@ -102,10 +110,16 @@ const (
 	IptableTypeChain = "chain"
 )
 
+var (
+	ErrInvalidIptableName error = errors.New("The IptableName is not a valid table")
+)
+
 // Manage the state of iptables rules
 // iptable://filter/INPUT/0
 type Iptable struct {
+	*Common                       `json:",inline" yaml:",inline"`
 	stater				machine.Stater	`json:"-" yaml:"-"`
+	parsedURI			*url.URL				`json:"-" yaml:"-"`
 	Id						uint						`json:"id,omitempty" yaml:"id,omitempty"`
 	Table					IptableName			`json:"table" yaml:"table"`
 	Chain					IptableChain		`json:"chain" yaml:"chain"`
@@ -119,7 +133,6 @@ type Iptable struct {
 	Flags					[]ExtensionFlag	`json:"extension_flags,omitempty" yaml:"extension_flags,omitempty"`
 	Proto					IptableProto		`json:"proto,omitempty" yaml:"proto,omitempty"`
 	Jump					string					`json:"jump,omitempty" yaml:"jump,omitempty"`
-	State					string          `json:"state" yaml:"state"`
 	ChainLength		uint						`json:"-" yaml:"-"`
 
 	ResourceType	IptableType `json:"resourcetype,omitempty" yaml:"resourcetype,omitempty"`
@@ -128,22 +141,33 @@ type Iptable struct {
   UpdateCommand *command.Command `yaml:"-" json:"-"`
   DeleteCommand *command.Command `yaml:"-" json:"-"`
 
-	config ConfigurationValueGetter
+	config data.ConfigurationValueGetter
-	Resources ResourceMapper `yaml:"-" json:"-"`
+	Resources data.ResourceMapper `yaml:"-" json:"-"`
+}
+
+
+func (n IptableName) Validate() error {
+	switch n {
+	case IptableNameFilter, IptableNameNat, IptableNameMangel, IptableNameRaw, IptableNameSecurity:
+		return nil
+	default:
+		return ErrInvalidIptableName
+	}
 }
 
 func NewIptable() *Iptable {
-	i := &Iptable{ ResourceType: IptableTypeRule }
+	i := &Iptable{ ResourceType: IptableTypeRule, Common: &Common{ resourceType: IptableTypeName } }
 	i.CreateCommand, i.ReadCommand, i.UpdateCommand, i.DeleteCommand = i.ResourceType.NewCRUD()
 	return i
 }
 
-func (i *Iptable) SetResourceMapper(resources ResourceMapper) {
+func (i *Iptable) SetResourceMapper(resources data.ResourceMapper) {
 	i.Resources = resources
 }
 
-func (i *Iptable) Clone() Resource {
+func (i *Iptable) Clone() data.Resource {
 	newIpt :=  &Iptable {
+		Common: i.Common,
 		Id: i.Id,
 		Table: i.Table,
 		Chain: i.Chain,
@@ -154,7 +178,6 @@ func (i *Iptable) Clone() Resource {
 		Match: i.Match,
 		Proto: i.Proto,
 		ResourceType: i.ResourceType,
-		State: i.State,
 	}
 	newIpt.CreateCommand, newIpt.ReadCommand, newIpt.UpdateCommand, newIpt.DeleteCommand = newIpt.ResourceType.NewCRUD()
 	return newIpt
@@ -178,9 +201,9 @@ func (i *Iptable) Notify(m *machine.EventMessage) {
 					return
 				}
 			}
-			i.State = "absent"
+			i.Common.State = "absent"
 		case "present":
-			i.State = "present"
+			i.Common.State = "present"
 		}
 	case machine.EXITSTATEEVENT:
 	}
@@ -190,28 +213,32 @@ func (i *Iptable) URI() string {
 	return fmt.Sprintf("iptable://%s/%s/%d", i.Table, i.Chain, i.Id)
 }
 
-func (i *Iptable) SetURI(uri string) error {
+func (i *Iptable) SetURI(uri string) (err error) {
-	resourceUri, e := url.Parse(uri)
+	i.parsedURI, err = url.Parse(uri)
-	if e == nil {
+	if err == nil {
-		if resourceUri.Scheme == "iptable" {
+		fields := strings.FieldsFunc(i.parsedURI.Path, func(c rune) bool { return c == '/' })
-			i.Table = IptableName(resourceUri.Hostname())
+		fieldsLen := len(fields)
-			fields := strings.Split(resourceUri.Path, "/")
+		if i.parsedURI.Scheme == "iptable" && fieldsLen > 0 {
-			i.Chain = IptableChain(fields[1])
+			i.Table = IptableName(i.parsedURI.Hostname())
-			if len(fields) < 3 {
+			if err = i.Table.Validate(); err != nil {
+				return err
+			}
+			i.Chain = IptableChain(fields[0])
+			if fieldsLen < 2 {
 				i.ResourceType = IptableTypeChain
 			} else {
 				i.ResourceType = IptableTypeRule
-				id, _ := strconv.ParseUint(fields[2], 10, 32)
+				id, _ := strconv.ParseUint(fields[1], 10, 32)
 				i.Id = uint(id)
 			}
 		} else {
-			e = fmt.Errorf("%w: %s is not an iptable rule", ErrInvalidResourceURI, uri)
+			err = fmt.Errorf("%w: %s is not an iptable rule", ErrInvalidResourceURI, uri)
 		}
 	}
-	return e
+	return
 }
 
-func (i *Iptable) UseConfig(config ConfigurationValueGetter) {
+func (i *Iptable) UseConfig(config data.ConfigurationValueGetter) {
 	i.config = config
 }
 
@@ -251,7 +278,7 @@ func (i *Iptable) NewCRUD() (create *command.Command, read *command.Command, upd
 
 func (i *Iptable) Apply() error {
 	ctx := context.Background()
-	switch i.State {
+	switch i.Common.State {
 	case "absent":
 	case "present":
 		err := i.Create(ctx)
@@ -263,15 +290,25 @@ func (i *Iptable) Apply() error {
 	return e
 }
 
-func (i *Iptable) Load(r io.Reader) error {
+func (i *Iptable) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(i)
+	err = f.StringDecoder(string(docData)).Decode(i)
+	return
+}
+
+func (i *Iptable) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(i)
+	return
+}
+
+func (i *Iptable) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(i)
+	return
 }
 
 func (i *Iptable) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(i)
+	return i.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
-
 func (i *Iptable) ResolveId(ctx context.Context) string {
 //	uri := fmt.Sprintf("%s?gateway=%s&interface=%s&rtid=%s&metric=%d&type=%s&scope=%s",
 //		n.To, n.Gateway, n.Interface, n.Rtid, n.Metric, n.RouteType, n.Scope)
@@ -462,6 +499,14 @@ func (i *Iptable) Read(ctx context.Context) ([]byte, error) {
 	return yaml.Marshal(i)
 }
 
+func (i *Iptable) Update(ctx context.Context) error {
+	return i.Create(ctx)
+}
+
+func (i *Iptable) Delete(ctx context.Context) error {
+	return nil
+}
+
 func (i *Iptable) Type() string { return "iptable" }
 
 func (i *IptableType) NewCRUD() (create *command.Command, read *command.Command, update *command.Command, del *command.Command) {
@@ -582,7 +627,7 @@ func NewIptableReadCommand() *command.Command {
 				lineNumber++
 			}
 		}
-		i.State = state
+		i.Common.State = state
 		if numberOfLines > 0 {
 			i.ChainLength = uint(numberOfLines) - 1
 		} else {
@@ -621,9 +666,9 @@ func NewIptableReadChainCommand() *command.Command {
 			if ruleFields[0] == "-A" {
 				flags := ruleFields[2:]
 				if i.SetRule(flags) {
-					i.State = "present"
+					i.Common.State = "present"
 				} else {
-					i.State = "absent"
+					i.Common.State = "absent"
 				}
 			}
 		}
@@ -670,13 +715,13 @@ func ChainExtractor(out []byte, target any) error {
 		case "-N", "-A":
 			chain := ruleFields[1]
 			if chain == string(i.Chain) {
-				i.State = "present"
+				i.Common.State = "present"
 				return nil
 			} else {
-				i.State = "absent"
+				i.Common.State = "absent"
 			}
 		default:
-			i.State = "absent"
+			i.Common.State = "absent"
 		}
 	}
 	return nil
@@ -686,7 +731,7 @@ func RuleExtractor(out []byte, target any) (err error) {
 	ipt := target.(*Iptable)
 	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
 	err = fmt.Errorf("Failed to extract rule by Id: %d", ipt.Id)
-	ipt.State = "absent"
+	ipt.Common.State = "absent"
 	var lineIndex uint = 1
 	if uint(len(lines)) >= ipt.Id {
 		lineIndex = ipt.Id
@@ -697,7 +742,7 @@ func RuleExtractor(out []byte, target any) (err error) {
 	slog.Info("RuleExtractor()", "lines", lines, "line", lines[lineIndex], "fields", ruleFields, "index", lineIndex)
 	if ruleFields[0] == "-A" {
 		if ipt.SetRule(ruleFields[2:]) {
-			ipt.State = "present"
+			ipt.Common.State = "present"
 			err = nil
 		}
 	}
@@ -711,7 +756,7 @@ func RuleExtractorMatchFlags(out []byte, target any) (err error) {
 	err = fmt.Errorf("Failed to extract rule")
 	if linesCount > 0 {
 		ipt.ChainLength = linesCount - 1
-		ipt.State = "absent"
+		ipt.Common.State = "absent"
 		for linesIndex, line := range lines {
 			ruleFields := strings.Split(strings.TrimSpace(line), " ")
 			slog.Info("RuleExtractorMatchFlags()", "lines", lines, "line", line, "fields", ruleFields, "index", linesIndex)
@@ -720,7 +765,7 @@ func RuleExtractorMatchFlags(out []byte, target any) (err error) {
 				if ipt.MatchRule(flags) {
 					slog.Info("RuleExtractorMatchFlags()", "flags", flags, "ipt", ipt)
 					err = nil
-					ipt.State = "present"
+					ipt.Common.State = "present"
 					ipt.Id = uint(linesIndex)
 					return
 				}
@@ -751,7 +796,7 @@ func RuleExtractorById(out []byte, target any) (err error) {
 			}
 		}
 	}
-	ipt.State = state
+	ipt.Common.State = state
 	return
 }
 
@@ -774,13 +819,13 @@ func NewIptableChainReadCommand() *command.Command {
 			case "-N", "-A":
 				chain := ruleFields[1]
 				if chain == string(i.Chain) {
-					i.State = "present"
+					i.Common.State = "present"
 					return nil
 				} else {
-					i.State = "absent"
+					i.Common.State = "absent"
 				}
 			default:
-				i.State = "absent"
+				i.Common.State = "absent"
 			}
 		}
 		return nil

internal/resource/mock_config_test.go

@@ -0,0 +1,17 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package resource
+
+import (
+_	"fmt"
+_	"github.com/stretchr/testify/assert"
+_	"os"
+_	"strings"
+_	"testing"
+)
+
+type MockConfig func(key string) (value any, err error)
+
+func (m MockConfig) GetValue(key string) (value any, err error) {
+	return m(key)
+}

internal/resource/mock_converter_test.go

@@ -0,0 +1,40 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package resource
+
+import (
+	"decl/internal/data"
+)
+
+type MockConverter struct {
+	InjectType func() data.TypeName
+	InjectEmit func(document data.Document, filter data.ElementSelector) (data.Resource, error)
+	InjectExtract func(resource data.Resource, filter data.ElementSelector) (data.Document, error)
+	InjectExtractMany func(resource data.Resource, filter data.ElementSelector) ([]data.Document, error)
+	InjectEmitMany func(documents []data.Document, filter data.ElementSelector) (data.Resource, error)
+	InjectClose func() error
+}
+
+func (m *MockConverter) Emit(document data.Document, filter data.ElementSelector) (data.Resource, error) {
+	return m.InjectEmit(document, filter)
+}
+
+func (m *MockConverter) EmitMany(documents []data.Document, filter data.ElementSelector) (data.Resource, error) {
+	return m.InjectEmitMany(documents, filter)
+}
+
+func (m *MockConverter) Extract(resource data.Resource, filter data.ElementSelector) (data.Document, error) {
+	return m.InjectExtract(resource, filter)
+}
+
+func (m *MockConverter) ExtractMany(resource data.Resource, filter data.ElementSelector) ([]data.Document, error) {
+	return m.InjectExtractMany(resource, filter)
+}
+
+func (m *MockConverter) Type() data.TypeName {
+  return m.InjectType()
+}
+
+func (m *MockConverter) Close() error {
+	return m.InjectClose()
+}

internal/resource/mock_foo_converter_test.go

@@ -0,0 +1,44 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package resource
+
+import (
+	"decl/internal/types"
+	"decl/internal/data"
+	"decl/internal/folio"
+	"net/url"
+)
+
+var (
+	TestConverterTypes *types.Types[data.Converter] = types.New[data.Converter]()
+)
+
+func RegisterConverterMocks() {
+	TestConverterTypes.Register([]string{"file"}, func(u *url.URL) data.Converter {
+		f := NewFileConverter()
+		return f
+	})
+}
+
+
+func NewFileConverter() *MockConverter {
+  return &MockConverter {
+		InjectType: func() data.TypeName { return "file" },
+		InjectEmit: func(document data.Document, filter data.ElementSelector) (data.Resource, error) {
+			return nil, nil
+		},
+		InjectExtract: func(resource data.Resource, filter data.ElementSelector) (document data.Document, err error) {
+			document = folio.DocumentRegistry.NewDocument("")
+			return
+		},
+		InjectExtractMany: func(resource data.Resource, filter data.ElementSelector) ([]data.Document, error) {
+			return nil, nil
+		},
+		InjectEmitMany: func(documents []data.Document, filter data.ElementSelector) (data.Resource, error) {
+			return nil, nil
+		},
+		InjectClose: func() error {
+			return nil
+		},
+  }
+}

internal/resource/mock_resource_test.go

@@ -8,6 +8,7 @@ _ "gopkg.in/yaml.v3"
 	"encoding/json"
 _	"fmt"
 	"gitea.rosskeen.house/rosskeen.house/machine"
+	"decl/internal/data"
 )
 
 type MockResource struct {
@@ -21,7 +22,7 @@ type MockResource struct {
   InjectStateMachine func() machine.Stater
 }
 
-func (m *MockResource) Clone() Resource {
+func (m *MockResource) Clone() data.Resource {
   return nil
 }
 

internal/resource/network_route.go

@@ -16,10 +16,16 @@ _	"strconv"
 	"strings"
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
+	"decl/internal/data"
+	"decl/internal/folio"
+)
+
+const (
+	NetworkRouteTypeName TypeName = "route"
 )
 
 func init() {
-	ResourceTypes.Register([]string{"route"}, func(u *url.URL) Resource {
+	folio.DocumentRegistry.ResourceTypes.Register([]string{"route"}, func(u *url.URL) data.Resource {
 		n := NewNetworkRoute()
 		return n
 	})
@@ -108,6 +114,7 @@ const (
 
 // Manage the state of network routes
 type NetworkRoute struct {
+	*Common												`json:",inline" yaml:",inline"`
 	stater		machine.Stater			`json:"-" yaml:"-"`
 	Id        string
 	To        string              `json:"to" yaml:"to"`
@@ -124,23 +131,23 @@ type NetworkRoute struct {
 	UpdateCommand *Command `yaml:"-" json:"-"`
 	DeleteCommand *Command `yaml:"-" json:"-"`
 
-	State     string              `json:"state" yaml:"state"`
+	config data.ConfigurationValueGetter
-	config ConfigurationValueGetter
+	Resources data.ResourceMapper `json:"-" yaml:"-"`
-	Resources ResourceMapper `json:"-" yaml:"-"`
 }
 
 func NewNetworkRoute() *NetworkRoute {
-	n := &NetworkRoute{Rtid: NetworkRouteTableMain}
+	n := &NetworkRoute{Rtid: NetworkRouteTableMain, Common: &Common{ resourceType: NetworkRouteTypeName } }
 	n.CreateCommand, n.ReadCommand, n.UpdateCommand, n.DeleteCommand = n.NewCRUD()
 	return n
 }
 
-func (n *NetworkRoute) SetResourceMapper(resources ResourceMapper) {
+func (n *NetworkRoute) SetResourceMapper(resources data.ResourceMapper) {
 	n.Resources = resources
 }
 
-func (n *NetworkRoute) Clone() Resource {
+func (n *NetworkRoute) Clone() data.Resource {
 	newn := &NetworkRoute {
+		Common: n.Common,
 		Id: n.Id,
 		To: n.To,
 		Interface: n.Interface,
@@ -150,7 +157,6 @@ func (n *NetworkRoute) Clone() Resource {
 		RouteType: n.RouteType,
 		Scope: n.Scope,
 		Proto: n.Proto,
-		State: n.State,
 	}
 	newn.CreateCommand, newn.ReadCommand, newn.UpdateCommand, newn.DeleteCommand = n.NewCRUD()
 	return newn
@@ -174,9 +180,9 @@ func (n *NetworkRoute) Notify(m *machine.EventMessage) {
 					return
 				}
 			}
-			n.State = "absent"
+			n.Common.State = "absent"
 		case "present":
-			n.State = "present"
+			n.Common.State = "present"
 		}
 	case machine.EXITSTATEEVENT:
 	}
@@ -192,6 +198,14 @@ func (n *NetworkRoute) Create(ctx context.Context) error {
 	return nil
 }
 
+func (n *NetworkRoute) Update(ctx context.Context) error {
+	return n.Create(ctx)
+}
+
+func (n *NetworkRoute) Delete(ctx context.Context) error {
+	return nil
+}
+
 func (n *NetworkRoute) URI() string {
 	return fmt.Sprintf("route://%s", n.Id)
 }
@@ -200,7 +214,7 @@ func (n *NetworkRoute) SetURI(uri string) error {
 	return nil
 }
 
-func (n *NetworkRoute) UseConfig(config ConfigurationValueGetter) {
+func (n *NetworkRoute) UseConfig(config data.ConfigurationValueGetter) {
 	n.config = config
 }
 
@@ -209,19 +223,30 @@ func (n *NetworkRoute) Validate() error {
 }
 
 func (n *NetworkRoute) Apply() error {
-	switch n.State {
+	switch n.Common.State {
 	case "absent":
 	case "present":
 	}
 	return nil
 }
 
-func (n *NetworkRoute) Load(r io.Reader) error {
+func (n *NetworkRoute) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(n)
+	err = f.StringDecoder(string(docData)).Decode(n)
+	return
+}
+
+func (n *NetworkRoute) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(n)
+	return
+}
+
+func (n *NetworkRoute) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(n)
+	return
 }
 
 func (n *NetworkRoute) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(n)
+	return n.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
 func (n *NetworkRoute) ResolveId(ctx context.Context) string {
@@ -495,9 +520,9 @@ func NewNetworkRouteReadCommand() *Command {
 					n.SetField(fields[i], fields[i + 1])
 				}
 			}
-			n.State = "present"
+			n.Common.State = "present"
 		} else {
-			n.State = "absent"
+			n.Common.State = "absent"
 		}
 		return nil
 	}

internal/resource/onerror.go

@@ -1,66 +0,0 @@
-// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
-
-package resource
-
-import (
-	"errors"
-	"encoding/json"
-	"gopkg.in/yaml.v3"
-)
-
-var (
-	ErrInvalidOnErrorStrategy = errors.New("Invalid OnError strategy")
-)
-
-type OnError string
-
-const (
-	OnErrorStop = "stop"
-	OnErrorFail = "fail"
-	OnErrorSkip = "skip"
-)
-
-func NewOnError() OnError {
-	return OnErrorFail
-}
-
-func (o OnError) Strategy() string {
-	switch o {
-	case OnErrorStop, OnErrorFail, OnErrorSkip:
-		return string(o)
-	}
-	return ""
-}
-
-
-func (o OnError) Validate() error {
-        switch o {
-        case OnErrorStop, OnErrorFail, OnErrorSkip:
-                return nil
-        default:
-                return ErrInvalidOnErrorStrategy
-        }
-}
-
-func (o *OnError) UnmarshalValue(value string) (err error) {
-        if err = OnError(value).Validate(); err == nil {
-                *o = OnError(value)
-        }
-        return
-}
-
-func (o *OnError) UnmarshalJSON(jsonData []byte) error {
-        var s string
-        if unmarshalErr := json.Unmarshal(jsonData, &s); unmarshalErr != nil {
-                return unmarshalErr
-        }
-        return o.UnmarshalValue(s)
-}
-
-func (o *OnError) UnmarshalYAML(value *yaml.Node) error {
-        var s string
-        if err := value.Decode(&s); err != nil {
-                return err
-        }
-        return o.UnmarshalValue(s)
-}

internal/resource/onerror_test.go

@@ -1,21 +0,0 @@
-// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
-
-package resource
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func TestOnErrorStrategies(t *testing.T) {
-	for _, v := range []struct{ strategy OnError; expected OnError; validate error }{
-		{ strategy: OnErrorFail, expected: "fail", validate: nil },
-		{ strategy: OnErrorSkip, expected: "skip", validate: nil },
-		{ strategy: OnError("unknown"), expected: "", validate: ErrInvalidOnErrorStrategy },
-	}{
-		o := v.strategy
-		assert.Equal(t, v.expected, o.Strategy())
-		assert.ErrorIs(t, o.Validate(), v.validate)
-	}
-
-}

internal/resource/package.go

@@ -18,6 +18,13 @@ import (
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
 	"decl/internal/command"
+	"decl/internal/data"
+	"decl/internal/folio"
+	"decl/internal/tempdir"
+)
+
+var (
+	PackageTempDir tempdir.Path = "jx_package_resource"
 )
 
 type PackageType string
@@ -32,8 +39,11 @@ const (
 	PackageTypeYum PackageType = "yum"
 )
 
-var ErrUnsupportedPackageType error = errors.New("The PackageType is not supported on this system")
+var (
-var ErrInvalidPackageType error = errors.New("invalid PackageType value")
+	ErrUnsupportedPackageType error = errors.New("The PackageType is not supported on this system")
+	ErrInvalidPackageType error = errors.New("invalid PackageType value")
+	ErrRpmPackageInstalled error = errors.New("is already installed")
+)
 
 var SystemPackageType PackageType = FindSystemPackageType()
 
@@ -44,6 +54,7 @@ type Package struct {
 	Required    string	`json:"required,omitempty" yaml:"required,omitempty"`
 	Version     string	`yaml:"version,omitempty" json:"version,omitempty"`
 	PackageType PackageType	`yaml:"type" json:"type"`
+	SourceRef   folio.ResourceReference `yaml:"sourceref,omitempty" json:"sourceref,omitempty"`
 
 	CreateCommand *command.Command `yaml:"-" json:"-"`
 	ReadCommand   *command.Command `yaml:"-" json:"-"`
@@ -51,13 +62,15 @@ type Package struct {
 	DeleteCommand *command.Command `yaml:"-" json:"-"`
 	// state attributes
 	State string `yaml:"state,omitempty" json:"state,omitempty"`
-	config ConfigurationValueGetter
+	config data.ConfigurationValueGetter
-	Resources ResourceMapper `yaml:"-" json:"-"`
+	Resources data.ResourceMapper `yaml:"-" json:"-"`
 }
 
 func init() {
-	ResourceTypes.Register([]string{"package", string(PackageTypeApk), string(PackageTypeApt), string(PackageTypeDeb), string(PackageTypeDnf), string(PackageTypeRpm), string(PackageTypePip), string(PackageTypeYum)}, func(u *url.URL) Resource {
+	folio.DocumentRegistry.ResourceTypes.Register([]string{"package", string(PackageTypeApk), string(PackageTypeApt), string(PackageTypeDeb), string(PackageTypeDnf), string(PackageTypeRpm), string(PackageTypePip), string(PackageTypeYum)}, func(u *url.URL) data.Resource {
 		p := NewPackage()
+		e := p.SetParsedURI(u)
+		slog.Info("PackageFactory SetParsedURI()", "error", e)
 		return p
 	})
 }
@@ -76,11 +89,11 @@ func NewPackage() *Package {
 	return &Package{ PackageType: SystemPackageType }
 }
 
-func (p *Package) SetResourceMapper(resources ResourceMapper) {
+func (p *Package) SetResourceMapper(resources data.ResourceMapper) {
 	p.Resources = resources
 }
 
-func (p *Package) Clone() Resource {
+func (p *Package) Clone() data.Resource {
 	newp := &Package {
 		Name: p.Name,
 		Required: p.Required,
@@ -124,8 +137,10 @@ func (p *Package) Notify(m *machine.EventMessage) {
 				if triggerErr := p.StateMachine().Trigger("created"); triggerErr == nil {
 					return
 				}
+			} else {
+				panic(e)
 			}
-			p.State = "absent"
+//			p.State = "absent"
 		case "start_update":
 			if e := p.Update(ctx); e == nil {
 				if triggerErr := p.StateMachine().Trigger("updated"); triggerErr == nil {
@@ -154,30 +169,37 @@ func (p *Package) Notify(m *machine.EventMessage) {
 }
 
 func (p *Package) URI() string {
-	return fmt.Sprintf("package://%s?version=%s&type=%s", p.Name, p.Version, p.PackageType)
+	return fmt.Sprintf("package://%s?version=%s&type=%s", p.Name, url.QueryEscape(p.Version), p.PackageType)
 }
 
+
 func (p *Package) SetURI(uri string) error {
 	resourceUri, e := url.Parse(uri)
 	if e == nil {
-		if resourceUri.Scheme == "package" {
+		e = p.SetParsedURI(resourceUri)
-			p.Name = filepath.Join(resourceUri.Hostname(), resourceUri.Path)
-			p.Version = resourceUri.Query().Get("version")
-			if p.Version == "" {
-				p.Version = "latest"
-			}
-			p.PackageType = PackageType(resourceUri.Query().Get("type"))
-			if p.PackageType == "" {
-				e = fmt.Errorf("%w: %s is not a package known resource ", ErrInvalidResourceURI, uri)
-			}
-		} else {
-			e = fmt.Errorf("%w: %s is not a package resource ", ErrInvalidResourceURI, uri)
-		}
 	}
 	return e
 }
 
-func (p *Package) UseConfig(config ConfigurationValueGetter) {
+func (p *Package) SetParsedURI(uri *url.URL) (err error) {
+	if uri.Scheme == "package" {
+		p.Name = filepath.Join(uri.Hostname(), uri.Path)
+		p.Version = uri.Query().Get("version")
+		if p.Version == "" {
+			p.Version = "latest"
+		}
+		indicatedPackageType := PackageType(uri.Query().Get("type"))
+		if indicatedPackageType.Validate() != nil {
+			p.PackageType = SystemPackageType
+		}
+	} else {
+		err = fmt.Errorf("%w: %s is not a package resource ", ErrInvalidResourceURI, uri.String())
+	}
+	p.CreateCommand, p.ReadCommand, p.UpdateCommand, p.DeleteCommand = p.PackageType.NewCRUD()
+	return
+}
+
+func (p *Package) UseConfig(config data.ConfigurationValueGetter) {
 	p.config = config
 }
 
@@ -237,16 +259,38 @@ func (p *Package) ResolveId(ctx context.Context) string {
 	return p.Name
 }
 
-func (p *Package) Create(ctx context.Context) error {
+func (p *Package) Create(ctx context.Context) (err error) {
 	if p.Version == "latest" {
 		p.Version = ""
 	}
-	_, err := p.CreateCommand.Execute(p)
+
-	if err != nil {
+
-		return err
+	if source := p.SourceRef.Lookup(p.Resources); source != nil {
+		r, _ := source.ContentReaderStream()
+		if p.CreateCommand.StdinAvailable {
+			p.CreateCommand.SetStdinReader(r)
+		} else {
+			if err = PackageTempDir.Create(); err != nil {
+				return
 			}
-	_,e := p.Read(ctx)
+			defer PackageTempDir.Remove()
-	return e
+			defer func() { p.Source = "" }()
+			if p.Source, err = PackageTempDir.CreateFileFromReader(fmt.Sprintf("%s.rpm", p.Name), r); err != nil {
+				return
+			}
+		}
+	}
+
+	if _, err = p.CreateCommand.Execute(p); err != nil {
+		msg := err.Error()
+		lenMsg := len(msg) - 1
+		lenErr := len(ErrRpmPackageInstalled.Error())
+		if msg[lenMsg - lenErr:lenMsg] != ErrRpmPackageInstalled.Error() {
+			return
+		}
+	}
+	_, err = p.Read(ctx)
+	return
 }
 
 func (p *Package) Update(ctx context.Context) error {
@@ -263,25 +307,35 @@ func (p *Package) Delete(ctx context.Context) error {
 }
 
 
-func (p *Package) Apply() error {
+func (p *Package) Apply() (err error) {
 	if p.Version == "latest" {
 		p.Version = ""
 	}
-	_, err := p.CreateCommand.Execute(p)
+	if _, err = p.CreateCommand.Execute(p); err != nil {
-	if err != nil {
+		return
-		return err
 	}
 
-	_,e := p.Read(context.Background())
+	_, err = p.Read(context.Background())
-	return e
+	return
 }
 
-func (p *Package) Load(r io.Reader) error {
+func (p *Package) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(p)
+	err = f.StringDecoder(string(docData)).Decode(p)
+	return
+}
+
+func (p *Package) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(p)
+	return
+}
+
+func (p *Package) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(p)
+	return
 }
 
 func (p *Package) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(p)
+	return p.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
 func (p *Package) Type() string { return "package" }
@@ -295,6 +349,7 @@ func (p *Package) Read(ctx context.Context) (resourceYaml []byte, err error) {
 		} else {
 			err = fmt.Errorf("%w - %w", ErrResourceStateAbsent, err)
 		}
+		slog.Info("Package.Read()", "package", p, "error", err)
 	} else {
 		err = ErrUnsupportedPackageType
 	}
@@ -374,28 +429,34 @@ func (p *PackageType) NewReadPackagesCommand() (read *command.Command) {
 	case PackageTypeDeb:
 		return NewDebReadPackagesCommand()
 	case PackageTypeDnf:
-//		return NewDnfReadPackagesCommand()
+		return NewDnfReadPackagesCommand()
 	case PackageTypeRpm:
-//		return NewRpmReadPackagesCommand()
+		return NewRpmReadPackagesCommand()
 	case PackageTypePip:
-//		return NewPipReadPackagesCommand()
+		return NewPipReadPackagesCommand()
 	case PackageTypeYum:
-//		return NewYumReadPackagesCommand()
+		return NewYumReadPackagesCommand()
 	default:
 	}
 	return nil
 }
 
-func (p *PackageType) UnmarshalValue(value string) error {
+func (p PackageType) Validate() error {
-	switch value {
+	switch p {
-	case string(PackageTypeApk), string(PackageTypeApt), string(PackageTypeDeb), string(PackageTypeDnf), string(PackageTypeRpm), string(PackageTypePip), string(PackageTypeYum):
+	case PackageTypeApk, PackageTypeApt, PackageTypeDeb, PackageTypeDnf, PackageTypeRpm, PackageTypePip, PackageTypeYum:
-		*p = PackageType(value)
 		return nil
 	default:
 		return ErrInvalidPackageType
 	}
 }
 
+func (p *PackageType) UnmarshalValue(value string) (err error) {
+	if err = PackageType(value).Validate(); err == nil {
+		*p = PackageType(value)
+	}
+	return
+}
+
 func (p *PackageType) UnmarshalJSON(data []byte) error {
 	var s string
 	if unmarshalRouteTypeErr := json.Unmarshal(data, &s); unmarshalRouteTypeErr != nil {
@@ -412,6 +473,10 @@ func (p *PackageType) UnmarshalYAML(value *yaml.Node) error {
 	return p.UnmarshalValue(s)
 }
 
+func (p PackageType) Exists() bool {
+	return p.NewReadCommand().Exists()
+}
+
 func NewApkCreateCommand() *command.Command {
 	c := command.NewCommand()
 	c.Path = "apk"
@@ -745,7 +810,7 @@ func NewDnfCreateCommand() *command.Command {
 		command.CommandArg("install"),
 		command.CommandArg("-q"),
 		command.CommandArg("-y"),
-		command.CommandArg("{{ .Name }}{{ if .Required }}{{ .Required }}{{ else }}>=0.0.0{{ end }}"),
+		command.CommandArg("{{ .Name }}{{ if .Required }}{{ .Required }}{{ else }} >= 0.0.0{{ end }}"),
 	}
 	return c
 }
@@ -763,20 +828,28 @@ func NewDnfReadCommand() *command.Command {
 		p := target.(*Package)
 		slog.Info("Extract()", "out", out)
 		pkginfo := strings.Split(string(out), "\n")
-		for _, packageLines := range pkginfo {
+		for _, packageLines := range pkginfo[1:] {
+			if len(packageLines) > 0 {
 				fields := strings.Fields(packageLines)
+				slog.Info("DnfReadCommaond.Extract()", "fields", fields, "package", p)
+
 				packageNameField := strings.Split(fields[0], ".")
-			packageName := strings.TrimSpace(packageNameField[0])
+				lenName := len(packageNameField)
-			//packageArch := strings.TrimSpace(packageNameField[1])
+				packageName := strings.TrimSpace(strings.Join(packageNameField[0:lenName - 1], "."))
 			
 				if packageName == p.Name {
 					p.State = "present"
 					packageVersionField := strings.Split(fields[1], ":")
+					if len(packageVersionField) > 1 {
 						//packageEpoch := strings.TrimSpace(packageVersionField[0])
-				packageVersion := strings.TrimSpace(packageVersionField[1])
+						p.Version = strings.TrimSpace(packageVersionField[1])
-				p.Version = packageVersion
+					} else {
+						p.Version = strings.TrimSpace(packageVersionField[0])
+					}
 					return nil
 				}
+				slog.Info("DnfReadCommaond.Extract()", "package", packageName, "package", p)
+			}
 		}
 		p.State = "absent"
 		slog.Info("Extract()", "package", p)
@@ -809,11 +882,51 @@ func NewDnfDeleteCommand() *command.Command {
 	return c
 }
 
+func NewDnfReadPackagesCommand() *command.Command {
+	c := command.NewCommand()
+	c.Path = "dnf"
+	c.FailOnError = false
+	c.Split = false
+	c.Args = []command.CommandArg{
+		command.CommandArg("-q"),
+		command.CommandArg("list"),
+		command.CommandArg("installed"),
+	}
+	c.Extractor = func(out []byte, target any) error {
+		Packages := target.(*[]*Package)
+		lines := strings.Split(strings.TrimSpace(string(out)), "\n")
+		lineIndex := 0
+		for _, line := range lines[1:] {
+			installedPackage := strings.Fields(strings.TrimSpace(line))
+			if len(*Packages) <= lineIndex + 1 {
+				*Packages = append(*Packages, NewPackage())
+			}
+			p := (*Packages)[lineIndex]
+			packageNameFields := strings.Split(installedPackage[0], ".")
+			packageName := packageNameFields[0]
+			//packageArch := packageNameFields[1]
+			packageVersionFields := strings.Split(installedPackage[1], ":")
+			if len(packageVersionFields) > 1 {
+				p.Version = packageVersionFields[1]
+			} else {
+				p.Version = packageVersionFields[0]
+			}
+			p.Name = packageName
+			p.State = "present"
+			p.PackageType = PackageTypeDnf
+			lineIndex++
+		}
+		return nil
+	}
+	return c
+}
+
 
 func NewRpmCreateCommand() *command.Command {
 	c := command.NewCommand()
 	c.Path = "rpm"
 	c.Split = false
+	c.StdinAvailable = false
 	c.Args = []command.CommandArg{
 		command.CommandArg("-i"),
 		command.CommandArg("{{ .Source }}"),
@@ -833,11 +946,13 @@ func NewRpmReadCommand() *command.Command {
 		slog.Info("Extract()", "out", out)
 		pkginfo := strings.Split(string(out), "\n")
 		for _, packageLine := range pkginfo {
+			// package-name-ver-rel.arch
 			packageFields := strings.Split(packageLine, "-")
 			numberOfFields := len(packageFields)
 			if numberOfFields > 2 {
-				packageName := strings.Join(packageFields[:numberOfFields - 3], "-")
+				packageName := strings.Join(packageFields[:numberOfFields - 2], "-")
 				packageVersion := strings.Join(packageFields[numberOfFields - 2:numberOfFields - 1], "-")
+				slog.Info("Package[RPM].Extract()", "name", packageName, "version", packageVersion, "package", p)
 				if packageName == p.Name {
 					p.State = "present"
 					p.Version = packageVersion
@@ -872,6 +987,10 @@ func NewRpmDeleteCommand() *command.Command {
 	return c
 }
 
+func NewRpmReadPackagesCommand() *command.Command {
+	return nil
+}
+
 func NewPipCreateCommand() *command.Command {
 	c := command.NewCommand()
 	c.Path = "pip"
@@ -932,6 +1051,10 @@ func NewPipDeleteCommand() *command.Command {
 	return c
 }
 
+func NewPipReadPackagesCommand() *command.Command {
+	return nil
+}
+
 func NewYumCreateCommand() *command.Command {
 	c := command.NewCommand()
 	c.Path = "yum"
@@ -1003,3 +1126,7 @@ func NewYumDeleteCommand() *command.Command {
 	}
 	return c
 }
+
+func NewYumReadPackagesCommand() *command.Command {
+	return nil
+}

internal/resource/package_test.go

@@ -69,6 +69,9 @@ type: apk
 }
 
 func TestReadAptPackage(t *testing.T) {
+	if ! PackageTypeApt.Exists() {
+		t.Skip("Unsupported package type")
+	}
 	decl := `
 name: vim
 required: ">1.1.1"
@@ -102,15 +105,16 @@ state: absent
 type: %s 
 `, SystemPackageType)
 
-	decl := `
+	decl := fmt.Sprintf(`
 name: missing
-type: apt
+type: %s
-`
+`, SystemPackageType)
+
 	p := NewPackage()
 	assert.NotNil(t, p)
 	loadErr := p.LoadDecl(decl)
 	assert.Nil(t, loadErr)
-	p.ReadCommand = NewAptReadCommand()
+	p.ReadCommand = SystemPackageType.NewReadCommand()
 /*
 	p.ReadCommand.Executor = func(value any) ([]byte, error) {
 		return []byte(``), fmt.Errorf("exit status 1 dpkg-query: package 'makef' is not installed and no information is available\nUse dpkg --info (= dpkg-deb --info) to examine archive files.\n")
@@ -138,6 +142,10 @@ func TestPackageSetURI(t *testing.T) {
 }
 
 func TestReadDebPackage(t *testing.T) {
+	if ! PackageTypeDeb.Exists() {
+		t.Skip("Unsupported package type")
+	}
+
 	decl := `
 name: vim
 source: vim-8.2.3995-1ubuntu2.17.deb
@@ -187,3 +195,27 @@ Version: 1.2.2
 	assert.ErrorIs(t, readErr, ErrUnsupportedPackageType)
 
 }
+
+func TestPackageSourceRef(t *testing.T) {
+	decl := `
+name: vim
+sourceref: https://localhost/vim-8.2.3995-1ubuntu2.17.deb
+type: deb
+`
+
+	p := NewPackage()
+	assert.NotNil(t, p)
+	loadErr := p.LoadDecl(decl)
+	assert.Nil(t, loadErr)
+
+	p.ReadCommand = NewDebReadCommand()
+	p.ReadCommand.CommandExists = func() error { return command.ErrUnknownCommand }
+	p.ReadCommand.Executor = func(value any) ([]byte, error) {
+		return []byte(`
+Package: vim
+Version: 1.2.2
+`), nil
+	}
+	_, readErr := p.Read(context.Background())
+	assert.ErrorIs(t, readErr, ErrUnsupportedPackageType)
+}

internal/resource/pki.go

@@ -14,6 +14,8 @@ import (
 	"decl/internal/codec"
 	"decl/internal/ext"
 	"decl/internal/transport"
+	"decl/internal/data"
+	"decl/internal/folio"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"crypto/rsa"
@@ -26,6 +28,10 @@ import (
 	"strings"
 )
 
+const (
+	PKITypeName TypeName = "pki"
+)
+
 // Describes the type of certificate file the resource represents
 type EncodingType string
 
@@ -38,9 +44,9 @@ var ErrPKIInvalidEncodingType error = errors.New("Invalid EncodingType")
 var ErrPKIFailedDecodingPemBlock error = errors.New("Failed decoding pem block")
 
 func init() {
-	ResourceTypes.Register([]string{"pki"}, func(u *url.URL) Resource {
+	ResourceTypes.Register([]string{"pki"}, func(u *url.URL) data.Resource {
 		k := NewPKI()
-		ref := ResourceReference(filepath.Join(u.Hostname(), u.Path))
+		ref := folio.ResourceReference(filepath.Join(u.Hostname(), u.Path))
 		if len(ref) > 0 {
 			k.PrivateKeyRef = ref
 		}
@@ -58,16 +64,17 @@ type Certificate struct {
 }
 
 type PKI struct {
+	*Common `json:",inline" yaml:",inline"`
 	stater machine.Stater `json:"-" yaml:"-"`
 	PrivateKeyPem string `json:"privatekey,omitempty" yaml:"privatekey,omitempty"`
 	PublicKeyPem string `json:"publickey,omitempty" yaml:"publickey,omitempty"`
 	CertificatePem string `json:"certificate,omitempty" yaml:"certificate,omitempty"`
 
-	PrivateKeyRef ResourceReference `json:"privatekeyref,omitempty" yaml:"privatekeyref,omitempty"` // Describes a resource URI to read/write the private key content
+	PrivateKeyRef folio.ResourceReference `json:"privatekeyref,omitempty" yaml:"privatekeyref,omitempty"` // Describes a resource URI to read/write the private key content
-	PublicKeyRef ResourceReference `json:"publickeyref,omitempty" yaml:"publickeyref,omitempty"`  // Describes a resource URI to read/write the public key content 
+	PublicKeyRef folio.ResourceReference `json:"publickeyref,omitempty" yaml:"publickeyref,omitempty"`  // Describes a resource URI to read/write the public key content 
-	CertificateRef ResourceReference `json:"certificateref,omitempty" yaml:"certificateref,omitempty"` // Describes a resource URI to read/write the certificate content
+	CertificateRef folio.ResourceReference `json:"certificateref,omitempty" yaml:"certificateref,omitempty"` // Describes a resource URI to read/write the certificate content
 
-	SignedByRef ResourceReference `json:"signedbyref,omitempty" yaml:"signedbyref,omitempty"` // Describes a resource URI for the signing cert
+	SignedByRef folio.ResourceReference `json:"signedbyref,omitempty" yaml:"signedbyref,omitempty"` // Describes a resource URI for the signing cert
 
 	privateKey *rsa.PrivateKey `json:"-" yaml:"-"`
 	publicKey *rsa.PublicKey `json:"-" yaml:"-"`
@@ -77,22 +84,23 @@ type PKI struct {
 	Bits int `json:"bits" yaml:"bits"`
 	EncodingType EncodingType `json:"type" yaml:"type"`
 	//State    string   `json:"state,omitempty" yaml:"state,omitempty"`
-	config ConfigurationValueGetter
+	config data.ConfigurationValueGetter
-	Resources ResourceMapper `json:"-" yaml:"-"`
+	Resources data.ResourceMapper `json:"-" yaml:"-"`
 }
 
 func NewPKI() *PKI {
-	p := &PKI{ EncodingType: EncodingTypePem, Bits: 2048 }
+	p := &PKI{ EncodingType: EncodingTypePem, Bits: 2048, Common: &Common{ resourceType: PKITypeName } }
 	return p
 }
 
-func (k *PKI) SetResourceMapper(resources ResourceMapper) {
+func (k *PKI) SetResourceMapper(resources data.ResourceMapper) {
 	slog.Info("PKI.SetResourceMapper()", "resources", resources)
 	k.Resources = resources
 }
 
-func (k *PKI) Clone() Resource {
+func (k *PKI) Clone() data.Resource {
 	return &PKI {
+		Common: k.Common,
 		EncodingType: k.EncodingType,
 		//State: k.State,
 	}
@@ -171,7 +179,7 @@ func (k *PKI) SetURI(uri string) error {
 	resourceUri, e := url.Parse(uri)
 	if e == nil {
 		if resourceUri.Scheme == "pki" {
-			k.PrivateKeyRef = ResourceReference(fmt.Sprintf("pki://%s", filepath.Join(resourceUri.Hostname(), resourceUri.Path)))
+			k.PrivateKeyRef = folio.ResourceReference(fmt.Sprintf("pki://%s", filepath.Join(resourceUri.Hostname(), resourceUri.Path)))
 		} else {
 			e = fmt.Errorf("%w: %s is not a cert", ErrInvalidResourceURI, uri)
 		}
@@ -179,7 +187,7 @@ func (k *PKI) SetURI(uri string) error {
 	return e
 }
 
-func (k *PKI) UseConfig(config ConfigurationValueGetter) {
+func (k *PKI) UseConfig(config data.ConfigurationValueGetter) {
 	k.config = config
 }
 
@@ -201,12 +209,25 @@ func (k *PKI) Apply() error {
 	return nil
 }
 
-func (k *PKI) LoadDecl(yamlResourceDeclaration string) (err error) {
+func (k *PKI) Load(docData []byte, f codec.Format) (err error) {
-	d := codec.NewYAMLStringDecoder(yamlResourceDeclaration)
+	err = f.StringDecoder(string(docData)).Decode(k)
-	err = d.Decode(k)
 	return
 }
 
+func (k *PKI) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(k)
+	return
+}
+
+func (k *PKI) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(k)
+	return
+}
+
+func (k *PKI) LoadDecl(yamlResourceDeclaration string) error {
+	return k.LoadString(yamlResourceDeclaration, codec.FormatYaml)
+}
+
 func (k *PKI) ResolveId(ctx context.Context) string {
 	return string(k.PrivateKeyRef)
 }

internal/resource/pki_test.go

@@ -8,23 +8,34 @@ _ "encoding/json"
 	"fmt"
 	"github.com/stretchr/testify/assert"
 _	"gopkg.in/yaml.v3"
-_	"io"
+	"io"
 _ "log"
 _	"os"
 	"decl/internal/transport"
 	"decl/internal/ext"
 	"decl/internal/codec"
+	"decl/internal/data"
+	"decl/internal/folio"
 	"strings"
 	"testing"
 	"path/filepath"
+	"net/url"
 )
 
-type TestResourceMapper func(key string) (any, bool)
+type TestResourceMapper func(key string) (data.Declaration, bool)
 
-func (rm TestResourceMapper) Get(key string) (any, bool) {
+func (rm TestResourceMapper) Get(key string) (data.Declaration, bool) {
 	return rm(key)
 }
 
+func (rm TestResourceMapper) Has(key string) (ok bool) {
+	_, ok = rm(key)
+	return
+}
+
+func (rm TestResourceMapper) Set(key string, value data.Declaration) {
+}
+
 type StringContentReadWriter func() (any, error)
 
 func (s StringContentReadWriter) ContentWriterStream() (*transport.Writer, error) {
@@ -37,6 +48,21 @@ func (s StringContentReadWriter) ContentReaderStream() (*transport.Reader, error
 	return r.(*transport.Reader), e
 }
 
+func (s StringContentReadWriter) Apply() error { return nil }
+func (s StringContentReadWriter) Clone() data.Declaration { return nil }
+func (s StringContentReadWriter) Load(docData []byte, f codec.Format) (err error) { return }
+func (s StringContentReadWriter) LoadReader(r io.ReadCloser, f codec.Format) (err error) { return }
+func (s StringContentReadWriter) LoadString(docData string, f codec.Format) (err error) { return }
+func (s StringContentReadWriter) LoadDecl(yamlResourceDeclaration string) error { return nil }
+func (s StringContentReadWriter) ResolveId(ctx context.Context) (string) { return "" }
+func (s StringContentReadWriter) SetURI(uri string) (error) { return nil }
+func (s StringContentReadWriter) SetParsedURI(uri *url.URL) (error) { return nil }
+func (s StringContentReadWriter) URI() (string) { return "" }
+func (s StringContentReadWriter) Validate() (error) { return nil }
+func (s StringContentReadWriter) ResourceType() data.TypeName { return "" }
+func (s StringContentReadWriter) Resource() data.Resource { return nil }
+
+
 func TestNewPKIKeysResource(t *testing.T) {
 	r := NewPKI()
 	assert.NotNil(t, r)
@@ -62,7 +88,7 @@ func TestPKIEncodeKeys(t *testing.T) {
 	r.PublicKey()
 	assert.NotNil(t, r.publicKey)
 
-	r.Resources = TestResourceMapper(func(key string) (any, bool) {
+	r.Resources = TestResourceMapper(func(key string) (data.Declaration, bool) {
 		switch key {
 		case "buffer://privatekey":
 			return StringContentReadWriter(func() (any, error) {
@@ -86,14 +112,14 @@ func TestPKIEncodeKeys(t *testing.T) {
 		return nil, false
 	})
 
-	r.PrivateKeyRef = ResourceReference("buffer://privatekey")
+	r.PrivateKeyRef = folio.ResourceReference("buffer://privatekey")
-	r.PublicKeyRef = ResourceReference("buffer://publickey")
+	r.PublicKeyRef = folio.ResourceReference("buffer://publickey")
 
 	r.Encode()
 	assert.Equal(t, "-----BEGIN RSA PRIVATE KEY-----", strings.Split(privateTarget.String(), "\n")[0])
 	assert.Equal(t, "-----BEGIN RSA PUBLIC KEY-----", strings.Split(publicTarget.String(), "\n")[0])
 
-	r.CertificateRef = ResourceReference("buffer://certificate")
+	r.CertificateRef = folio.ResourceReference("buffer://certificate")
 
 	e := r.GenerateCertificate()
 	assert.Nil(t, e)
@@ -121,9 +147,9 @@ type: pem
 	assert.NotNil(t, r.privateKey)
 	r.PublicKey()
 	assert.NotNil(t, r.publicKey)
-	r.PrivateKeyRef = ResourceReference(fmt.Sprintf("file://%s", privateKeyFile))
+	r.PrivateKeyRef = folio.ResourceReference(fmt.Sprintf("file://%s", privateKeyFile))
-	r.PublicKeyRef = ResourceReference(fmt.Sprintf("file://%s", publicKeyFile))
+	r.PublicKeyRef = folio.ResourceReference(fmt.Sprintf("file://%s", publicKeyFile))
-	r.CertificateRef = ResourceReference(fmt.Sprintf("file://%s", certFile))
+	r.CertificateRef = folio.ResourceReference(fmt.Sprintf("file://%s", certFile))
 	createErr := r.Create(context.Background())
 	assert.Nil(t, createErr)
 	assert.FileExists(t, privateKeyFile)
@@ -136,9 +162,9 @@ type: pem
 
 	read := NewPKI()
 	assert.NotNil(t, read)
-	read.PrivateKeyRef = ResourceReference(fmt.Sprintf("file://%s", privateKeyFile))
+	read.PrivateKeyRef = folio.ResourceReference(fmt.Sprintf("file://%s", privateKeyFile))
-	read.PublicKeyRef = ResourceReference(fmt.Sprintf("file://%s", publicKeyFile))
+	read.PublicKeyRef = folio.ResourceReference(fmt.Sprintf("file://%s", publicKeyFile))
-	read.CertificateRef = ResourceReference(fmt.Sprintf("file://%s", certFile))
+	read.CertificateRef = folio.ResourceReference(fmt.Sprintf("file://%s", certFile))
 	
 	_, readErr := read.Read(context.Background())
 	assert.Nil(t, readErr)

internal/resource/resource.go

@@ -4,14 +4,11 @@
 package resource
 
 import (
-	"context"
 	_ "encoding/json"
 	_ "fmt"
 	_ "gopkg.in/yaml.v3"
-	"net/url"
 	"gitea.rosskeen.house/rosskeen.house/machine"
-	"decl/internal/transport"
+	"decl/internal/data"
-	"log/slog"
 	"errors"
 )
 
@@ -19,10 +16,11 @@ import (
 var (
 	ErrInvalidResourceURI error = errors.New("Invalid resource URI")
 	ErrResourceStateAbsent = errors.New("Resource state absent")
+	ErrUnableToFindResource = errors.New("Unable to find resource - not loaded")
 )
 
 type ResourceReference string
-
+/*
 type ResourceSelector func(r *Declaration) bool
 
 type Resource interface {
@@ -82,8 +80,9 @@ type ResourceCrudder struct {
 	ResourceUpdater
 	ResourceDeleter
 }
+*/
 
-func NewResource(uri string) Resource {
+func NewResource(uri string) data.Resource {
 	r, e := ResourceTypes.New(uri)
 	if e == nil {
 		return r
@@ -91,18 +90,20 @@ func NewResource(uri string) Resource {
 	return nil
 }
 
+/*
+
 // Return a Content ReadWriter for the resource referred to.
-func (r ResourceReference) Lookup(look ResourceMapper) ContentReadWriter {
+func (r ResourceReference) Lookup(look data.ResourceMapper) data.ContentReadWriter {
 	slog.Info("ResourceReference.Lookup()", "resourcereference", r, "resourcemapper", look)
 	if look != nil {
 		if v,ok := look.Get(string(r)); ok {
-			return v.(ContentReadWriter)
+			return v.(data.ContentReadWriter)
 		}
 	}
 	return r
 }
 
-func (r ResourceReference) Dereference(look ResourceMapper) Resource {
+func (r ResourceReference) Dereference(look data.ResourceMapper) data.Resource {
 	slog.Info("ResourceReference.Dereference()", "resourcereference", r, "resourcemapper", look)
 	if look != nil {
 		if v,ok := look.Get(string(r)); ok {
@@ -131,6 +132,7 @@ func (r ResourceReference) ContentReaderStream() (*transport.Reader, error) {
 func (r ResourceReference) ContentWriterStream() (*transport.Writer, error) {
 	return transport.NewWriterURI(string(r))
 }
+*/
 
 func StorageMachine(sub machine.Subscriber) machine.Stater {
 // start_destroy -> absent -> start_create -> present -> start_destroy

internal/resource/resource_test.go

@@ -26,6 +26,8 @@ func TestMain(m *testing.M) {
 		log.Fatal(err)
 	}
 
+	RegisterConverterMocks()
+
 	ProcessTestUserName, ProcessTestGroupName = ProcessUserName()
 	rc := m.Run()
 

internal/resource/schema.go

@@ -11,11 +11,21 @@ import (
 	"embed"
 	"net/http"
 	"log/slog"
+	"decl/internal/folio"
 )
 
+//go:embed schemas/config/*.schema.json
 //go:embed schemas/*.schema.json
 var schemaFiles embed.FS
 
+var schemaFilesUri folio.URI = "file://resource/schemas/*.schema.json"
+
+
+func init() {
+	folio.DocumentRegistry.Schemas[schemaFilesUri] = schemaFiles
+	folio.DocumentRegistry.DefaultSchema = schemaFilesUri
+}
+
 type Schema struct {
 	schema gojsonschema.JSONLoader
 }

internal/resource/schemas/codec.schema.json

@@ -0,0 +1,8 @@
+{
+  "$id": "codec.schema.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "codec",
+  "type": "string",
+  "description": "Supported serialization encode/decode formats",
+  "enum": [ "yaml", "json", "protobuf" ]
+}

internal/resource/schemas/document.schema.json

@@ -3,8 +3,18 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "document",
   "type": "object",
-  "required": [ "resources" ],
+  "required": [
+  ],
   "properties": {
+    "configurations": {
+      "type": "array",
+      "description": "Configurations list",
+      "items": {
+        "oneOf": [
+          { "$ref": "config/block.schema.json" }
+        ]
+      }
+    },
     "resources": {
       "type": "array",
       "description": "Resources list",
@@ -28,4 +38,3 @@
     }
   }
 }
-

internal/resource/service.go

@@ -13,10 +13,15 @@ _	"log/slog"
 	"gopkg.in/yaml.v3"
 	"gitea.rosskeen.house/rosskeen.house/machine"
 	"decl/internal/codec"
+	"decl/internal/data"
 	"encoding/json"
 	"strings"
 )
 
+const (
+	ServiceTypeName TypeName = "service"
+)
+
 type ServiceManagerType string
 
 const (
@@ -25,6 +30,7 @@ const (
 )
 
 type Service struct {
+	*Common `yaml:",inline" json:",inline"`
 	stater	machine.Stater `yaml:"-" json:"-"`
 	Name	string	`json:"name" yaml:"name"`
 	ServiceManagerType ServiceManagerType `json:"servicemanager,omitempty" yaml:"servicemanager,omitempty"`
@@ -34,13 +40,12 @@ type Service struct {
 	UpdateCommand *Command `yaml:"-" json:"-"`
 	DeleteCommand *Command `yaml:"-" json:"-"`
 
-	State string `yaml:"state,omitempty" json:"state,omitempty"`
+	config data.ConfigurationValueGetter
-	config ConfigurationValueGetter
+	Resources data.ResourceMapper `yaml:"-" json:"-"`
-	Resources ResourceMapper `yaml:"-" json:"-"`
 }
 
 func init() {
-	ResourceTypes.Register([]string{"service"}, func(u *url.URL) Resource {
+	ResourceTypes.Register([]string{"service"}, func(u *url.URL) data.Resource {
 		s := NewService()
 		s.Name = filepath.Join(u.Hostname(), u.Path)
 		s.CreateCommand, s.ReadCommand, s.UpdateCommand, s.DeleteCommand = s.ServiceManagerType.NewCRUD()
@@ -49,7 +54,7 @@ func init() {
 }
 
 func NewService() *Service {
-	return &Service{ ServiceManagerType: ServiceManagerTypeSystemd }
+	return &Service{ ServiceManagerType: ServiceManagerTypeSystemd, Common: &Common{ resourceType: ServiceTypeName } }
 }
 
 func (s *Service) StateMachine() machine.Stater {
@@ -70,11 +75,11 @@ func (s *Service) Notify(m *machine.EventMessage) {
 					return
 				}
 			}
-			s.State = "absent"
+			s.Common.State = "absent"
 		case "created":
-			s.State = "present"
+			s.Common.State = "present"
 		case "running":
-			s.State = "running"
+			s.Common.State = "running"
 		}
 	case machine.EXITSTATEEVENT:
 	}
@@ -96,7 +101,7 @@ func (s *Service) SetURI(uri string) error {
 	return e
 }
 
-func (s *Service) UseConfig(config ConfigurationValueGetter) {
+func (s *Service) UseConfig(config data.ConfigurationValueGetter) {
 	s.config = config
 }
 
@@ -108,12 +113,13 @@ func (s *Service) Validate() error {
 	return nil
 }
 
-func (s *Service) SetResourceMapper(resources ResourceMapper) {
+func (s *Service) SetResourceMapper(resources data.ResourceMapper) {
 	s.Resources = resources
 }
 
-func (s *Service) Clone() Resource {
+func (s *Service) Clone() data.Resource {
 	news := &Service{
+		Common: s.Common,
 		Name: s.Name,
 		ServiceManagerType: s.ServiceManagerType,
 	}
@@ -125,12 +131,23 @@ func (s *Service) Apply() error {
 	return nil
 }
 
-func (s *Service) Load(r io.Reader) error {
+func (s *Service) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(s)
+	err = f.StringDecoder(string(docData)).Decode(s)
+	return
+}
+
+func (s *Service) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(s)
+	return
+}
+
+func (s *Service) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(s)
+	return
 }
 
 func (s *Service) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(s)
+	return s.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
 func (s *Service) UnmarshalJSON(data []byte) error {
@@ -171,6 +188,10 @@ func (s *Service) Read(ctx context.Context) ([]byte, error) {
 	return yaml.Marshal(s)
 }
 
+func (s *Service) Update(ctx context.Context) error {
+	return nil
+}
+
 func (s *Service) Delete(ctx context.Context) error {
 	return nil
 }

internal/resource/types.go

@@ -8,11 +8,13 @@ import (
 _	"net/url"
 	"strings"
 	"decl/internal/types"
+	"decl/internal/data"
+	"decl/internal/folio"
 )
 
 var (
 	ErrUnknownResourceType        = errors.New("Unknown resource type")
-	ResourceTypes          *types.Types[Resource] = types.New[Resource]()
+	ResourceTypes          *types.Types[data.Resource] = folio.DocumentRegistry.ResourceTypes
 )
 
 type TypeName string //`json:"type"`

internal/resource/user.go

@@ -18,6 +18,8 @@ _	"os"
 	"strings"
 	"decl/internal/codec"
 	"decl/internal/command"
+	"decl/internal/data"
+	"decl/internal/folio"
 )
 
 type decodeUser User
@@ -25,8 +27,9 @@ type decodeUser User
 type UserType string
 
 const (
-	UserTypeAddUser = "adduser"
+	UserTypeName TypeName = "user"
-	UserTypeUserAdd = "useradd"
+	UserTypeAddUser UserType = "adduser"
+	UserTypeUserAdd UserType = "useradd"
 )
 
 var ErrUnsupportedUserType error = errors.New("The UserType is not supported on this system")
@@ -35,6 +38,7 @@ var ErrInvalidUserType error = errors.New("invalid UserType value")
 var SystemUserType UserType = FindSystemUserType()
 
 type User struct {
+	*Common             `json:",inline" yaml:",inline"`
 	stater		machine.Stater	`json:"-" yaml:"-"`
 	Name       string   `json:"name" yaml:"name"`
 	UID        string   `json:"uid,omitempty" yaml:"uid,omitempty"`
@@ -50,17 +54,16 @@ type User struct {
 	ReadCommand *command.Command   `json:"-" yaml:"-"`
 	UpdateCommand *command.Command `json:"-" yaml:"-"`
 	DeleteCommand *command.Command `json:"-" yaml:"-"`
-	State string           `json:"state,omitempty" yaml:"state,omitempty"`
+	config data.ConfigurationValueGetter
-	config ConfigurationValueGetter
+	Resources data.ResourceMapper `json:"-" yaml:"-"`
-	Resources ResourceMapper `json:"-" yaml:"-"`
 }
 
 func NewUser() *User {
-	return &User{ CreateHome: true }
+	return &User{ CreateHome: true, Common: &Common{ resourceType: UserTypeName } }
 }
 
 func init() {
-	ResourceTypes.Register([]string{"user"}, func(u *url.URL) Resource {
+	folio.DocumentRegistry.ResourceTypes.Register([]string{"user"}, func(u *url.URL) data.Resource {
 		user := NewUser()
 		user.Name = u.Hostname()
 		user.UID = LookupUIDString(u.Hostname())
@@ -85,12 +88,13 @@ func FindSystemUserType() UserType {
 	return UserTypeAddUser
 }
 
-func (u *User) SetResourceMapper(resources ResourceMapper) {
+func (u *User) SetResourceMapper(resources data.ResourceMapper) {
 	u.Resources = resources
 }
 
-func (u *User) Clone() Resource {
+func (u *User) Clone() data.Resource {
 	newu := &User {
+		Common: u.Common,
 		Name: u.Name,
 		UID: u.UID,
 		Group: u.Group,
@@ -99,7 +103,6 @@ func (u *User) Clone() Resource {
 		Home: u.Home,
 		CreateHome: u.CreateHome,
 		Shell: u.Shell,
-		State: u.State,
 		UserType: u.UserType,
 	}
 	newu.CreateCommand, newu.ReadCommand, newu.UpdateCommand, newu.DeleteCommand = u.UserType.NewCRUD()
@@ -123,11 +126,11 @@ func (u *User) Notify(m *machine.EventMessage) {
 				if triggerErr := u.StateMachine().Trigger("state_read"); triggerErr == nil {
 					return
 				} else {
-					u.State = "absent"
+					u.Common.State = "absent"
 					panic(triggerErr)
 				}
 			} else {
-				u.State = "absent"
+				u.Common.State = "absent"
 				panic(readErr)
 			}
 		case "start_delete":
@@ -135,11 +138,11 @@ func (u *User) Notify(m *machine.EventMessage) {
 				if triggerErr := u.StateMachine().Trigger("deleted"); triggerErr == nil {
 					return
 				} else {
-					u.State = "present"
+					u.Common.State = "present"
 					panic(triggerErr)
 				}
 			} else {
-				u.State = "present"
+				u.Common.State = "present"
 				panic(deleteErr)
 			}
 		case "start_create":
@@ -148,11 +151,11 @@ func (u *User) Notify(m *machine.EventMessage) {
 					return
 				}
 			}
-			u.State = "absent"
+			u.Common.State = "absent"
 		case "absent":
-			u.State = "absent"
+			u.Common.State = "absent"
 		case "present", "created", "read":
-			u.State = "present"
+			u.Common.State = "present"
 		}
 	case machine.EXITSTATEEVENT:
 	}
@@ -174,7 +177,7 @@ func (u *User) URI() string {
 	return fmt.Sprintf("user://%s", u.Name)
 }
 
-func (u *User) UseConfig(config ConfigurationValueGetter) {
+func (u *User) UseConfig(config data.ConfigurationValueGetter) {
 	u.config = config
 }
 
@@ -188,7 +191,7 @@ func (u *User) Validate() error {
 
 func (u *User) Apply() error {
 	ctx := context.Background()
-	switch u.State {
+	switch u.Common.State {
 	case "present":
 		_, NoUserExists := LookupUID(u.Name)
 		if NoUserExists != nil {
@@ -202,12 +205,23 @@ func (u *User) Apply() error {
 	return nil
 }
 
-func (u *User) Load(r io.Reader) error {
+func (u *User) Load(docData []byte, f codec.Format) (err error) {
-	return codec.NewYAMLDecoder(r).Decode(u)
+	err = f.StringDecoder(string(docData)).Decode(u)
+	return
+}
+
+func (u *User) LoadReader(r io.ReadCloser, f codec.Format) (err error) {
+	err = f.Decoder(r).Decode(u)
+	return
+}
+
+func (u *User) LoadString(docData string, f codec.Format) (err error) {
+	err = f.StringDecoder(docData).Decode(u)
+	return
 }
 
 func (u *User) LoadDecl(yamlResourceDeclaration string) error {
-	return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(u)
+	return u.LoadString(yamlResourceDeclaration, codec.FormatYaml)
 }
 
 func (u *User) Type() string { return "user" }
@@ -225,7 +239,7 @@ func (u *User) Create(ctx context.Context) (error) {
 func (u *User) Read(ctx context.Context) ([]byte, error) {
 	exErr := u.ReadCommand.Extractor(nil, u)
 	if exErr != nil {
-		u.State = "absent"
+		u.Common.State = "absent"
 	}
 	if yaml, yamlErr := yaml.Marshal(u); yamlErr != nil {
 		return yaml, yamlErr
@@ -234,6 +248,10 @@ func (u *User) Read(ctx context.Context) ([]byte, error) {
 	}
 }
 
+func (u *User) Update(ctx context.Context) (error) {
+	return u.Create(ctx)
+}
+
 func (u *User) Delete(ctx context.Context) (error) {
 	_, err := u.DeleteCommand.Execute(u)
 	if err != nil {
@@ -358,7 +376,7 @@ func NewReadUsersCommand() *command.Command {
 			if readGroup, groupErr := user.LookupGroupId(userRecord[3]); groupErr == nil {
 				u.Group = readGroup.Name
 			}
-			u.State = "present"
+			u.Common.State = "present"
 			u.UserType = SystemUserType
 			lineIndex++
 		}
@@ -423,7 +441,7 @@ func NewUserReadCommand() *command.Command {
 	c := command.NewCommand()
 	c.Extractor = func(out []byte, target any) error {
 		u := target.(*User)
-		u.State = "absent"
+		u.Common.State = "absent"
 		var readUser *user.User
 		var e error
 		if u.Name != "" {
@@ -445,7 +463,7 @@ func NewUserReadCommand() *command.Command {
 				return groupErr
 			}
 			if u.UID != "" {
-				u.State = "present"
+				u.Common.State = "present"
 			}
 		}
 		return e