add openpgp resources

Makefile

@@ -33,7 +33,7 @@ run:
 run-alpine:
 	docker run -it -v $(HOME)/.git-credentials:/root/.git-credentials -v $(HOME)/.gitconfig:/root/.gitconfig -v /var/run/docker.sock:/var/run/docker.sock -v $(shell pwd):/src golang:1.22.6-alpine sh
 build-container:
-	docker run -it -v $(HOME)/.git-credentials:/root/.git-credentials -v $(HOME)/.gitconfig:/root/.gitconfig -v /var/run/docker.sock:/var/run/docker.sock -v $(shell pwd):/src -w /src rosskeenhouse/build-golang:1.22.6-alpine sh
+	docker run -it -v $(HOME)/.git-credentials:/root/.git-credentials -v /tmp:/tmp -v $(HOME)/.gitconfig:/root/.gitconfig -v /var/run/docker.sock:/var/run/docker.sock -e WORKSPACE_PATH=$(shell pwd) -v $(shell pwd):/src -w /src rosskeenhouse/build-golang:1.22.6-alpine sh
 clean:
 	go clean -modcache
 	rm jx

examples/install.jx.yaml

@@ -0,0 +1,3 @@
+# Import the built-in install document which install the jx binary.
+imports:
+- file://documents/install.jx.yaml

internal/client/client.go

@@ -100,8 +100,10 @@ func (a *App) SetOutput(uri string) (err error) {
 
 // Each document has an `imports` keyword which can be used to load dependencies
 func (a *App) LoadDocumentImports() error {
+	slog.Info("Client.LoadDocumentImports()", "documents", a.Documents)
 	for i, d := range a.Documents {
 		importedDocs := d.ImportedDocuments()
+		slog.Info("Client.LoadDocumentImports()", "imported", importedDocs)
 		for _, importedDocument := range importedDocs {
 			docURI := folio.URI(importedDocument.GetURI())
 			if _, ok := a.ImportedMap[docURI]; !ok {

internal/client/resources_test.go

@@ -0,0 +1,162 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package client
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"fmt"
+	"context"
+	"decl/internal/folio"
+	"decl/internal/resource"
+	"decl/internal/codec"
+	"log/slog"
+	"os"
+	"io"
+	"strings"
+)
+
+	var containerDoc string = `
+imports:
+- %s
+resources:
+- type: container
+  transition: create
+  attributes:
+    image: rosskeenhouse/build-golang:1.22.6-alpine
+    name: jx-client-resources-test
+    hostconfig:
+      autoremove: false
+      mounts:
+      - type: "bind"
+        source: "%s"
+        target: "/src"
+      - type: "bind"
+        source: "%s"
+        target: "%s"
+    workingdir: "/src"
+    entrypoint:
+    - "/src/jx"
+    cmd:
+    - apply
+    - %s
+    wait: true
+---
+resources:
+- type: container
+  transition: delete
+  attributes:
+    name: jx-client-resources-test
+`
+
+// create a container
+// run a test inside the container
+func TestUserResource(t *testing.T) {
+	ctx := context.Background()
+	c := NewClient()
+	assert.NotNil(t, c)
+
+	TempDir.Mkdir("testresources", 0700)
+	tmpresourcespath := TempDir.FilePath("testresources")
+	configurations := fmt.Sprintf(`
+configurations:
+- name: tmpdir
+  values:
+    prefix: %s
+`, tmpresourcespath)
+
+	assert.Nil(t, TempDir.CreateFile("config.jx.yaml", configurations))
+
+	configURI := TempDir.URIPath("config.jx.yaml")
+	//assert.Nil(t, c.Import([]string{configURI}))
+
+	testUserFile := fmt.Sprintf(`
+imports:
+- %s
+resources:
+- type: file
+  config: tmpdir
+  transition: update
+  attributes:
+    path: testdir
+    mode: 0600
+    state: present
+- type: group
+  transition: update
+  attributes:
+    name: testuser
+- type: group
+  transition: update
+  attributes:
+    name: testgroup
+- type: user
+  transition: update
+  attributes:
+    name: testuser
+    gecos: "my test account"
+    home: "/home/testuser"
+    createhome: true
+    group: testuser
+    groups:
+    - testgroup
+    - testuser
+    appendgroups: true
+    state: present
+`, configURI)
+
+	assert.Nil(t, TempDir.CreateFile("test_userfile.jx.yaml", testUserFile))
+
+	for _, resourceTestDoc := range []string{
+		TempDir.FilePath("test_userfile.jx.yaml"),
+	} {
+		content := fmt.Sprintf(containerDoc, configURI, os.Getenv("WORKSPACE_PATH"), TempDir, TempDir, resourceTestDoc)
+		assert.Nil(t, TempDir.CreateFile("run-tests.jx.yaml", content))
+
+		runTestsDocument := TempDir.URIPath("run-tests.jx.yaml")
+		assert.Nil(t, c.Import([]string{runTestsDocument}))
+		assert.Nil(t, c.LoadDocumentImports())
+
+		assert.Nil(t, c.Apply(ctx, false))
+
+		applied, ok := folio.DocumentRegistry.GetDocument(folio.URI(runTestsDocument))
+		assert.True(t, ok)
+
+		cont := applied.ResourceDeclarations[0].Resource().(*resource.Container)
+
+		slog.Info("TestUserResources", "stdout", cont.Stdout, "stderr", cont.Stderr)
+		slog.Info("TestUserResources", "doc", applied, "container", applied.ResourceDeclarations[0])
+		assert.Equal(t, 0, len(applied.Errors))
+		assert.Greater(t, len(cont.Stdout), 0)
+
+		resultReader := io.NopCloser(strings.NewReader(cont.Stdout))
+		decoder := codec.NewDecoder(resultReader, codec.FormatYaml)
+
+		result := folio.NewDocument(nil)
+		assert.Nil(t, decoder.Decode(folio.NewDocument(nil)))
+		assert.Nil(t, decoder.Decode(result))
+
+		uri := fmt.Sprintf("file://%s", resourceTestDoc)
+		//testDoc := folio.DocumentRegistry.NewDocument(folio.URI(uri))
+		docs, loadErr := folio.DocumentRegistry.Load(folio.URI(uri))
+		assert.Nil(t, loadErr)
+		testDoc := docs[0]
+
+		var added int = 0
+		diffs, diffsErr := testDoc.(*folio.Document).Diff(result, nil)
+		assert.Nil(t, diffsErr)
+		assert.Greater(t, len(diffs), 1)
+		for _, line := range strings.Split(diffs, "\n") {
+			if len(line) > 0 {
+				switch line[0] {
+				case '+':
+					slog.Info("TestUserResources Diff", "line", line, "added", added)
+					added++
+				case '-':
+					assert.Fail(t, "resource attribute missing", line)
+				}
+			}
+		}
+		assert.Equal(t, 4, added)
+	}
+
+}

internal/config/openpgp.go

@@ -0,0 +1,90 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package config
+
+import (
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"crypto/rsa"
+	"crypto/rand"
+	"encoding/pem"
+	"encoding/json"
+	"github.com/ProtonMail/go-crypto/openpgp"
+	"github.com/ProtonMail/go-crypto/openpgp/packet"
+)
+
+type OpenPGP struct {
+	Armored string
+	entities openpgp.EntityList
+}
+
+func (o *OpenPGP) Read() (yamlData []byte, err error) {
+	pemReader := io.NopCloser(strings.NewReader(o.Armored))
+	o.entities, err = openpgp.ReadArmoredKeyRing(pemReader)
+	return
+}
+
+
+func (o *OpenPGP) UnmarshalJSON(data []byte) error {
+	if unmarshalErr := json.Unmarshal(data, o); unmarshalErr != nil {
+		return unmarshalErr
+	}
+	return nil
+}
+
+func (o *OpenPGP) UnmarshalYAML(value *yaml.Node) error {
+	type decodeOpenPGP OpenPGP
+	if unmarshalErr := value.Decode((*decodeOpenPGP)(o)); unmarshalErr != nil {
+		return unmarshalErr
+	}
+	return nil
+}
+
+func (o *OpenPGP) Clone() data.Configuration {
+	jsonGeneric, _ := json.Marshal(c)
+	clone := NewOpenPGP()
+	if unmarshalErr := json.Unmarshal(jsonGeneric, &clone); unmarshalErr != nil {
+		panic(unmarshalErr)
+	}
+	return clone
+}
+
+func (o *OpenPGP) Type() string {
+	return "openpgp"
+}
+
+func (o *OpenPGP) GetEntityIndex(key string) (index int, field string, err error) {
+	values := strings.SplitN(key, ".", 2)
+	if len(values) == 2 {
+		if index, err = strconv.Atoi(values[0]); err == nil {
+			field = values[1]
+		}
+	} else {
+		err = data.ErrUnknownConfigurationKey
+	}
+	return
+}
+
+func (o *OpenPGP) GetValue(name string) (result any, err error) {
+	var ok bool
+	if result, ok = (*c)[name]; !ok {
+		err = data.ErrUnknownConfigurationKey
+	}
+	return
+}
+
+
+// Expected key: 0.PrivateKey
+func (o *OpenPGP) Has(key string) (ok bool) {
+	index, field, err := o.GetEntityIndex(key)
+	if len(o.entities) > index && err == nil {
+		switch key {
+		case PublicKey:
+			ok = o.entities[index].PrimaryKey != nil
+		case PrivateKey:
+			ok = o.entities[index].PrimaryKey != nil
+		}
+	}
+	return
+}
+

internal/config/openpgp_test.go

@@ -0,0 +1,31 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package config
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"crypto/x509"
+)
+
+func TestNewOpenPGPConfig(t *testing.T) {
+	p := NewOpenPGP()
+	assert.NotNil(t, p)
+}
+
+func TestNewOpenPGPConfigYAML(t *testing.T) {
+	p := NewOpenPGP()
+	assert.NotNil(t, p)
+
+	config := `
+openpgp:
+  publickey:
+
+`
+
+	yamlErr := c.LoadYAML(config)
+	assert.Nil(t, yamlErr)
+	crt, err := c.GetValue("catemplate")
+	assert.Nil(t, err)
+	assert.Equal(t, []string{"RKH"}, crt.(*x509.Certificate).Subject.Organization)
+}

internal/data/command.go

@@ -0,0 +1,27 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package data
+
+import (
+)
+
+var (
+)
+
+type CommandExecutor interface {
+	Execute(value any) ([]byte, error)
+}
+
+type CommandOutputExtractor interface {
+	Extract(output []byte, target any) error
+}
+
+type CommandChecker interface {
+	Exists() error
+}
+
+type Commander interface {
+	CommandExecutor
+	CommandOutputExtractor
+	CommandChecker
+}

internal/data/resource.go

@@ -106,6 +106,13 @@ type FileResource interface {
 	SetGzipContent(bool)
 }
 
+type ExecResource interface {
+	Start() error
+	Wait() error
+	StdoutPipe() (io.ReadCloser, error)
+	StderrPipe() (io.ReadCloser, error)
+}
+
 type Signed interface {
 	Signature() Signature
 }

internal/folio/declaration.go

@@ -219,6 +219,7 @@ func (d *Declaration) Apply(stateTransition string) (result error) {
 			}
 		}
 
+		slog.Info("Declaration.Apply() - read", "state", stater.CurrentState(), "declaration", d)
 		result = stater.Trigger("read")
 		currentState := stater.CurrentState()
 		switch currentState {

internal/folio/document.go

@@ -236,15 +236,17 @@ func (d *Document) GetSchemaFiles() (schemaFs fs.FS) {
 		return
 	}
 	schemaFs, _  = d.Registry.Schemas.Get(d.Registry.DefaultSchema)
+	slog.Info("Document.GetSchemaFiles()", "schemaFs", schemaFs)
 	return
 }
 
 func (d *Document) Validate() error {
 	jsonDocument, jsonErr := d.JSON()
-	slog.Info("document.Validate() json", "err", jsonErr)
+	slog.Info("Document.Validate() json", "err", jsonErr)
 	if jsonErr == nil {
 		s := schema.New("document", d.GetSchemaFiles())
 		err := s.Validate(string(jsonDocument))
+		slog.Info("Document.Validate()", "error", err)
 		if err != nil {
 			return err
 		}
@@ -535,6 +537,35 @@ func (d *Document) DiffState(output io.Writer) (returnOutput string, diffErr err
 	return d.Diff(clone, output)
 }
 
+func (d *Document) YamlDiff(with data.Document) (diffs []*yamldiff.YamlDiff, diffErr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			diffErr = fmt.Errorf("%s", r)
+		}
+	}()
+
+	opts := []yamldiff.DoOptionFunc{}
+	ydata, yerr := d.YAML()
+	if yerr != nil {
+		return nil, yerr
+	}
+	yamlDiff,yamlDiffErr := yamldiff.Load(string(ydata))
+	if yamlDiffErr != nil {
+		return nil, yamlDiffErr
+	}
+	wdata,werr := with.YAML()
+	if werr != nil {
+		return nil, werr
+	}
+	withDiff,withDiffErr := yamldiff.Load(string(wdata))
+	if withDiffErr != nil {
+		return nil, withDiffErr
+	}
+	
+	slog.Info("Document.Diff() ", "document.yaml", ydata, "with.yaml", wdata)
+	return yamldiff.Do(yamlDiff, withDiff, opts...), nil
+}
+
 func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput string, diffErr error) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -542,37 +573,21 @@ func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput stri
 			diffErr = fmt.Errorf("%s", r)
 		}
 	}()
-	slog.Info("Document.Diff()")
+
-	opts := []yamldiff.DoOptionFunc{}
 	if output == nil {
 		output = &strings.Builder{}
 	}
-	ydata, yerr := d.YAML()
+
-	if yerr != nil {
+	var diffs []*yamldiff.YamlDiff
-		return "", yerr
+	diffs, diffErr = d.YamlDiff(with)
-	}
-	yamlDiff,yamlDiffErr := yamldiff.Load(string(ydata))
-	if yamlDiffErr != nil {
-		return "", yamlDiffErr
-	}
 	
-	wdata,werr := with.YAML()
+	for _,docDiffResults := range diffs {
-	if werr != nil {
-		return "", werr
-	}
-	withDiff,withDiffErr := yamldiff.Load(string(wdata))
-	if withDiffErr != nil {
-		return "", withDiffErr
-	}
-	
-	for _,docDiffResults := range yamldiff.Do(yamlDiff, withDiff, opts...) {
 		slog.Info("Diff()", "diff", docDiffResults, "dump", docDiffResults.Dump())
 		_,e := output.Write([]byte(docDiffResults.Dump()))
 		if e != nil {
 			return "", e
 		}
 	}
-	slog.Info("Document.Diff() ", "document.yaml", ydata, "with.yaml", wdata)
 	if stringOutput, ok := output.(*strings.Builder); ok {
 		return stringOutput.String(), nil
 	}
@@ -585,7 +600,7 @@ func (d *Document) UnmarshalValue(value *DocumentType) error {
 }
 */
 
-func (d *Document) UnmarshalYAML(value *yaml.Node) error {
+func (d *Document) UnmarshalYAML(value *yaml.Node) (err error) {
 	type decodeDocument Document
 	t := &DocumentType{}
 	if unmarshalDocumentErr := value.Decode(t); unmarshalDocumentErr != nil {
@@ -595,20 +610,22 @@ func (d *Document) UnmarshalYAML(value *yaml.Node) error {
 	if unmarshalResourcesErr := value.Decode((*decodeDocument)(d)); unmarshalResourcesErr != nil {
 		return unmarshalResourcesErr
 	}
+	err = d.loadImports()
 	d.assignConfigurationsDocument()
 	d.assignResourcesDocument()
-	return d.loadImports()
+	return 
 }
 
-func (d *Document) UnmarshalJSON(data []byte) error {
+func (d *Document) UnmarshalJSON(data []byte) (err error) {
 	type decodeDocument Document
 	t := (*decodeDocument)(d)
 	if unmarshalDocumentErr := json.Unmarshal(data, t); unmarshalDocumentErr != nil {
 		return unmarshalDocumentErr
 	}
+	err = d.loadImports()
 	d.assignConfigurationsDocument()
 	d.assignResourcesDocument()
-	return d.loadImports()
+	return 
 }
 
 func (d *Document) AddError(e error) {

internal/folio/resourcereference.go

@@ -38,7 +38,7 @@ func (r ResourceReference) Lookup(look data.ResourceMapper) ContentReadWriter {
 	slog.Info("ResourceReference.Lookup()", "resourcereference", r, "resourcemapper", look)
 	if look != nil {
 		if v,ok := look.Get(string(r)); ok {
-			return v.(ContentReadWriter)
+			return v.Resource().(ContentReadWriter)
 		}
 	}
 	return r
@@ -70,3 +70,7 @@ func (r ResourceReference) ContentReaderStream() (*transport.Reader, error) {
 func (r ResourceReference) ContentWriterStream() (*transport.Writer, error) {
 	return URI(r).ContentWriterStream()
 }
+
+func (r ResourceReference) IsEmpty() bool {
+	return URI(r).IsEmpty()
+}

tests/mocks/container.go

@@ -7,6 +7,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/api/types/image"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"io"
@@ -23,11 +24,16 @@ type MockContainerClient struct {
 	InjectContainerRemove     func(context.Context, string, container.RemoveOptions) error
 	InjectContainerStop       func(context.Context, string, container.StopOptions) error
 	InjectContainerWait				func(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.WaitResponse, <-chan error)
+	InjectContainerLogs				func(ctx context.Context, containerID string, options container.LogsOptions) (io.ReadCloser, error)
 	InjectImagePull           func(ctx context.Context, refStr string, options image.PullOptions) (io.ReadCloser, error)
 	InjectImagePush           func(ctx context.Context, image string, options image.PushOptions) (io.ReadCloser, error)
 	InjectImageInspectWithRaw func(ctx context.Context, imageID string) (types.ImageInspect, []byte, error)
 	InjectImageRemove         func(ctx context.Context, imageID string, options image.RemoveOptions) ([]image.DeleteResponse, error)
 	InjectImageBuild					func(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
+	InjectVolumeCreate        func(ctx context.Context, options volume.CreateOptions) (volume.Volume, error)
+	InjectVolumeList          func(ctx context.Context, options volume.ListOptions) (volume.ListResponse, error)
+	InjectVolumeInspect       func(ctx context.Context, volumeID string) (volume.Volume, error)
+	InjectVolumeRemove        func(ctx context.Context, volumeID string, force bool) (error)
 	InjectClose               func() error
 }
 
@@ -35,6 +41,10 @@ func (m *MockContainerClient) ContainerWait(ctx context.Context, containerID str
 	return m.InjectContainerWait(ctx, containerID, condition)
 }
 
+func (m *MockContainerClient) ContainerLogs(ctx context.Context, containerID string, options container.LogsOptions) (io.ReadCloser, error) {
+	return m.InjectContainerLogs(ctx, containerID, options)
+}
+
 func (m *MockContainerClient) ImageRemove(ctx context.Context, imageID string, options image.RemoveOptions) ([]image.DeleteResponse, error) {
 	return m.InjectImageRemove(ctx, imageID, options)
 }
@@ -100,3 +110,19 @@ func (m *MockContainerClient) NetworkList(ctx context.Context, options network.L
 func (m *MockContainerClient) NetworkInspect(ctx context.Context, networkID string, options network.InspectOptions) (network.Inspect, error) {
 	return m.InjectNetworkInspect(ctx, networkID, options)
 }
+
+func (m *MockContainerClient) VolumeCreate(ctx context.Context, options volume.CreateOptions) (volume.Volume, error) {
+	return m.InjectVolumeCreate(ctx, options)
+}
+
+func (m *MockContainerClient) VolumeList(ctx context.Context, options volume.ListOptions) (volume.ListResponse, error) {
+	return m.InjectVolumeList(ctx, options)
+}
+
+func (m *MockContainerClient) VolumeInspect(ctx context.Context, volumeID string) (volume.Volume, error) {
+	return m.InjectVolumeInspect(ctx, volumeID)
+}
+
+func (m *MockContainerClient) VolumeRemove(ctx context.Context, volumeID string, force bool) (error) {
+	return m.InjectVolumeRemove(ctx, volumeID, force)
+}