From 35899c86a5b8a28a88a8fa72a49f6fcd032a671c Mon Sep 17 00:00:00 2001 From: Matthew Rich Date: Sun, 10 Nov 2024 10:27:31 -0800 Subject: [PATCH] add openpgp resources --- Makefile | 2 +- examples/install.jx.yaml | 3 + internal/client/client.go | 2 + internal/client/resources_test.go | 162 ++++++++++++++++++++++++++++ internal/config/openpgp.go | 90 ++++++++++++++++ internal/config/openpgp_test.go | 31 ++++++ internal/data/command.go | 27 +++++ internal/data/resource.go | 7 ++ internal/folio/declaration.go | 1 + internal/folio/document.go | 69 +++++++----- internal/folio/resourcereference.go | 6 +- tests/mocks/container.go | 26 +++++ 12 files changed, 398 insertions(+), 28 deletions(-) create mode 100644 examples/install.jx.yaml create mode 100644 internal/client/resources_test.go create mode 100644 internal/config/openpgp.go create mode 100644 internal/config/openpgp_test.go create mode 100644 internal/data/command.go diff --git a/Makefile b/Makefile index 79d85d4..400d4bd 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ run: run-alpine: docker run -it -v $(HOME)/.git-credentials:/root/.git-credentials -v $(HOME)/.gitconfig:/root/.gitconfig -v /var/run/docker.sock:/var/run/docker.sock -v $(shell pwd):/src golang:1.22.6-alpine sh build-container: - docker run -it -v $(HOME)/.git-credentials:/root/.git-credentials -v $(HOME)/.gitconfig:/root/.gitconfig -v /var/run/docker.sock:/var/run/docker.sock -v $(shell pwd):/src -w /src rosskeenhouse/build-golang:1.22.6-alpine sh + docker run -it -v $(HOME)/.git-credentials:/root/.git-credentials -v /tmp:/tmp -v $(HOME)/.gitconfig:/root/.gitconfig -v /var/run/docker.sock:/var/run/docker.sock -e WORKSPACE_PATH=$(shell pwd) -v $(shell pwd):/src -w /src rosskeenhouse/build-golang:1.22.6-alpine sh clean: go clean -modcache rm jx diff --git a/examples/install.jx.yaml b/examples/install.jx.yaml new file mode 100644 index 0000000..34f7cb2 --- /dev/null +++ b/examples/install.jx.yaml @@ -0,0 +1,3 @@ +# Import the built-in install document which install the jx binary. +imports: +- file://documents/install.jx.yaml diff --git a/internal/client/client.go b/internal/client/client.go index a8b8a5c..5bc6d8c 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -100,8 +100,10 @@ func (a *App) SetOutput(uri string) (err error) { // Each document has an `imports` keyword which can be used to load dependencies func (a *App) LoadDocumentImports() error { + slog.Info("Client.LoadDocumentImports()", "documents", a.Documents) for i, d := range a.Documents { importedDocs := d.ImportedDocuments() + slog.Info("Client.LoadDocumentImports()", "imported", importedDocs) for _, importedDocument := range importedDocs { docURI := folio.URI(importedDocument.GetURI()) if _, ok := a.ImportedMap[docURI]; !ok { diff --git a/internal/client/resources_test.go b/internal/client/resources_test.go new file mode 100644 index 0000000..93325ba --- /dev/null +++ b/internal/client/resources_test.go @@ -0,0 +1,162 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package client + +import ( + "github.com/stretchr/testify/assert" + "testing" + "fmt" + "context" + "decl/internal/folio" + "decl/internal/resource" + "decl/internal/codec" + "log/slog" + "os" + "io" + "strings" +) + + var containerDoc string = ` +imports: +- %s +resources: +- type: container + transition: create + attributes: + image: rosskeenhouse/build-golang:1.22.6-alpine + name: jx-client-resources-test + hostconfig: + autoremove: false + mounts: + - type: "bind" + source: "%s" + target: "/src" + - type: "bind" + source: "%s" + target: "%s" + workingdir: "/src" + entrypoint: + - "/src/jx" + cmd: + - apply + - %s + wait: true +--- +resources: +- type: container + transition: delete + attributes: + name: jx-client-resources-test +` + +// create a container +// run a test inside the container +func TestUserResource(t *testing.T) { + ctx := context.Background() + c := NewClient() + assert.NotNil(t, c) + + TempDir.Mkdir("testresources", 0700) + tmpresourcespath := TempDir.FilePath("testresources") + configurations := fmt.Sprintf(` +configurations: +- name: tmpdir + values: + prefix: %s +`, tmpresourcespath) + + assert.Nil(t, TempDir.CreateFile("config.jx.yaml", configurations)) + + configURI := TempDir.URIPath("config.jx.yaml") + //assert.Nil(t, c.Import([]string{configURI})) + + testUserFile := fmt.Sprintf(` +imports: +- %s +resources: +- type: file + config: tmpdir + transition: update + attributes: + path: testdir + mode: 0600 + state: present +- type: group + transition: update + attributes: + name: testuser +- type: group + transition: update + attributes: + name: testgroup +- type: user + transition: update + attributes: + name: testuser + gecos: "my test account" + home: "/home/testuser" + createhome: true + group: testuser + groups: + - testgroup + - testuser + appendgroups: true + state: present +`, configURI) + + assert.Nil(t, TempDir.CreateFile("test_userfile.jx.yaml", testUserFile)) + + for _, resourceTestDoc := range []string{ + TempDir.FilePath("test_userfile.jx.yaml"), + } { + content := fmt.Sprintf(containerDoc, configURI, os.Getenv("WORKSPACE_PATH"), TempDir, TempDir, resourceTestDoc) + assert.Nil(t, TempDir.CreateFile("run-tests.jx.yaml", content)) + + runTestsDocument := TempDir.URIPath("run-tests.jx.yaml") + assert.Nil(t, c.Import([]string{runTestsDocument})) + assert.Nil(t, c.LoadDocumentImports()) + + assert.Nil(t, c.Apply(ctx, false)) + + applied, ok := folio.DocumentRegistry.GetDocument(folio.URI(runTestsDocument)) + assert.True(t, ok) + + cont := applied.ResourceDeclarations[0].Resource().(*resource.Container) + + slog.Info("TestUserResources", "stdout", cont.Stdout, "stderr", cont.Stderr) + slog.Info("TestUserResources", "doc", applied, "container", applied.ResourceDeclarations[0]) + assert.Equal(t, 0, len(applied.Errors)) + assert.Greater(t, len(cont.Stdout), 0) + + resultReader := io.NopCloser(strings.NewReader(cont.Stdout)) + decoder := codec.NewDecoder(resultReader, codec.FormatYaml) + + result := folio.NewDocument(nil) + assert.Nil(t, decoder.Decode(folio.NewDocument(nil))) + assert.Nil(t, decoder.Decode(result)) + + uri := fmt.Sprintf("file://%s", resourceTestDoc) + //testDoc := folio.DocumentRegistry.NewDocument(folio.URI(uri)) + docs, loadErr := folio.DocumentRegistry.Load(folio.URI(uri)) + assert.Nil(t, loadErr) + testDoc := docs[0] + + var added int = 0 + diffs, diffsErr := testDoc.(*folio.Document).Diff(result, nil) + assert.Nil(t, diffsErr) + assert.Greater(t, len(diffs), 1) + for _, line := range strings.Split(diffs, "\n") { + if len(line) > 0 { + switch line[0] { + case '+': + slog.Info("TestUserResources Diff", "line", line, "added", added) + added++ + case '-': + assert.Fail(t, "resource attribute missing", line) + } + } + } + assert.Equal(t, 4, added) + } + +} diff --git a/internal/config/openpgp.go b/internal/config/openpgp.go new file mode 100644 index 0000000..8da7bff --- /dev/null +++ b/internal/config/openpgp.go @@ -0,0 +1,90 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package config + +import ( + "crypto/x509" + "crypto/x509/pkix" + "crypto/rsa" + "crypto/rand" + "encoding/pem" + "encoding/json" + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/packet" +) + +type OpenPGP struct { + Armored string + entities openpgp.EntityList +} + +func (o *OpenPGP) Read() (yamlData []byte, err error) { + pemReader := io.NopCloser(strings.NewReader(o.Armored)) + o.entities, err = openpgp.ReadArmoredKeyRing(pemReader) + return +} + + +func (o *OpenPGP) UnmarshalJSON(data []byte) error { + if unmarshalErr := json.Unmarshal(data, o); unmarshalErr != nil { + return unmarshalErr + } + return nil +} + +func (o *OpenPGP) UnmarshalYAML(value *yaml.Node) error { + type decodeOpenPGP OpenPGP + if unmarshalErr := value.Decode((*decodeOpenPGP)(o)); unmarshalErr != nil { + return unmarshalErr + } + return nil +} + +func (o *OpenPGP) Clone() data.Configuration { + jsonGeneric, _ := json.Marshal(c) + clone := NewOpenPGP() + if unmarshalErr := json.Unmarshal(jsonGeneric, &clone); unmarshalErr != nil { + panic(unmarshalErr) + } + return clone +} + +func (o *OpenPGP) Type() string { + return "openpgp" +} + +func (o *OpenPGP) GetEntityIndex(key string) (index int, field string, err error) { + values := strings.SplitN(key, ".", 2) + if len(values) == 2 { + if index, err = strconv.Atoi(values[0]); err == nil { + field = values[1] + } + } else { + err = data.ErrUnknownConfigurationKey + } + return +} + +func (o *OpenPGP) GetValue(name string) (result any, err error) { + var ok bool + if result, ok = (*c)[name]; !ok { + err = data.ErrUnknownConfigurationKey + } + return +} + + +// Expected key: 0.PrivateKey +func (o *OpenPGP) Has(key string) (ok bool) { + index, field, err := o.GetEntityIndex(key) + if len(o.entities) > index && err == nil { + switch key { + case PublicKey: + ok = o.entities[index].PrimaryKey != nil + case PrivateKey: + ok = o.entities[index].PrimaryKey != nil + } + } + return +} + diff --git a/internal/config/openpgp_test.go b/internal/config/openpgp_test.go new file mode 100644 index 0000000..2b0cc53 --- /dev/null +++ b/internal/config/openpgp_test.go @@ -0,0 +1,31 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package config + +import ( + "github.com/stretchr/testify/assert" + "testing" + "crypto/x509" +) + +func TestNewOpenPGPConfig(t *testing.T) { + p := NewOpenPGP() + assert.NotNil(t, p) +} + +func TestNewOpenPGPConfigYAML(t *testing.T) { + p := NewOpenPGP() + assert.NotNil(t, p) + + config := ` +openpgp: + publickey: + +` + + yamlErr := c.LoadYAML(config) + assert.Nil(t, yamlErr) + crt, err := c.GetValue("catemplate") + assert.Nil(t, err) + assert.Equal(t, []string{"RKH"}, crt.(*x509.Certificate).Subject.Organization) +} diff --git a/internal/data/command.go b/internal/data/command.go new file mode 100644 index 0000000..a599a40 --- /dev/null +++ b/internal/data/command.go @@ -0,0 +1,27 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package data + +import ( +) + +var ( +) + +type CommandExecutor interface { + Execute(value any) ([]byte, error) +} + +type CommandOutputExtractor interface { + Extract(output []byte, target any) error +} + +type CommandChecker interface { + Exists() error +} + +type Commander interface { + CommandExecutor + CommandOutputExtractor + CommandChecker +} diff --git a/internal/data/resource.go b/internal/data/resource.go index d98b909..6510bb1 100644 --- a/internal/data/resource.go +++ b/internal/data/resource.go @@ -106,6 +106,13 @@ type FileResource interface { SetGzipContent(bool) } +type ExecResource interface { + Start() error + Wait() error + StdoutPipe() (io.ReadCloser, error) + StderrPipe() (io.ReadCloser, error) +} + type Signed interface { Signature() Signature } diff --git a/internal/folio/declaration.go b/internal/folio/declaration.go index 926e546..9ac817a 100644 --- a/internal/folio/declaration.go +++ b/internal/folio/declaration.go @@ -219,6 +219,7 @@ func (d *Declaration) Apply(stateTransition string) (result error) { } } + slog.Info("Declaration.Apply() - read", "state", stater.CurrentState(), "declaration", d) result = stater.Trigger("read") currentState := stater.CurrentState() switch currentState { diff --git a/internal/folio/document.go b/internal/folio/document.go index ee4f390..8cf8bc8 100644 --- a/internal/folio/document.go +++ b/internal/folio/document.go @@ -236,15 +236,17 @@ func (d *Document) GetSchemaFiles() (schemaFs fs.FS) { return } schemaFs, _ = d.Registry.Schemas.Get(d.Registry.DefaultSchema) + slog.Info("Document.GetSchemaFiles()", "schemaFs", schemaFs) return } func (d *Document) Validate() error { jsonDocument, jsonErr := d.JSON() - slog.Info("document.Validate() json", "err", jsonErr) + slog.Info("Document.Validate() json", "err", jsonErr) if jsonErr == nil { s := schema.New("document", d.GetSchemaFiles()) err := s.Validate(string(jsonDocument)) + slog.Info("Document.Validate()", "error", err) if err != nil { return err } @@ -535,6 +537,35 @@ func (d *Document) DiffState(output io.Writer) (returnOutput string, diffErr err return d.Diff(clone, output) } +func (d *Document) YamlDiff(with data.Document) (diffs []*yamldiff.YamlDiff, diffErr error) { + defer func() { + if r := recover(); r != nil { + diffErr = fmt.Errorf("%s", r) + } + }() + + opts := []yamldiff.DoOptionFunc{} + ydata, yerr := d.YAML() + if yerr != nil { + return nil, yerr + } + yamlDiff,yamlDiffErr := yamldiff.Load(string(ydata)) + if yamlDiffErr != nil { + return nil, yamlDiffErr + } + wdata,werr := with.YAML() + if werr != nil { + return nil, werr + } + withDiff,withDiffErr := yamldiff.Load(string(wdata)) + if withDiffErr != nil { + return nil, withDiffErr + } + + slog.Info("Document.Diff() ", "document.yaml", ydata, "with.yaml", wdata) + return yamldiff.Do(yamlDiff, withDiff, opts...), nil +} + func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput string, diffErr error) { defer func() { if r := recover(); r != nil { @@ -542,37 +573,21 @@ func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput stri diffErr = fmt.Errorf("%s", r) } }() - slog.Info("Document.Diff()") - opts := []yamldiff.DoOptionFunc{} + if output == nil { output = &strings.Builder{} } - ydata, yerr := d.YAML() - if yerr != nil { - return "", yerr - } - yamlDiff,yamlDiffErr := yamldiff.Load(string(ydata)) - if yamlDiffErr != nil { - return "", yamlDiffErr - } + + var diffs []*yamldiff.YamlDiff + diffs, diffErr = d.YamlDiff(with) - wdata,werr := with.YAML() - if werr != nil { - return "", werr - } - withDiff,withDiffErr := yamldiff.Load(string(wdata)) - if withDiffErr != nil { - return "", withDiffErr - } - - for _,docDiffResults := range yamldiff.Do(yamlDiff, withDiff, opts...) { + for _,docDiffResults := range diffs { slog.Info("Diff()", "diff", docDiffResults, "dump", docDiffResults.Dump()) _,e := output.Write([]byte(docDiffResults.Dump())) if e != nil { return "", e } } - slog.Info("Document.Diff() ", "document.yaml", ydata, "with.yaml", wdata) if stringOutput, ok := output.(*strings.Builder); ok { return stringOutput.String(), nil } @@ -585,7 +600,7 @@ func (d *Document) UnmarshalValue(value *DocumentType) error { } */ -func (d *Document) UnmarshalYAML(value *yaml.Node) error { +func (d *Document) UnmarshalYAML(value *yaml.Node) (err error) { type decodeDocument Document t := &DocumentType{} if unmarshalDocumentErr := value.Decode(t); unmarshalDocumentErr != nil { @@ -595,20 +610,22 @@ func (d *Document) UnmarshalYAML(value *yaml.Node) error { if unmarshalResourcesErr := value.Decode((*decodeDocument)(d)); unmarshalResourcesErr != nil { return unmarshalResourcesErr } + err = d.loadImports() d.assignConfigurationsDocument() d.assignResourcesDocument() - return d.loadImports() + return } -func (d *Document) UnmarshalJSON(data []byte) error { +func (d *Document) UnmarshalJSON(data []byte) (err error) { type decodeDocument Document t := (*decodeDocument)(d) if unmarshalDocumentErr := json.Unmarshal(data, t); unmarshalDocumentErr != nil { return unmarshalDocumentErr } + err = d.loadImports() d.assignConfigurationsDocument() d.assignResourcesDocument() - return d.loadImports() + return } func (d *Document) AddError(e error) { diff --git a/internal/folio/resourcereference.go b/internal/folio/resourcereference.go index 8faabd2..89fdd0d 100644 --- a/internal/folio/resourcereference.go +++ b/internal/folio/resourcereference.go @@ -38,7 +38,7 @@ func (r ResourceReference) Lookup(look data.ResourceMapper) ContentReadWriter { slog.Info("ResourceReference.Lookup()", "resourcereference", r, "resourcemapper", look) if look != nil { if v,ok := look.Get(string(r)); ok { - return v.(ContentReadWriter) + return v.Resource().(ContentReadWriter) } } return r @@ -70,3 +70,7 @@ func (r ResourceReference) ContentReaderStream() (*transport.Reader, error) { func (r ResourceReference) ContentWriterStream() (*transport.Writer, error) { return URI(r).ContentWriterStream() } + +func (r ResourceReference) IsEmpty() bool { + return URI(r).IsEmpty() +} diff --git a/tests/mocks/container.go b/tests/mocks/container.go index 32b2faf..cf05762 100644 --- a/tests/mocks/container.go +++ b/tests/mocks/container.go @@ -7,6 +7,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/volume" "github.com/docker/docker/api/types/image" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "io" @@ -23,11 +24,16 @@ type MockContainerClient struct { InjectContainerRemove func(context.Context, string, container.RemoveOptions) error InjectContainerStop func(context.Context, string, container.StopOptions) error InjectContainerWait func(ctx context.Context, containerID string, condition container.WaitCondition) (<-chan container.WaitResponse, <-chan error) + InjectContainerLogs func(ctx context.Context, containerID string, options container.LogsOptions) (io.ReadCloser, error) InjectImagePull func(ctx context.Context, refStr string, options image.PullOptions) (io.ReadCloser, error) InjectImagePush func(ctx context.Context, image string, options image.PushOptions) (io.ReadCloser, error) InjectImageInspectWithRaw func(ctx context.Context, imageID string) (types.ImageInspect, []byte, error) InjectImageRemove func(ctx context.Context, imageID string, options image.RemoveOptions) ([]image.DeleteResponse, error) InjectImageBuild func(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) + InjectVolumeCreate func(ctx context.Context, options volume.CreateOptions) (volume.Volume, error) + InjectVolumeList func(ctx context.Context, options volume.ListOptions) (volume.ListResponse, error) + InjectVolumeInspect func(ctx context.Context, volumeID string) (volume.Volume, error) + InjectVolumeRemove func(ctx context.Context, volumeID string, force bool) (error) InjectClose func() error } @@ -35,6 +41,10 @@ func (m *MockContainerClient) ContainerWait(ctx context.Context, containerID str return m.InjectContainerWait(ctx, containerID, condition) } +func (m *MockContainerClient) ContainerLogs(ctx context.Context, containerID string, options container.LogsOptions) (io.ReadCloser, error) { + return m.InjectContainerLogs(ctx, containerID, options) +} + func (m *MockContainerClient) ImageRemove(ctx context.Context, imageID string, options image.RemoveOptions) ([]image.DeleteResponse, error) { return m.InjectImageRemove(ctx, imageID, options) } @@ -100,3 +110,19 @@ func (m *MockContainerClient) NetworkList(ctx context.Context, options network.L func (m *MockContainerClient) NetworkInspect(ctx context.Context, networkID string, options network.InspectOptions) (network.Inspect, error) { return m.InjectNetworkInspect(ctx, networkID, options) } + +func (m *MockContainerClient) VolumeCreate(ctx context.Context, options volume.CreateOptions) (volume.Volume, error) { + return m.InjectVolumeCreate(ctx, options) +} + +func (m *MockContainerClient) VolumeList(ctx context.Context, options volume.ListOptions) (volume.ListResponse, error) { + return m.InjectVolumeList(ctx, options) +} + +func (m *MockContainerClient) VolumeInspect(ctx context.Context, volumeID string) (volume.Volume, error) { + return m.InjectVolumeInspect(ctx, volumeID) +} + +func (m *MockContainerClient) VolumeRemove(ctx context.Context, volumeID string, force bool) (error) { + return m.InjectVolumeRemove(ctx, volumeID, force) +}