add document imports
Some checks are pending
Lint / golangci-lint (push) Waiting to run
Declarative Tests / test (push) Waiting to run
Declarative Tests / build-fedora (push) Waiting to run
Declarative Tests / build-ubuntu-focal (push) Waiting to run

This commit is contained in:
Matthew Rich 2024-09-24 19:22:49 +00:00
parent 38f8831275
commit 55fd39f09d
8 changed files with 244 additions and 9 deletions

View File

@ -25,6 +25,7 @@ type DocumentType struct {
URI URI `json:"source,omitempty" yaml:"source,omitempty"` URI URI `json:"source,omitempty" yaml:"source,omitempty"`
Format codec.Format `json:"format,omitempty" yaml:"format,omitempty"` Format codec.Format `json:"format,omitempty" yaml:"format,omitempty"`
Requires Dependencies `json:"requires,omitempty" yaml:"requires,omitempty"` Requires Dependencies `json:"requires,omitempty" yaml:"requires,omitempty"`
Imports []URI `json:"imports,omitempty" yaml:"imports,omitempty"`
} }
type Document struct { type Document struct {
@ -32,6 +33,7 @@ type Document struct {
URI URI `json:"source,omitempty" yaml:"source,omitempty"` URI URI `json:"source,omitempty" yaml:"source,omitempty"`
Format codec.Format `json:"format,omitempty" yaml:"format,omitempty"` Format codec.Format `json:"format,omitempty" yaml:"format,omitempty"`
Requires Dependencies `json:"requires,omitempty" yaml:"requires,omitempty"` Requires Dependencies `json:"requires,omitempty" yaml:"requires,omitempty"`
Imports []URI `json:"imports,omitempty" yaml:"imports,omitempty"`
uris mapper.Store[string, data.Declaration] uris mapper.Store[string, data.Declaration]
ResourceDeclarations []*Declaration `json:"resources,omitempty" yaml:"resources,omitempty"` ResourceDeclarations []*Declaration `json:"resources,omitempty" yaml:"resources,omitempty"`
configNames mapper.Store[string, data.Block] `json:"-" yaml:"-"` configNames mapper.Store[string, data.Block] `json:"-" yaml:"-"`
@ -48,6 +50,14 @@ func NewDocument(r *Registry) *Document {
return &Document{ Registry: r, Format: codec.FormatYaml, uris: mapper.New[string, data.Declaration](), configNames: mapper.New[string, data.Block]() } return &Document{ Registry: r, Format: codec.FormatYaml, uris: mapper.New[string, data.Declaration](), configNames: mapper.New[string, data.Block]() }
} }
func (d *Document) GetURI() string {
return string(d.URI)
}
func (d *Document) SetURI(uri string) {
d.URI = URI(uri)
}
func (d *Document) Types() data.TypesRegistry[data.Resource] { func (d *Document) Types() data.TypesRegistry[data.Resource] {
return d.Registry.ResourceTypes return d.Registry.ResourceTypes
} }
@ -116,6 +126,27 @@ func (d *Document) Clone() data.Document {
return clone return clone
} }
func (d *Document) ImportedDocuments() (documents []data.Document) {
documents = make([]data.Document, 0, len(d.Imports))
for _, uri := range d.Imports {
if doc, ok := DocumentRegistry.GetDocument(uri); ok {
documents = append(documents, doc)
}
}
return
}
func (d *Document) loadImports() (err error) {
for _, uri := range d.Imports {
if ! DocumentRegistry.HasDocument(uri) {
if _, err = DocumentRegistry.Load(uri); err != nil {
return
}
}
}
return
}
func (d *Document) assignResourcesDocument() { func (d *Document) assignResourcesDocument() {
slog.Info("Document.assignResourcesDocument()", "declarations", d.ResourceDeclarations, "len", len(d.ResourceDeclarations)) slog.Info("Document.assignResourcesDocument()", "declarations", d.ResourceDeclarations, "len", len(d.ResourceDeclarations))
for i := range d.ResourceDeclarations { for i := range d.ResourceDeclarations {
@ -457,6 +488,16 @@ func (d *Document) AppendConfigurations(docs []data.Document) {
} }
} }
// Generate a diff of the loaded document against the current resource state
func (d *Document) DiffState(output io.Writer) (returnOutput string, diffErr error) {
clone := d.Clone()
diffErr = clone.Apply("read")
if diffErr != nil {
return "", diffErr
}
return d.Diff(clone, output)
}
func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput string, diffErr error) { func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput string, diffErr error) {
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
@ -501,6 +542,12 @@ func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput stri
return "", nil return "", nil
} }
/*
func (d *Document) UnmarshalValue(value *DocumentType) error {
d.Requires = value.Requires
}
*/
func (d *Document) UnmarshalYAML(value *yaml.Node) error { func (d *Document) UnmarshalYAML(value *yaml.Node) error {
type decodeDocument Document type decodeDocument Document
t := &DocumentType{} t := &DocumentType{}
@ -513,6 +560,7 @@ func (d *Document) UnmarshalYAML(value *yaml.Node) error {
} }
d.assignConfigurationsDocument() d.assignConfigurationsDocument()
d.assignResourcesDocument() d.assignResourcesDocument()
d.loadImports()
return nil return nil
} }
@ -524,6 +572,7 @@ func (d *Document) UnmarshalJSON(data []byte) error {
} }
d.assignConfigurationsDocument() d.assignConfigurationsDocument()
d.assignResourcesDocument() d.assignResourcesDocument()
d.loadImports()
return nil return nil
} }

View File

@ -18,6 +18,7 @@ import (
var ( var (
TestResourceTypes *types.Types[data.Resource] = types.New[data.Resource]() TestResourceTypes *types.Types[data.Resource] = types.New[data.Resource]()
TestConfigurationTypes *types.Types[data.Configuration] = types.New[data.Configuration]() TestConfigurationTypes *types.Types[data.Configuration] = types.New[data.Configuration]()
TestConverterTypes *types.Types[data.Converter] = types.New[data.Converter]()
) )
func TestNewDocument(t *testing.T) { func TestNewDocument(t *testing.T) {
@ -36,7 +37,7 @@ func TestDocumentLoader(t *testing.T) {
DocumentRegistry.ResourceTypes = TestResourceTypes DocumentRegistry.ResourceTypes = TestResourceTypes
slog.Info("TestDocumentLoader", "rt", TestResourceTypes) slog.Info("TestDocumentLoader", "rt", TestResourceTypes)
file, _ := filepath.Abs(filepath.Join(TempDir, "foo.txt")) file, _ := filepath.Abs(TempDir.FilePath("foo.txt"))
document := fmt.Sprintf(` document := fmt.Sprintf(`
--- ---
@ -207,3 +208,69 @@ resources:
}) })
assert.Equal(t, 1, len(resources)) assert.Equal(t, 1, len(resources))
} }
func TestDocumentImports(t *testing.T) {
DocumentRegistry.ResourceTypes = TestResourceTypes
DocumentRegistry.ConverterTypes = TestConverterTypes
cycleDocPath := TempDir.FilePath("cycle.jx.yaml")
srcDoc := fmt.Sprintf(`
imports:
- %s
resources:
- type: testuser
attributes:
name: "testuser"
uid: "10022"
home: "/home/testuser"
`, cycleDocPath)
srcDocPath, err := TempDir.CreateFileFromReader("src.jx.yaml", strings.NewReader(srcDoc))
assert.Nil(t, err)
srcDocPathURI := fmt.Sprintf("file://%s", srcDocPath)
document := fmt.Sprintf(`
---
imports:
- %s
resources:
- type: foo
attributes:
name: "foo.txt"
- type: bar
attributes:
name: "bar.txt"
`, srcDocPathURI)
docPath, err := TempDir.CreateFileFromReader("doc.jx.yaml", strings.NewReader(document))
assert.Nil(t, err)
docPathURI := fmt.Sprintf("file://%s", docPath)
cycle := fmt.Sprintf(`
---
imports:
- %s
- %s
resources:
- type: foo
attributes:
name: "foo2.txt"
`, srcDocPathURI, docPathURI)
_, err = TempDir.CreateFileFromReader("cycle.jx.yaml", strings.NewReader(cycle))
assert.Nil(t, err)
cycleDocPathURI := fmt.Sprintf("file://%s", cycleDocPath)
_, loadErr := DocumentRegistry.Load(URI(docPathURI))
assert.Nil(t, loadErr)
assert.True(t, DocumentRegistry.HasDocument(URI(srcDocPathURI)))
assert.True(t, DocumentRegistry.HasDocument(URI(cycleDocPathURI)))
}

View File

@ -11,16 +11,17 @@ _ "github.com/stretchr/testify/assert"
"os/exec" "os/exec"
_ "path/filepath" _ "path/filepath"
"testing" "testing"
"decl/internal/tempdir"
) )
var TempDir string var TempDir tempdir.Path = "testfolio"
var ProcessTestUserName string var ProcessTestUserName string
var ProcessTestGroupName string var ProcessTestGroupName string
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
var err error var err error
TempDir, err = os.MkdirTemp("", "testfolio") err = TempDir.Create()
if err != nil || TempDir == "" { if err != nil || TempDir == "" {
log.Fatal(err) log.Fatal(err)
} }
@ -29,10 +30,11 @@ func TestMain(m *testing.M) {
RegisterMocks() RegisterMocks()
RegisterConfigurationMocks() RegisterConfigurationMocks()
RegisterConverterMocks()
rc := m.Run() rc := m.Run()
os.RemoveAll(TempDir) TempDir.Remove()
os.Exit(rc) os.Exit(rc)
} }

View File

@ -0,0 +1,38 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package folio
import (
"encoding/json"
"decl/internal/data"
)
type MockConverter struct {
InjectType func() data.TypeName `json:"-" yaml:"-"`
InjectEmit func(data.Document, data.ElementSelector) (data.Resource, error) `json:"-" yaml:"-"`
InjectExtract func(data.Resource, data.ElementSelector) (data.Document, error) `json:"-" yaml:"-"`
InjectClose func() (error) `json:"-" yaml:"-"`
}
func (m *MockConverter) Type() data.TypeName {
return m.InjectType()
}
func (m *MockConverter) Emit(document data.Document, filter data.ElementSelector) (data.Resource, error) {
return m.InjectEmit(document, filter)
}
func (m *MockConverter) Extract(sourceResource data.Resource, filter data.ElementSelector) (data.Document, error) {
return m.InjectExtract(sourceResource, filter)
}
func (m *MockConverter) Close() error {
return m.InjectClose()
}
func (m *MockConverter) UnmarshalJSON(data []byte) error {
if err := json.Unmarshal(data, m); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,52 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package folio
import (
"decl/internal/data"
"decl/internal/codec"
"path/filepath"
"net/url"
)
func RegisterConverterMocks() {
TestConverterTypes.Register([]string{"file"}, func(u *url.URL) data.Converter {
var uri URI
uri.SetURL(u)
f := NewMockFileConverter(uri)
f.Name = filepath.Join(u.Hostname(), u.Path)
return f
})
}
type MockFileConverter struct {
*MockConverter `json:"-" yaml:"-"`
Name string `json:"name" yaml:"name"`
}
func NewMockConverter(typename string, uri URI) *MockConverter {
return &MockConverter {
InjectType: func() data.TypeName { return data.TypeName(typename) },
InjectEmit: func(document data.Document, filter data.ElementSelector) (res data.Resource, err error) {
return
},
InjectExtract: func(res data.Resource, filter data.ElementSelector) (doc data.Document, err error) {
doc = DocumentRegistry.NewDocument(uri)
if r, readErr := uri.ContentReaderStream(); readErr != nil {
return doc, readErr
} else {
err = doc.LoadReader(r, codec.FormatYaml)
defer r.Close()
}
return
},
InjectClose: func() error { return nil },
}
}
func NewMockFileConverter(uri URI) *MockFileConverter {
f := &MockFileConverter {}
f.MockConverter = NewMockConverter("file", uri)
return f
}

View File

@ -30,6 +30,11 @@ func RegisterMocks() {
f.Name = filepath.Join(u.Hostname(), u.Path) f.Name = filepath.Join(u.Hostname(), u.Path)
return f return f
}) })
TestResourceTypes.Register([]string{"file"}, func(u *url.URL) data.Resource {
f := NewFileResource()
f.Name = filepath.Join(u.Hostname(), u.Path)
return f
})
} }
@ -57,6 +62,13 @@ type MockTestuser struct {
Home string `json:"home" yaml:"home"` Home string `json:"home" yaml:"home"`
} }
type MockFile struct {
stater machine.Stater `json:"-" yaml:"-"`
*MockResource `json:"-" yaml:"-"`
Name string `json:"name" yaml:"name"`
Size int `json:"size" yaml:"size"`
}
func NewMockResource(typename string, stater machine.Stater) *MockResource { func NewMockResource(typename string, stater machine.Stater) *MockResource {
return &MockResource { return &MockResource {
InjectType: func() string { return typename }, InjectType: func() string { return typename },
@ -99,3 +111,10 @@ func NewTestuserResource() *MockTestuser {
u.MockResource = NewMockResource("testuser", u.stater) u.MockResource = NewMockResource("testuser", u.stater)
return u return u
} }
func NewFileResource() *MockFile {
f := &MockFile {}
f.stater = data.StorageMachine(f)
f.MockResource = NewMockResource("file", f.stater)
return f
}

View File

@ -13,14 +13,14 @@ import (
func TestResourceReference(t *testing.T) { func TestResourceReference(t *testing.T) {
f := NewFooResource() f := NewFooResource()
resourceMapper := mapper.New[string, data.Declaration]() resourceMapper := mapper.New[string, data.Declaration]()
f.Name = TempDir f.Name = string(TempDir)
f.Size = 10 f.Size = 10
d := NewDeclaration() d := NewDeclaration()
d.Type = "foo" d.Type = "foo"
d.Attributes = f d.Attributes = f
resourceMapper[d.URI()] = d resourceMapper[d.URI()] = d
var foo ResourceReference = ResourceReference(fmt.Sprintf("foo://%s", TempDir)) var foo ResourceReference = ResourceReference(fmt.Sprintf("foo://%s", string(TempDir)))
u := foo.Parse() u := foo.Parse()
assert.Equal(t, "foo", u.Scheme) assert.Equal(t, "foo", u.Scheme)
assert.True(t, foo.Exists()) assert.True(t, foo.Exists())

View File

@ -15,6 +15,14 @@
"requires": { "requires": {
"$ref": "dependencies.schema.json" "$ref": "dependencies.schema.json"
}, },
"imports": {
"type": "array",
"description": "List of other documents to import",
"items": {
"type": "string",
"description": "Document URI"
}
},
"configurations": { "configurations": {
"type": "array", "type": "array",
"description": "Configurations list", "description": "Configurations list",