From 55fd39f09d02b0aaf2de2f8d6cc5d59cda84eb5d Mon Sep 17 00:00:00 2001 From: Matthew Rich Date: Tue, 24 Sep 2024 19:22:49 +0000 Subject: [PATCH] add document imports --- internal/folio/document.go | 49 +++++++++++++++ internal/folio/document_test.go | 69 ++++++++++++++++++++- internal/folio/folio_test.go | 8 ++- internal/folio/mock_converter_test.go | 38 ++++++++++++ internal/folio/mock_file_converter_test.go | 52 ++++++++++++++++ internal/folio/mock_foo_resource_test.go | 25 +++++++- internal/folio/resourcereference_test.go | 4 +- internal/folio/schemas/document.schema.json | 8 +++ 8 files changed, 244 insertions(+), 9 deletions(-) create mode 100644 internal/folio/mock_converter_test.go create mode 100644 internal/folio/mock_file_converter_test.go diff --git a/internal/folio/document.go b/internal/folio/document.go index bfd5525..1b8e0dd 100644 --- a/internal/folio/document.go +++ b/internal/folio/document.go @@ -25,6 +25,7 @@ type DocumentType struct { URI URI `json:"source,omitempty" yaml:"source,omitempty"` Format codec.Format `json:"format,omitempty" yaml:"format,omitempty"` Requires Dependencies `json:"requires,omitempty" yaml:"requires,omitempty"` + Imports []URI `json:"imports,omitempty" yaml:"imports,omitempty"` } type Document struct { @@ -32,6 +33,7 @@ type Document struct { URI URI `json:"source,omitempty" yaml:"source,omitempty"` Format codec.Format `json:"format,omitempty" yaml:"format,omitempty"` Requires Dependencies `json:"requires,omitempty" yaml:"requires,omitempty"` + Imports []URI `json:"imports,omitempty" yaml:"imports,omitempty"` uris mapper.Store[string, data.Declaration] ResourceDeclarations []*Declaration `json:"resources,omitempty" yaml:"resources,omitempty"` configNames mapper.Store[string, data.Block] `json:"-" yaml:"-"` @@ -48,6 +50,14 @@ func NewDocument(r *Registry) *Document { return &Document{ Registry: r, Format: codec.FormatYaml, uris: mapper.New[string, data.Declaration](), configNames: mapper.New[string, data.Block]() } } +func (d *Document) GetURI() string { + return string(d.URI) +} + +func (d *Document) SetURI(uri string) { + d.URI = URI(uri) +} + func (d *Document) Types() data.TypesRegistry[data.Resource] { return d.Registry.ResourceTypes } @@ -116,6 +126,27 @@ func (d *Document) Clone() data.Document { return clone } +func (d *Document) ImportedDocuments() (documents []data.Document) { + documents = make([]data.Document, 0, len(d.Imports)) + for _, uri := range d.Imports { + if doc, ok := DocumentRegistry.GetDocument(uri); ok { + documents = append(documents, doc) + } + } + return +} + +func (d *Document) loadImports() (err error) { + for _, uri := range d.Imports { + if ! DocumentRegistry.HasDocument(uri) { + if _, err = DocumentRegistry.Load(uri); err != nil { + return + } + } + } + return +} + func (d *Document) assignResourcesDocument() { slog.Info("Document.assignResourcesDocument()", "declarations", d.ResourceDeclarations, "len", len(d.ResourceDeclarations)) for i := range d.ResourceDeclarations { @@ -457,6 +488,16 @@ func (d *Document) AppendConfigurations(docs []data.Document) { } } +// Generate a diff of the loaded document against the current resource state +func (d *Document) DiffState(output io.Writer) (returnOutput string, diffErr error) { + clone := d.Clone() + diffErr = clone.Apply("read") + if diffErr != nil { + return "", diffErr + } + return d.Diff(clone, output) +} + func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput string, diffErr error) { defer func() { if r := recover(); r != nil { @@ -501,6 +542,12 @@ func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput stri return "", nil } +/* +func (d *Document) UnmarshalValue(value *DocumentType) error { + d.Requires = value.Requires +} +*/ + func (d *Document) UnmarshalYAML(value *yaml.Node) error { type decodeDocument Document t := &DocumentType{} @@ -513,6 +560,7 @@ func (d *Document) UnmarshalYAML(value *yaml.Node) error { } d.assignConfigurationsDocument() d.assignResourcesDocument() + d.loadImports() return nil } @@ -524,6 +572,7 @@ func (d *Document) UnmarshalJSON(data []byte) error { } d.assignConfigurationsDocument() d.assignResourcesDocument() + d.loadImports() return nil } diff --git a/internal/folio/document_test.go b/internal/folio/document_test.go index ffc676d..5a5d1cf 100644 --- a/internal/folio/document_test.go +++ b/internal/folio/document_test.go @@ -18,6 +18,7 @@ import ( var ( TestResourceTypes *types.Types[data.Resource] = types.New[data.Resource]() TestConfigurationTypes *types.Types[data.Configuration] = types.New[data.Configuration]() + TestConverterTypes *types.Types[data.Converter] = types.New[data.Converter]() ) func TestNewDocument(t *testing.T) { @@ -36,7 +37,7 @@ func TestDocumentLoader(t *testing.T) { DocumentRegistry.ResourceTypes = TestResourceTypes slog.Info("TestDocumentLoader", "rt", TestResourceTypes) - file, _ := filepath.Abs(filepath.Join(TempDir, "foo.txt")) + file, _ := filepath.Abs(TempDir.FilePath("foo.txt")) document := fmt.Sprintf(` --- @@ -207,3 +208,69 @@ resources: }) assert.Equal(t, 1, len(resources)) } + +func TestDocumentImports(t *testing.T) { + DocumentRegistry.ResourceTypes = TestResourceTypes + DocumentRegistry.ConverterTypes = TestConverterTypes + + cycleDocPath := TempDir.FilePath("cycle.jx.yaml") + + srcDoc := fmt.Sprintf(` +imports: +- %s +resources: +- type: testuser + attributes: + name: "testuser" + uid: "10022" + home: "/home/testuser" +`, cycleDocPath) + + srcDocPath, err := TempDir.CreateFileFromReader("src.jx.yaml", strings.NewReader(srcDoc)) + assert.Nil(t, err) + srcDocPathURI := fmt.Sprintf("file://%s", srcDocPath) + + + document := fmt.Sprintf(` +--- +imports: +- %s +resources: +- type: foo + attributes: + name: "foo.txt" +- type: bar + attributes: + name: "bar.txt" +`, srcDocPathURI) + + + docPath, err := TempDir.CreateFileFromReader("doc.jx.yaml", strings.NewReader(document)) + assert.Nil(t, err) + docPathURI := fmt.Sprintf("file://%s", docPath) + + cycle := fmt.Sprintf(` +--- +imports: +- %s +- %s +resources: +- type: foo + attributes: + name: "foo2.txt" +`, srcDocPathURI, docPathURI) + + _, err = TempDir.CreateFileFromReader("cycle.jx.yaml", strings.NewReader(cycle)) + assert.Nil(t, err) + cycleDocPathURI := fmt.Sprintf("file://%s", cycleDocPath) + + + _, loadErr := DocumentRegistry.Load(URI(docPathURI)) + assert.Nil(t, loadErr) + assert.True(t, DocumentRegistry.HasDocument(URI(srcDocPathURI))) + + assert.True(t, DocumentRegistry.HasDocument(URI(cycleDocPathURI))) + +} + + diff --git a/internal/folio/folio_test.go b/internal/folio/folio_test.go index ed7f7ff..034b4dc 100644 --- a/internal/folio/folio_test.go +++ b/internal/folio/folio_test.go @@ -11,16 +11,17 @@ _ "github.com/stretchr/testify/assert" "os/exec" _ "path/filepath" "testing" + "decl/internal/tempdir" ) -var TempDir string +var TempDir tempdir.Path = "testfolio" var ProcessTestUserName string var ProcessTestGroupName string func TestMain(m *testing.M) { var err error - TempDir, err = os.MkdirTemp("", "testfolio") + err = TempDir.Create() if err != nil || TempDir == "" { log.Fatal(err) } @@ -29,10 +30,11 @@ func TestMain(m *testing.M) { RegisterMocks() RegisterConfigurationMocks() + RegisterConverterMocks() rc := m.Run() - os.RemoveAll(TempDir) + TempDir.Remove() os.Exit(rc) } diff --git a/internal/folio/mock_converter_test.go b/internal/folio/mock_converter_test.go new file mode 100644 index 0000000..148bf78 --- /dev/null +++ b/internal/folio/mock_converter_test.go @@ -0,0 +1,38 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package folio + +import ( + "encoding/json" + "decl/internal/data" +) + +type MockConverter struct { + InjectType func() data.TypeName `json:"-" yaml:"-"` + InjectEmit func(data.Document, data.ElementSelector) (data.Resource, error) `json:"-" yaml:"-"` + InjectExtract func(data.Resource, data.ElementSelector) (data.Document, error) `json:"-" yaml:"-"` + InjectClose func() (error) `json:"-" yaml:"-"` +} + +func (m *MockConverter) Type() data.TypeName { + return m.InjectType() +} + +func (m *MockConverter) Emit(document data.Document, filter data.ElementSelector) (data.Resource, error) { + return m.InjectEmit(document, filter) +} + +func (m *MockConverter) Extract(sourceResource data.Resource, filter data.ElementSelector) (data.Document, error) { + return m.InjectExtract(sourceResource, filter) +} + +func (m *MockConverter) Close() error { + return m.InjectClose() +} + +func (m *MockConverter) UnmarshalJSON(data []byte) error { + if err := json.Unmarshal(data, m); err != nil { + return err + } + return nil +} diff --git a/internal/folio/mock_file_converter_test.go b/internal/folio/mock_file_converter_test.go new file mode 100644 index 0000000..a90692a --- /dev/null +++ b/internal/folio/mock_file_converter_test.go @@ -0,0 +1,52 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package folio + +import ( + "decl/internal/data" + "decl/internal/codec" + "path/filepath" + "net/url" +) + +func RegisterConverterMocks() { + TestConverterTypes.Register([]string{"file"}, func(u *url.URL) data.Converter { + var uri URI + uri.SetURL(u) + f := NewMockFileConverter(uri) + f.Name = filepath.Join(u.Hostname(), u.Path) + return f + }) +} + + +type MockFileConverter struct { + *MockConverter `json:"-" yaml:"-"` + Name string `json:"name" yaml:"name"` +} + +func NewMockConverter(typename string, uri URI) *MockConverter { + return &MockConverter { + InjectType: func() data.TypeName { return data.TypeName(typename) }, + InjectEmit: func(document data.Document, filter data.ElementSelector) (res data.Resource, err error) { + return + }, + InjectExtract: func(res data.Resource, filter data.ElementSelector) (doc data.Document, err error) { + doc = DocumentRegistry.NewDocument(uri) + if r, readErr := uri.ContentReaderStream(); readErr != nil { + return doc, readErr + } else { + err = doc.LoadReader(r, codec.FormatYaml) + defer r.Close() + } + return + }, + InjectClose: func() error { return nil }, + } +} + +func NewMockFileConverter(uri URI) *MockFileConverter { + f := &MockFileConverter {} + f.MockConverter = NewMockConverter("file", uri) + return f +} diff --git a/internal/folio/mock_foo_resource_test.go b/internal/folio/mock_foo_resource_test.go index a3ad24f..e3efdc9 100644 --- a/internal/folio/mock_foo_resource_test.go +++ b/internal/folio/mock_foo_resource_test.go @@ -30,6 +30,11 @@ func RegisterMocks() { f.Name = filepath.Join(u.Hostname(), u.Path) return f }) + TestResourceTypes.Register([]string{"file"}, func(u *url.URL) data.Resource { + f := NewFileResource() + f.Name = filepath.Join(u.Hostname(), u.Path) + return f + }) } @@ -57,6 +62,13 @@ type MockTestuser struct { Home string `json:"home" yaml:"home"` } +type MockFile struct { + stater machine.Stater `json:"-" yaml:"-"` + *MockResource `json:"-" yaml:"-"` + Name string `json:"name" yaml:"name"` + Size int `json:"size" yaml:"size"` +} + func NewMockResource(typename string, stater machine.Stater) *MockResource { return &MockResource { InjectType: func() string { return typename }, @@ -80,22 +92,29 @@ func NewMockResource(typename string, stater machine.Stater) *MockResource { } func NewFooResource() *MockFoo { - f := &MockFoo {} + f := &MockFoo {} f.stater = data.StorageMachine(f) f.MockResource = NewMockResource("foo", f.stater) return f } func NewBarResource() *MockBar { - b := &MockBar {} + b := &MockBar {} b.stater = data.StorageMachine(b) b.MockResource = NewMockResource("bar", b.stater) return b } func NewTestuserResource() *MockTestuser { - u := &MockTestuser {} + u := &MockTestuser {} u.stater = data.StorageMachine(u) u.MockResource = NewMockResource("testuser", u.stater) return u } + +func NewFileResource() *MockFile { + f := &MockFile {} + f.stater = data.StorageMachine(f) + f.MockResource = NewMockResource("file", f.stater) + return f +} diff --git a/internal/folio/resourcereference_test.go b/internal/folio/resourcereference_test.go index b38b8a5..012b0d3 100644 --- a/internal/folio/resourcereference_test.go +++ b/internal/folio/resourcereference_test.go @@ -13,14 +13,14 @@ import ( func TestResourceReference(t *testing.T) { f := NewFooResource() resourceMapper := mapper.New[string, data.Declaration]() - f.Name = TempDir + f.Name = string(TempDir) f.Size = 10 d := NewDeclaration() d.Type = "foo" d.Attributes = f resourceMapper[d.URI()] = d - var foo ResourceReference = ResourceReference(fmt.Sprintf("foo://%s", TempDir)) + var foo ResourceReference = ResourceReference(fmt.Sprintf("foo://%s", string(TempDir))) u := foo.Parse() assert.Equal(t, "foo", u.Scheme) assert.True(t, foo.Exists()) diff --git a/internal/folio/schemas/document.schema.json b/internal/folio/schemas/document.schema.json index 75578c3..7db6600 100644 --- a/internal/folio/schemas/document.schema.json +++ b/internal/folio/schemas/document.schema.json @@ -15,6 +15,14 @@ "requires": { "$ref": "dependencies.schema.json" }, + "imports": { + "type": "array", + "description": "List of other documents to import", + "items": { + "type": "string", + "description": "Document URI" + } + }, "configurations": { "type": "array", "description": "Configurations list",