add document imports

internal/folio/document.go

@@ -25,6 +25,7 @@ type DocumentType struct {
 	URI URI `json:"source,omitempty" yaml:"source,omitempty"`
 	Format codec.Format `json:"format,omitempty" yaml:"format,omitempty"`
 	Requires Dependencies `json:"requires,omitempty" yaml:"requires,omitempty"`
+	Imports []URI `json:"imports,omitempty" yaml:"imports,omitempty"`
 }
 
 type Document struct {
@@ -32,6 +33,7 @@ type Document struct {
 	URI URI `json:"source,omitempty" yaml:"source,omitempty"`
 	Format codec.Format `json:"format,omitempty" yaml:"format,omitempty"`
 	Requires Dependencies `json:"requires,omitempty" yaml:"requires,omitempty"`
+	Imports []URI `json:"imports,omitempty" yaml:"imports,omitempty"`
 	uris mapper.Store[string, data.Declaration]
 	ResourceDeclarations []*Declaration `json:"resources,omitempty" yaml:"resources,omitempty"`
 	configNames mapper.Store[string, data.Block] `json:"-" yaml:"-"`
@@ -48,6 +50,14 @@ func NewDocument(r *Registry) *Document {
 	return &Document{ Registry: r, Format: codec.FormatYaml, uris: mapper.New[string, data.Declaration](), configNames: mapper.New[string, data.Block]() }
 }
 
+func (d *Document) GetURI() string {
+	return string(d.URI)
+}
+
+func (d *Document) SetURI(uri string) {
+	d.URI = URI(uri)
+}
+
 func (d *Document) Types() data.TypesRegistry[data.Resource] {
 	return d.Registry.ResourceTypes
 }
@@ -116,6 +126,27 @@ func (d *Document) Clone() data.Document {
 	return clone
 }
 
+func (d *Document) ImportedDocuments() (documents []data.Document) {
+	documents = make([]data.Document, 0, len(d.Imports))
+	for _, uri := range d.Imports {
+		if doc, ok := DocumentRegistry.GetDocument(uri); ok {
+			documents = append(documents, doc)
+		}
+	}
+	return
+}
+
+func (d *Document) loadImports() (err error) {
+	for _, uri := range d.Imports {
+		if ! DocumentRegistry.HasDocument(uri) {
+			if _, err = DocumentRegistry.Load(uri); err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
 func (d *Document) assignResourcesDocument() {
 	slog.Info("Document.assignResourcesDocument()", "declarations", d.ResourceDeclarations, "len", len(d.ResourceDeclarations))
 	for i := range d.ResourceDeclarations {
@@ -457,6 +488,16 @@ func (d *Document) AppendConfigurations(docs []data.Document) {
 	}
 }
 
+// Generate a diff of the loaded document against the current resource state
+func (d *Document) DiffState(output io.Writer) (returnOutput string, diffErr error) {
+	clone := d.Clone()
+	diffErr = clone.Apply("read")
+	if diffErr != nil {
+		return "", diffErr
+	}
+	return d.Diff(clone, output)
+}
+
 func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput string, diffErr error) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -501,6 +542,12 @@ func (d *Document) Diff(with data.Document, output io.Writer) (returnOutput stri
 	return "", nil
 }
 
+/*
+func (d *Document) UnmarshalValue(value *DocumentType) error {
+	d.Requires = value.Requires
+}
+*/
+
 func (d *Document) UnmarshalYAML(value *yaml.Node) error {
 	type decodeDocument Document
 	t := &DocumentType{}
@@ -513,6 +560,7 @@ func (d *Document) UnmarshalYAML(value *yaml.Node) error {
 	}
 	d.assignConfigurationsDocument()
 	d.assignResourcesDocument()
+	d.loadImports()
 	return nil
 }
 
@@ -524,6 +572,7 @@ func (d *Document) UnmarshalJSON(data []byte) error {
 	}
 	d.assignConfigurationsDocument()
 	d.assignResourcesDocument()
+	d.loadImports()
 	return nil
 }
 

internal/folio/document_test.go

@@ -18,6 +18,7 @@ import (
 var (
 	TestResourceTypes *types.Types[data.Resource] = types.New[data.Resource]()
 	TestConfigurationTypes *types.Types[data.Configuration] = types.New[data.Configuration]()
+	TestConverterTypes *types.Types[data.Converter] = types.New[data.Converter]()
 )
 
 func TestNewDocument(t *testing.T) {
@@ -36,7 +37,7 @@ func TestDocumentLoader(t *testing.T) {
 	DocumentRegistry.ResourceTypes = TestResourceTypes
 	slog.Info("TestDocumentLoader", "rt", TestResourceTypes)
 
-	file, _ := filepath.Abs(filepath.Join(TempDir, "foo.txt"))
+	file, _ := filepath.Abs(TempDir.FilePath("foo.txt"))
 
 	document := fmt.Sprintf(`
 ---
@@ -207,3 +208,69 @@ resources:
 	})
 	assert.Equal(t, 1, len(resources))
 }
+
+func TestDocumentImports(t *testing.T) {
+	DocumentRegistry.ResourceTypes = TestResourceTypes
+	DocumentRegistry.ConverterTypes = TestConverterTypes
+
+	cycleDocPath := TempDir.FilePath("cycle.jx.yaml")
+
+	srcDoc := fmt.Sprintf(`
+imports:
+- %s
+resources:
+- type: testuser
+  attributes:
+    name: "testuser"
+    uid: "10022"
+    home: "/home/testuser"
+`, cycleDocPath)
+
+	srcDocPath, err := TempDir.CreateFileFromReader("src.jx.yaml", strings.NewReader(srcDoc))
+	assert.Nil(t, err)
+	srcDocPathURI := fmt.Sprintf("file://%s", srcDocPath)
+
+
+	document := fmt.Sprintf(`
+---
+imports:
+- %s
+resources:
+- type: foo
+  attributes:
+    name: "foo.txt"
+- type: bar
+  attributes:
+    name: "bar.txt"
+`, srcDocPathURI)
+	
+
+	docPath, err := TempDir.CreateFileFromReader("doc.jx.yaml", strings.NewReader(document))
+	assert.Nil(t, err)
+	docPathURI := fmt.Sprintf("file://%s", docPath)
+
+	cycle := fmt.Sprintf(`
+---
+imports:
+- %s
+- %s
+resources:
+- type: foo
+  attributes:
+    name: "foo2.txt"
+`, srcDocPathURI, docPathURI)
+
+	_, err = TempDir.CreateFileFromReader("cycle.jx.yaml", strings.NewReader(cycle))
+	assert.Nil(t, err)
+	cycleDocPathURI := fmt.Sprintf("file://%s", cycleDocPath)
+
+	
+	_, loadErr := DocumentRegistry.Load(URI(docPathURI))
+	assert.Nil(t, loadErr)
+	assert.True(t, DocumentRegistry.HasDocument(URI(srcDocPathURI)))
+
+	assert.True(t, DocumentRegistry.HasDocument(URI(cycleDocPathURI)))
+
+}
+
+

internal/folio/folio_test.go

@@ -11,16 +11,17 @@ _	"github.com/stretchr/testify/assert"
 	"os/exec"
 _	"path/filepath"
 	"testing"
+	"decl/internal/tempdir"
 )
 
-var TempDir string
+var TempDir tempdir.Path = "testfolio"
 
 var ProcessTestUserName string
 var ProcessTestGroupName string
 
 func TestMain(m *testing.M) {
 	var err error
-	TempDir, err = os.MkdirTemp("", "testfolio")
+	err = TempDir.Create()
 	if err != nil || TempDir == "" {
 		log.Fatal(err)
 	}
@@ -29,10 +30,11 @@ func TestMain(m *testing.M) {
 
 	RegisterMocks()
 	RegisterConfigurationMocks()
+	RegisterConverterMocks()
 
 	rc := m.Run()
 
-	os.RemoveAll(TempDir)
+	TempDir.Remove()
 	os.Exit(rc)
 }
 

internal/folio/mock_converter_test.go

@@ -0,0 +1,38 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package folio
+
+import (
+	"encoding/json"
+	"decl/internal/data"
+)
+
+type MockConverter struct {
+	InjectType func() data.TypeName `json:"-" yaml:"-"`
+	InjectEmit func(data.Document, data.ElementSelector) (data.Resource, error) `json:"-" yaml:"-"`
+	InjectExtract func(data.Resource, data.ElementSelector) (data.Document, error) `json:"-" yaml:"-"`
+	InjectClose func() (error) `json:"-" yaml:"-"`
+}
+
+func (m *MockConverter) Type() data.TypeName {
+  return m.InjectType()
+}
+
+func (m *MockConverter)	Emit(document data.Document, filter data.ElementSelector) (data.Resource, error) {
+	return m.InjectEmit(document, filter)
+}
+
+func (m *MockConverter)	Extract(sourceResource data.Resource, filter data.ElementSelector) (data.Document, error) {
+	return m.InjectExtract(sourceResource, filter)
+}
+
+func (m *MockConverter)	Close() error {
+	return m.InjectClose()
+}
+
+func (m *MockConverter) UnmarshalJSON(data []byte) error {
+  if err := json.Unmarshal(data, m); err != nil {
+    return err
+  }
+  return nil
+}

internal/folio/mock_file_converter_test.go

@@ -0,0 +1,52 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package folio
+
+import (
+	"decl/internal/data"
+	"decl/internal/codec"
+	"path/filepath"
+	"net/url"
+)
+
+func RegisterConverterMocks() {
+	TestConverterTypes.Register([]string{"file"}, func(u *url.URL) data.Converter {
+		var uri URI
+		uri.SetURL(u)
+		f := NewMockFileConverter(uri)
+		f.Name = filepath.Join(u.Hostname(), u.Path)
+		return f
+	})
+}
+
+
+type MockFileConverter struct {
+	*MockConverter `json:"-" yaml:"-"`
+	Name string `json:"name" yaml:"name"`
+}
+
+func NewMockConverter(typename string, uri URI) *MockConverter {
+	return &MockConverter {
+		InjectType: func() data.TypeName { return data.TypeName(typename) },
+		InjectEmit: func(document data.Document, filter data.ElementSelector) (res data.Resource, err error) {
+			return
+		},
+		InjectExtract: func(res data.Resource, filter data.ElementSelector) (doc data.Document, err error) {
+			doc = DocumentRegistry.NewDocument(uri)
+			if r, readErr := uri.ContentReaderStream(); readErr != nil {
+				return doc, readErr
+			} else {
+				err = doc.LoadReader(r, codec.FormatYaml)
+				defer r.Close()
+			}
+			return
+		},
+		InjectClose: func() error { return nil },
+	}
+}
+
+func NewMockFileConverter(uri URI) *MockFileConverter {
+	f := &MockFileConverter {}
+	f.MockConverter = NewMockConverter("file", uri)
+	return f
+}

internal/folio/mock_foo_resource_test.go

@@ -30,6 +30,11 @@ func RegisterMocks() {
 		f.Name = filepath.Join(u.Hostname(), u.Path)
 		return f
 	})
+	TestResourceTypes.Register([]string{"file"}, func(u *url.URL) data.Resource {
+		f := NewFileResource()
+		f.Name = filepath.Join(u.Hostname(), u.Path)
+		return f
+	})
 }
 
 
@@ -57,6 +62,13 @@ type MockTestuser struct {
 	Home string `json:"home" yaml:"home"`
 }
 
+type MockFile struct {
+	stater machine.Stater `json:"-" yaml:"-"`
+	*MockResource `json:"-" yaml:"-"`
+	Name string `json:"name" yaml:"name"`
+	Size int `json:"size" yaml:"size"`
+}
+
 func NewMockResource(typename string, stater machine.Stater) *MockResource {
 	return &MockResource {
 		InjectType: func() string { return typename },
@@ -80,22 +92,29 @@ func NewMockResource(typename string, stater machine.Stater) *MockResource {
 }
 
 func NewFooResource() *MockFoo {
-  f := &MockFoo {}
+	f := &MockFoo {}
 	f.stater = data.StorageMachine(f)
 	f.MockResource = NewMockResource("foo", f.stater)
 	return f
 }
 
 func NewBarResource() *MockBar {
-  b := &MockBar {}
+	b := &MockBar {}
 	b.stater = data.StorageMachine(b)
 	b.MockResource = NewMockResource("bar", b.stater)
 	return b
 }
 
 func NewTestuserResource() *MockTestuser {
-  u := &MockTestuser {}
+	u := &MockTestuser {}
 	u.stater = data.StorageMachine(u)
 	u.MockResource = NewMockResource("testuser", u.stater)
 	return u
 }
+
+func NewFileResource() *MockFile {
+	f := &MockFile {}
+	f.stater = data.StorageMachine(f)
+	f.MockResource = NewMockResource("file", f.stater)
+	return f
+}

internal/folio/resourcereference_test.go

@@ -13,14 +13,14 @@ import (
 func TestResourceReference(t *testing.T) {
 	f := NewFooResource()
 	resourceMapper := mapper.New[string, data.Declaration]()
-	f.Name = TempDir
+	f.Name = string(TempDir)
 	f.Size = 10
 	d := NewDeclaration()
 	d.Type = "foo"
 	d.Attributes = f
 	resourceMapper[d.URI()] = d
 
-	var foo ResourceReference = ResourceReference(fmt.Sprintf("foo://%s", TempDir))
+	var foo ResourceReference = ResourceReference(fmt.Sprintf("foo://%s", string(TempDir)))
 	u := foo.Parse()
 	assert.Equal(t, "foo", u.Scheme)
 	assert.True(t, foo.Exists())

internal/folio/schemas/document.schema.json

@@ -15,6 +15,14 @@
     "requires": {
       "$ref": "dependencies.schema.json"
     },
+    "imports": {
+      "type": "array",
+      "description": "List of other documents to import",
+      "items": {
+        "type": "string",
+        "description": "Document URI"
+      }
+    },
     "configurations": {
       "type": "array",
       "description": "Configurations list",