From c34a76981e9ad90d8016317f9cab5f14a2d301a6 Mon Sep 17 00:00:00 2001 From: Matthew Rich Date: Thu, 19 Sep 2024 08:03:23 +0000 Subject: [PATCH] move source/target converters to fan pkg --- internal/data/block.go | 24 ++ internal/data/config.go | 19 ++ internal/data/converter.go | 24 +- internal/data/data.go | 4 + internal/data/document.go | 18 +- internal/data/identifier.go | 10 + internal/data/resource.go | 18 ++ internal/data/signature.go | 12 + internal/data/types.go | 1 + internal/{source => fan}/container.go | 26 ++- internal/fan/dir.go | 176 +++++++++++++++ internal/fan/dir_test.go | 130 +++++++++++ internal/fan/fan.go | 30 +++ internal/fan/fan_test.go | 23 ++ internal/{source => fan}/group.go | 34 +-- internal/fan/http.go | 197 ++++++++++++++++ internal/{source => fan}/http_test.go | 2 +- internal/fan/iptable.go | 83 +++++++ internal/fan/jx.go | 271 +++++++++++++++++++++++ internal/fan/jx_test.go | 46 ++++ internal/fan/package.go | 76 +++++++ internal/{source => fan}/package_test.go | 6 +- internal/fan/tar.go | 217 ++++++++++++++++++ internal/fan/tar_test.go | 105 +++++++++ internal/{source => fan}/user.go | 35 +-- internal/{source => fan}/user_test.go | 6 +- internal/source/decl.go | 98 -------- internal/source/dir.go | 101 --------- internal/source/dir_test.go | 23 -- internal/source/docsource.go | 37 ---- internal/source/docsource_test.go | 48 ---- internal/source/http.go | 83 ------- internal/source/iptable.go | 69 ------ internal/source/package.go | 68 ------ internal/source/tar.go | 103 --------- internal/source/tar_test.go | 14 -- internal/source/types.go | 28 --- internal/source/types_test.go | 47 ---- internal/target/decl.go | 174 --------------- internal/target/doctarget.go | 36 --- internal/target/tar.go | 105 --------- internal/target/tar_test.go | 14 -- internal/target/types.go | 100 --------- internal/target/types_test.go | 90 -------- 44 files changed, 1545 insertions(+), 1286 deletions(-) create mode 100644 internal/data/block.go create mode 100644 internal/data/signature.go rename internal/{source => fan}/container.go (60%) create mode 100644 internal/fan/dir.go create mode 100644 internal/fan/dir_test.go create mode 100644 internal/fan/fan.go create mode 100644 internal/fan/fan_test.go rename internal/{source => fan}/group.go (58%) create mode 100644 internal/fan/http.go rename internal/{source => fan}/http_test.go (93%) create mode 100644 internal/fan/iptable.go create mode 100644 internal/fan/jx.go create mode 100644 internal/fan/jx_test.go create mode 100644 internal/fan/package.go rename internal/{source => fan}/package_test.go (79%) create mode 100644 internal/fan/tar.go create mode 100644 internal/fan/tar_test.go rename internal/{source => fan}/user.go (55%) rename internal/{source => fan}/user_test.go (78%) delete mode 100644 internal/source/decl.go delete mode 100644 internal/source/dir.go delete mode 100644 internal/source/dir_test.go delete mode 100644 internal/source/docsource.go delete mode 100644 internal/source/docsource_test.go delete mode 100644 internal/source/http.go delete mode 100644 internal/source/iptable.go delete mode 100644 internal/source/package.go delete mode 100644 internal/source/tar.go delete mode 100644 internal/source/tar_test.go delete mode 100644 internal/source/types.go delete mode 100644 internal/source/types_test.go delete mode 100644 internal/target/decl.go delete mode 100644 internal/target/doctarget.go delete mode 100644 internal/target/tar.go delete mode 100644 internal/target/tar_test.go delete mode 100644 internal/target/types.go delete mode 100644 internal/target/types_test.go diff --git a/internal/data/block.go b/internal/data/block.go new file mode 100644 index 0000000..0f068e6 --- /dev/null +++ b/internal/data/block.go @@ -0,0 +1,24 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package data + +import ( + "errors" +) + +var ( + ErrConfigUndefinedName = errors.New("Config block is missing a defined name") +) + + +type Block interface { + Identifier + ConfigurationType() TypeName + Loader + Validator + NewConfiguration(uri *string) error + ConfigurationValueGetter + Configuration() Configuration + Clone() Block +} + diff --git a/internal/data/config.go b/internal/data/config.go index 215a204..21572ca 100644 --- a/internal/data/config.go +++ b/internal/data/config.go @@ -3,8 +3,27 @@ package data import ( + "errors" +) + +var ( + ErrUnknownConfigurationType = errors.New("Unknown configuration type") + ErrUnknownConfigurationKey = errors.New("Unknown configuration key") ) type ConfigurationValueGetter interface { GetValue(key string) (any, error) } + +type ConfigurationValueChecker interface { + Has(key string) bool +} + +type Configuration interface { + Identifier + Type() string + Reader + ConfigurationValueGetter + ConfigurationValueChecker + Clone() Configuration +} diff --git a/internal/data/converter.go b/internal/data/converter.go index 91ad83d..e334139 100644 --- a/internal/data/converter.go +++ b/internal/data/converter.go @@ -3,24 +3,38 @@ package data import ( + "errors" +) + +var ( + ErrUnsupportedConversion = errors.New("Unsupported conversion") ) // Convert a resource to a document and a document to a resource type Emitter interface { - Emit(document Document, filter ResourceSelector) (Resource, error) + Emit(document Document, filter ElementSelector) (Resource, error) } -type Extracter interface { - Extract(resource Resource, filter ResourceSelector) (Document, error) +type Extractor interface { + Extract(resource Resource, filter ElementSelector) (Document, error) } type Converter interface { Typer Emitter - Extracter + Extractor + Close() error } type ManyExtractor interface { - ExtractMany(resource Resource, filter ResourceSelector) ([]Document, error) + ExtractMany(resource Resource, filter ElementSelector) ([]Document, error) +} + +type ManyEmitter interface { + EmitMany(documents []Document, filter ElementSelector) (Resource, error) +} + +type DirectoryConverter interface { + SetRelative(flag bool) } diff --git a/internal/data/data.go b/internal/data/data.go index 0e4de62..cd22df0 100644 --- a/internal/data/data.go +++ b/internal/data/data.go @@ -26,6 +26,10 @@ type Deleter interface { Delete(context.Context) error } +type Info interface { + ReadStat() error +} + type Crudder interface { Creator Reader diff --git a/internal/data/document.go b/internal/data/document.go index 6899086..8c0e9df 100644 --- a/internal/data/document.go +++ b/internal/data/document.go @@ -8,6 +8,7 @@ import ( "decl/internal/codec" "io" "decl/internal/mapper" + "net/url" ) var ( @@ -38,14 +39,29 @@ type Document interface { mapper.Mapper NewResource(uri string) (Resource, error) + NewResourceFromParsedURI(uri *url.URL) (Resource, error) + AddDeclaration(Declaration) + AddResourceDeclaration(resourceType string, resourceDeclaration Resource) + Types() (TypesRegistry[Resource]) // Resources() []Declaration SetConfig(config Document) ConfigDoc() Document + + HasConfig(string) bool + GetConfig(string) Block + Len() int ResolveIds(ctx context.Context) Filter(filter DeclarationSelector) []Declaration + Declarations() []Declaration - //Diff(with *Document, output io.Writer) (returnOutput string, diffErr error) + CheckConstraints() bool + Failures() int + + ConfigFilter(filter BlockSelector) []Block + AppendConfigurations([]Document) + Diff(with Document, output io.Writer) (returnOutput string, diffErr error) + Clone() Document } diff --git a/internal/data/identifier.go b/internal/data/identifier.go index f9fd589..7cf80e5 100644 --- a/internal/data/identifier.go +++ b/internal/data/identifier.go @@ -4,6 +4,7 @@ package data import ( "errors" + "net/url" ) var ( @@ -13,6 +14,11 @@ var ( type Identifier interface { URI() string SetURI(string) error + SetParsedURI(*url.URL) error +} + +type DocumentElement interface { + Identifier } type Selector[Item comparable] func(r Item) bool @@ -20,3 +26,7 @@ type Selector[Item comparable] func(r Item) bool type ResourceSelector Selector[Resource] type DeclarationSelector Selector[Declaration] + +type BlockSelector Selector[Block] + +type ElementSelector Selector[DocumentElement] diff --git a/internal/data/resource.go b/internal/data/resource.go index 94c79b8..694d2d8 100644 --- a/internal/data/resource.go +++ b/internal/data/resource.go @@ -51,6 +51,11 @@ func NewResourceMapper() ResourceMapper { return mapper.New[string, Declaration]() } +type ContentHasher interface { + Hash() []byte + HashHexString() string +} + type ContentIdentifier interface { ContentType() string } @@ -82,10 +87,23 @@ type ContentGetSetter interface { } type FileResource interface { + SetBasePath(int) FilePath() string SetFileInfo(fs.FileInfo) error FileInfo() fs.FileInfo ContentGetSetter + GetContentSourceRef() string SetContentSourceRef(uri string) + SetFS(fs.FS) + PathNormalization(bool) + NormalizePath() error + GetTarget() string } +type Signed interface { + Signature() Signature +} + +type FileInfoGetter interface { + Stat() (fs.FileInfo, error) +} diff --git a/internal/data/signature.go b/internal/data/signature.go new file mode 100644 index 0000000..db9ab05 --- /dev/null +++ b/internal/data/signature.go @@ -0,0 +1,12 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package data + +import ( +) + +type Signature interface { + Verify(ContentHasher) error + SetHexString(string) error + String() string +} diff --git a/internal/data/types.go b/internal/data/types.go index dc5da75..e4af876 100644 --- a/internal/data/types.go +++ b/internal/data/types.go @@ -10,6 +10,7 @@ type Factory[Product comparable] func(*url.URL) Product type TypesRegistry[Product comparable] interface { New(uri string) (result Product, err error) + NewFromParsedURI(uri *url.URL) (result Product, err error) Has(typename string) bool //Get(string) Factory[Product] } diff --git a/internal/source/container.go b/internal/fan/container.go similarity index 60% rename from internal/source/container.go rename to internal/fan/container.go index 1a695ec..e3da713 100644 --- a/internal/source/container.go +++ b/internal/fan/container.go @@ -1,6 +1,6 @@ // Copyright 2024 Matthew Rich . All rights reserved. -package source +package fan import ( "context" @@ -10,6 +10,8 @@ _ "gopkg.in/yaml.v3" "net/url" _ "path/filepath" "decl/internal/resource" + "decl/internal/folio" + "decl/internal/data" _ "os" _ "io" "github.com/docker/docker/api/types/container" @@ -36,32 +38,40 @@ func NewContainer(containerClientApi resource.ContainerClient) *Container { } func init() { - SourceTypes.Register([]string{"container"}, func(u *url.URL) DocSource { + folio.DocumentRegistry.ConverterTypes.Register([]string{"container"}, func(u *url.URL) data.Converter { c := NewContainer(nil) return c }) } -func (c *Container) Type() string { return "container" } +func (c *Container) Type() data.TypeName { return "container" } -func (c *Container) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { +func (c *Container) Extract(sourceResource data.Resource, filter data.ElementSelector) (document data.Document, err error) { var extractErr error ctx := context.Background() - slog.Info("container source ExtractResources()", "container", c) + slog.Info("container source Extract()", "container", c) containers, err := c.apiClient.ContainerList(ctx, container.ListOptions{All: true}) if err != nil { return nil, err } - document := resource.NewDocument() + document = folio.DocumentRegistry.NewDocument(folio.URI(sourceResource.URI())) for _, container := range containers { runningContainer := resource.NewContainer(nil) if inspectErr := runningContainer.Inspect(ctx, container.ID); inspectErr != nil { extractErr = fmt.Errorf("%w: %w", extractErr, inspectErr) } - document.AddResourceDeclaration("container", runningContainer) + document.(*folio.Document).AddResourceDeclaration("container", runningContainer) } - return []*resource.Document{document}, extractErr + return document, extractErr +} + +func (c *Container) Emit(document data.Document, filter data.ElementSelector) (resource data.Resource, err error) { + return nil, nil +} + +func (c *Container) Close() error { + return nil } diff --git a/internal/fan/dir.go b/internal/fan/dir.go new file mode 100644 index 0000000..bac7d25 --- /dev/null +++ b/internal/fan/dir.go @@ -0,0 +1,176 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( + "context" +_ "encoding/json" + "fmt" +_ "gopkg.in/yaml.v3" + "net/url" + "path/filepath" + "decl/internal/data" + "decl/internal/folio" + "os" +_ "io" + "log/slog" + "decl/internal/fs" +) + +type Dir struct { + Path string `yaml:"path" json:"path"` + Relative bool `yaml:"relative" json:"relative"` + subDirsStack []string `yaml:"-" json:"-"` + fs *fs.WalkDir `yaml:"-" json:"-"` +} + +func NewDir() *Dir { + return &Dir{ + subDirsStack: make([]string, 0, 100), + } +} + +func init() { + folio.DocumentRegistry.ConverterTypes.Register([]string{"file"}, func(u *url.URL) data.Converter { + t := NewDir() + t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path)) + t.Relative = false + return t + }) + +} + +func (d *Dir) SetRelative(flag bool) { d.Relative = flag } + +func (d *Dir) Type() data.TypeName { return "dir" } + +func (d *Dir) ExtractDirectory(path string, document data.Document) (err error) { + ctx := context.Background() + files, readDirErr := os.ReadDir(path) + slog.Info("fan.Dir.ExtractDirectory()", "path", path, "error", readDirErr) + if readDirErr != nil { + return readDirErr + } + + for _,file := range files { + filePath := filepath.Join(path, file.Name()) + u := fmt.Sprintf("file://%s", filePath) + var f data.Resource + if f, err = document.NewResource(u); err != nil { + return + } + if _, err = f.Read(ctx); err != nil { + return + } + + if file.IsDir() { + d.subDirsStack = append(d.subDirsStack, filePath) + } + } + return nil +} + +func (d *Dir) isParent(m *map[string]int, path string, containingDirectoryPath string) (newCDP string, cdpCount int) { + newCDP = containingDirectoryPath + cdpCount = (*m)[containingDirectoryPath] + pathLen := len(path) + for i, p := range path { + if p == '/' || i == pathLen { + sPath := path[:i] + if len(sPath) > 0 { + (*m)[sPath]++ + superDirCount := (*m)[sPath] + if superDirCount >= cdpCount { + newCDP = sPath + cdpCount = superDirCount + } + } + } + } + return +} + +func (d *Dir) LCPath(files []string) (lcPath string) { + parentPaths := make(map[string]int) + var containingDirectoryPath string + for _,filePath := range files { + containingDirectoryPath, _ = d.isParent(&parentPaths, filePath, containingDirectoryPath) + } + lcPath = containingDirectoryPath + return +} + +func (d *Dir) Emit(document data.Document, filter data.ElementSelector) (resourceTarget data.Resource, err error) { + if document == nil || document.Len() <= 0 { + return nil, ErrEmptyDocument + } + + dirFileDeclaration := folio.NewDeclaration() + dirFileDeclaration.Type = "file" + if err = dirFileDeclaration.NewResource(nil); err != nil { + return + } + + parentPaths := make(map[string]int) + var containingDirectoryPath string + for _,res := range document.Filter(func(d data.Declaration) bool { + return d.ResourceType() == "file" + }) { + var f data.FileResource = res.(*folio.Declaration).Attributes.(data.FileResource) + var parent string + + if f.FileInfo().IsDir() { + parent, err = filepath.Abs(f.FilePath()) + } else { + parent, err = filepath.Abs(filepath.Dir(f.FilePath())) + } + if err != nil { + return + } + + containingDirectoryPath, _ = d.isParent(&parentPaths, parent, containingDirectoryPath) + } + uri := fmt.Sprintf("file://%s", containingDirectoryPath) + if err = dirFileDeclaration.SetURI(uri); err != nil { + return + } + + resourceTarget = dirFileDeclaration.Attributes + return +} + +func (d *Dir) Extract(resourceSource data.Resource, filter data.ElementSelector) (document data.Document, err error) { + ctx := context.Background() + if resourceSource.Type() != "file" { + return nil, fmt.Errorf("%w", ErrInvalidResource) + } + slog.Info("fan.Dir.Extract()", "path", d.Path, "resource", resourceSource) + d.Path = resourceSource.(data.FileResource).FilePath() + document = folio.DocumentRegistry.NewDocument("") + + d.fs = fs.NewWalkDir(os.DirFS(d.Path), d.Path, func(fsys fs.FS, path string, file fs.DirEntry) (err error) { + u := fmt.Sprintf("file://%s", path) + slog.Info("Fan.Dir.Extract() WalkDir", "file", u, "root", d.Path) + if path != "" { + var f data.Resource + if f, err = document.NewResource(u); err != nil { + return + } + if d.Relative { + f.(data.FileResource).SetBasePath(len(d.Path) + 1) + slog.Info("Fan.Dir.Extract() WalkDir Relative", "file", f, "path", path) + } + slog.Info("Fan.Dir.Extract() WalkDir Resource.Read", "file", f) + _, err = f.Read(ctx) + } + return + }) + + slog.Info("Fan.Dir.Extract()", "fs", d.fs) + err = d.fs.Walk(nil) + return +} + +func (d *Dir) Close() error { + return nil +} diff --git a/internal/fan/dir_test.go b/internal/fan/dir_test.go new file mode 100644 index 0000000..96e7d8d --- /dev/null +++ b/internal/fan/dir_test.go @@ -0,0 +1,130 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( + "github.com/stretchr/testify/assert" + "testing" + "decl/internal/folio" + "decl/internal/data" + "log/slog" + "path/filepath" + "os" + "fmt" +) + +func TestNewDirSource(t *testing.T) { + s := NewDir() + assert.NotNil(t, s) +} + +func TestExtractDirectory(t *testing.T) { + s := NewDir() + assert.NotNil(t, s) + + document := folio.DocumentRegistry.NewDocument("") + assert.NotNil(t, document) + + assert.Nil(t, s.ExtractDirectory(TempDir, document)) + assert.Greater(t, 2, document.Len()) + +} + +func TestIsParent(t *testing.T) { + s := NewDir() + assert.NotNil(t, s) + + m := map[string]int{ + "/foo/bar": 3, + "/foo": 1, + } + res, count := s.isParent(&m, "/foo/bar/baz/quuz", "/foo/bar") + + assert.Equal(t, "/foo/bar", res) + assert.Equal(t, 4, count) + assert.Equal(t, 2, m["/foo"]) +} + +func TestLCPath(t *testing.T) { + s := NewDir() + assert.NotNil(t, s) + result := s.LCPath([]string{ + "/foo/bar/baz/quuz", + "/foo/bar/baz/quuz/abc.txt", + "/foo/bar/baz/quuz/def.txt", + "/foo/bar/baz/quz/ghi.txt", + "/foo/bar/kiw", + "/tmp", + }) + assert.Equal(t, "/foo/bar", result) + + result = s.LCPath([]string{ + "/foo/bar/baz/quuz", + "/foo/eer/voo", + "/foo/bar/baz/quuz/abc.txt", + "/foo/bar/baz/quuz/def.txt", + "/foo/bar/baz/quz/ghi.txt", + "/foo/bar/kiw", + "/tmp", + "/usr", + "/usr/lib", + }) + assert.Equal(t, "/foo", result) +} + +func BenchmarkLCPath(b *testing.B) { + s := NewDir() + assert.NotNil(b, s) + for i := 0; i < b.N; i++ { + s.LCPath([]string{ + "/foo/bar/baz/quuz", + "/foo/eer/voo", + "/foo/bar/baz/quuz/abc.txt", + "/foo/bar/baz/quuz/def.txt", + "/foo/bar/baz/quz/ghi.txt", + "/foo/bar/kiw", + "/tmp", + "/usr", + "/usr/lib", + }) + } +} + +func TestEmit(t *testing.T) { + s := NewDir() + assert.NotNil(t, s) + + contextDir, _ := filepath.Abs(filepath.Join(TempDir, "context")) + etcDir := filepath.Join(contextDir, "etc") + binDir := filepath.Join(contextDir, "bin") + usrDir := filepath.Join(contextDir, "usr") + usrLibDir := filepath.Join(contextDir, "usr/lib") + usrBinDir := filepath.Join(contextDir, "usr/bin") + + assert.Nil(t, os.Mkdir(contextDir, os.ModePerm)) + assert.Nil(t, os.Mkdir(etcDir, os.ModePerm)) + assert.Nil(t, os.Mkdir(binDir, os.ModePerm)) + assert.Nil(t, os.Mkdir(usrDir, os.ModePerm)) + assert.Nil(t, os.Mkdir(usrLibDir, os.ModePerm)) + assert.Nil(t, os.Mkdir(usrBinDir, os.ModePerm)) + + + decl := folio.NewDeclaration() + srcFile := fmt.Sprintf("file://%s", contextDir) + resErr := decl.NewResource(&srcFile) + assert.Nil(t, resErr) + + slog.Info("TestEmit()", "file", decl, "res", decl.Attributes) + + document, extractErr := s.Extract(decl.Resource(), nil) + slog.Info("TestEmit() - Extract", "document", document, "error", extractErr) + assert.Nil(t, extractErr) + assert.Greater(t, document.Len(), 4) + + res, emitErr := s.Emit(document, nil) + slog.Info("TestEmit()", "res", res, "error", emitErr) + + assert.Nil(t, emitErr) + assert.Equal(t, contextDir, res.(data.FileResource).FilePath()) + +} diff --git a/internal/fan/fan.go b/internal/fan/fan.go new file mode 100644 index 0000000..228a004 --- /dev/null +++ b/internal/fan/fan.go @@ -0,0 +1,30 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( + "errors" +) + + +// Convert a resource to a document and a document to a resource +/* +type Emitter interface { + Emit(document *resource.Document) (resource.Resource, error) +} + +type Extracter interface { + Extract(resource resource.Resource, filter resource.ResourceSelector) (*resource.Document, error) +} + +type Converter interface { + Emitter + Extracter +} +*/ + +var ( + ErrInvalidSource error = errors.New("Invalid source") + ErrInvalidResource error = errors.New("Invalid resource") + ErrEmptyDocument error = errors.New("Document containers no resources") +) diff --git a/internal/fan/fan_test.go b/internal/fan/fan_test.go new file mode 100644 index 0000000..601251a --- /dev/null +++ b/internal/fan/fan_test.go @@ -0,0 +1,23 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( + "testing" + "os" + "log" +) + +var TempDir string + +func TestMain(m *testing.M) { + var err error + TempDir, err = os.MkdirTemp("", "testfan") + if err != nil || TempDir == "" { + log.Fatal(err) + } + //folio.DocumentRegistry.ResourceTypes = resource.ResourceTypes + rc := m.Run() + os.RemoveAll(TempDir) + os.Exit(rc) +} diff --git a/internal/source/group.go b/internal/fan/group.go similarity index 58% rename from internal/source/group.go rename to internal/fan/group.go index 5964506..43f4dd1 100644 --- a/internal/source/group.go +++ b/internal/fan/group.go @@ -1,6 +1,6 @@ // Copyright 2024 Matthew Rich . All rights reserved. -package source +package fan import ( _ "context" @@ -10,6 +10,8 @@ _ "gopkg.in/yaml.v3" "net/url" _ "path/filepath" "decl/internal/resource" + "decl/internal/folio" + "decl/internal/data" _ "os" _ "io" "log/slog" @@ -24,7 +26,7 @@ func NewGroup() *Group { } func init() { - SourceTypes.Register([]string{"group"}, func(u *url.URL) DocSource { + folio.DocumentRegistry.ConverterTypes.Register([]string{"group"}, func(u *url.URL) data.Converter { groupSource := NewGroup() groupType := u.Query().Get("type") if len(groupType) > 0 { @@ -35,23 +37,21 @@ func init() { } -func (g *Group) Type() string { return "group" } - -func (g *Group) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { - documents := make([]*resource.Document, 0, 100) +func (g *Group) Type() data.TypeName { return "group" } +func (g *Group) Extract(sourceResource data.Resource, filter data.ElementSelector) (document data.Document, err error) { slog.Info("group source ExtractResources()", "group", g) Groups := make([]*resource.Group, 0, 100) cmd := g.GroupType.NewReadGroupsCommand() if cmd == nil { - return documents, resource.ErrUnsupportedGroupType + return document, resource.ErrUnsupportedGroupType } if out, err := cmd.Execute(g); err == nil { - slog.Info("group source ExtractResources()", "output", out) + slog.Info("group source Extract()", "output", out) if exErr := cmd.Extractor(out, &Groups); exErr != nil { - return documents, exErr + return document, exErr } - document := resource.NewDocument() + document = folio.DocumentRegistry.NewDocument("group://-") for _, grp := range Groups { if grp == nil { grp = resource.NewGroup() @@ -59,10 +59,18 @@ func (g *Group) ExtractResources(filter ResourceSelector) ([]*resource.Document, grp.GroupType = g.GroupType document.AddResourceDeclaration("group", grp) } - documents = append(documents, document) } else { slog.Info("group source ExtractResources()", "output", out, "error", err) - return documents, err + return document, err } - return documents, nil + return document, nil } + +func (g *Group) Emit(document data.Document, filter data.ElementSelector) (resourceTarget data.Resource, err error) { + return nil, data.ErrUnsupportedConversion +} + +func (g *Group) Close() error { + return nil +} + diff --git a/internal/fan/http.go b/internal/fan/http.go new file mode 100644 index 0000000..093a565 --- /dev/null +++ b/internal/fan/http.go @@ -0,0 +1,197 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( +_ "context" +_ "encoding/json" + "fmt" +_ "gopkg.in/yaml.v3" + "net/url" +_ "net/http" +_ "path/filepath" +_ "decl/internal/resource" + "decl/internal/codec" + "decl/internal/data" + "decl/internal/folio" +_ "os" + "io" + "errors" + "log/slog" +) + +type HTTP struct { + Endpoint folio.URI `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + url *url.URL `yaml:"-" json:"-"` + Format codec.Format `yaml:"format,omitempty" json:"format,omitempty"` + + reader io.ReadCloser `yaml:"-" json:"-"` + writer io.WriteCloser `yaml:"-" json:"-"` + decoder codec.Decoder `yaml:"-" json:"-"` + encoder codec.Encoder `yaml:"-" json:"-"` + closer func() error `yaml:"-" json:"-"` + index int `yaml:"-" json:"-"` + signature data.Signature `yaml:"-" json:"-"` +} + +func NewHTTP() *HTTP { + return &HTTP{ Format: codec.FormatYaml, index: 0, closer: func() error { return nil } } +} + +func init() { + folio.DocumentRegistry.ConverterTypes.Register([]string{"http","https"}, func(u *url.URL) data.Converter { + t := NewHTTP() + t.Endpoint = folio.URI(u.String()) + t.url = u + return t + }) +} + +func (h *HTTP) Type() data.TypeName { return "http" } + +/* +func (h *HTTP) setencoder(target data.ContentIdentifier) { + if formatErr := h.Format.Set(target.ContentType()); formatErr != nil { + h.Format = codec.FormatYaml + if format,ok := h.url.Query()["format"]; ok { + if queryFormatErr := h.Format.Set(format[0]); queryFormatErr != nil { + h.Format = codec.FormatYaml + } + } + } + if h.encoder == nil { + h.encoder = codec.NewEncoder(h.writer, h.Format) + } +} +*/ + +func (h *HTTP) setdecoder(source data.ContentIdentifier) { + if h.decoder == nil { + _ = h.Format.Set(source.ContentType()) + h.decoder = codec.NewDecoder(h.reader, h.Format) + } +} + +func (h *HTTP) Extract(sourceResource data.Resource, filter data.ElementSelector) (document data.Document, err error) { + if h.index == 0 { + if sourceResource == nil { + if len(h.Endpoint) > 0 { + sourceResource, err = h.Endpoint.NewResource(nil) + } else { + return nil, ErrInvalidSource + } + } + slog.Info("HTTP.Extract()", "source", sourceResource, "error", err) + var jxSourceFile data.FileResource = sourceResource.(data.FileResource) + h.reader, err = jxSourceFile.(data.ContentGetter).GetContent(nil) + slog.Info("HTTP.Extract()", "file", h, "error", err) + if err != nil { + return + } + h.signature = sourceResource.(data.Signed).Signature() + h.setdecoder(jxSourceFile.(data.ContentIdentifier)) + slog.Info("HTTP.Extract()", "jx", h) + } + + u := fmt.Sprintf("%s?index=%d", sourceResource.URI(), h.index) + document = folio.DocumentRegistry.NewDocument(folio.URI(u)) + err = h.decoder.Decode(document) + slog.Info("HTTP.Extract()", "doc", document, "http", h, "error", err) + h.index++ + if err != nil { + return + } + if err = document.Validate(); err != nil { + return + } + + if h.signature.String() != "" { + if v, ok := sourceResource.(data.ContentHasher); ok { + err = h.signature.Verify(v) + } + } + return + +/* + defer h.Close() + documentSignature := h.transport.Signature() + + hash := sha256.New() + sumReadData := iofilter.NewReader(h.transport, func(p []byte, readn int, readerr error) (n int, err error) { + hash.Write(p) + return + }) + + decoder := codec.NewYAMLDecoder(sumReadData) + index := 0 + for { + doc = folio.DocumentRegistry.NewDocument(folio.URI(u)) + + doc := resource.NewDocument() + e := decoder.Decode(doc) + if errors.Is(e, io.EOF) { + break + } + if e != nil { + return documents, e + } + if validationErr := doc.Validate(); validationErr != nil { + return documents, validationErr + } + documents = append(documents, doc) + index++ + } + + if documentSignature != "" { + sig := &signature.Ident{} + sigErr := sig.VerifySum(hash.Sum(nil), []byte(documentSignature)) + if sigErr != nil { + return documents, sigErr + } + } +*/ +} + + +func (h *HTTP) ExtractMany(resourceSource data.Resource, filter data.ElementSelector) (documents []data.Document, err error) { + documents = make([]data.Document, 0, 100) + defer h.Close() + + h.index = 0 + for { + var doc data.Document + if doc, err = h.Extract(resourceSource, filter); err == nil { + documents = append(documents, doc) + } else { + if errors.Is(err, io.EOF) { + err = nil + //documents = append(documents, doc) + } + break + } + } + slog.Info("HTTP.ExtractMany()", "file", h, "error", err) + return +} + +func (h *HTTP) Emit(document data.Document, filter data.ElementSelector) (resource data.Resource, err error) { + return nil, nil +} + +func (h *HTTP) Close() (err error) { +/* + if h.decoder != nil { + h.decoder.Close() + } +*/ + if h.encoder != nil { + h.encoder.Close() + } + if h.reader != nil { + h.reader.Close() + } + if h.writer != nil { + h.writer.Close() + } + return +} diff --git a/internal/source/http_test.go b/internal/fan/http_test.go similarity index 93% rename from internal/source/http_test.go rename to internal/fan/http_test.go index 4d3d02f..51a686c 100644 --- a/internal/source/http_test.go +++ b/internal/fan/http_test.go @@ -1,6 +1,6 @@ // Copyright 2024 Matthew Rich . All rights reserved. -package source +package fan import ( "github.com/stretchr/testify/assert" diff --git a/internal/fan/iptable.go b/internal/fan/iptable.go new file mode 100644 index 0000000..6d72e6a --- /dev/null +++ b/internal/fan/iptable.go @@ -0,0 +1,83 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( +_ "context" +_ "encoding/json" + "fmt" +_ "gopkg.in/yaml.v3" + "net/url" +_ "path/filepath" + "decl/internal/data" + "decl/internal/resource" + "decl/internal/folio" +_ "os" +_ "io" + "strings" + "log/slog" +) + +type Iptable struct { + Table string `yaml:"table" json:"table"` + Chain string `yaml:"chain" json:"chain"` +} + +func NewIptable() *Iptable { + return &Iptable{} +} + +func init() { + folio.DocumentRegistry.ConverterTypes.Register([]string{"iptable"}, func(u *url.URL) data.Converter { + t := NewIptable() + t.Table = u.Hostname() + elements := strings.FieldsFunc(u.Path, func(c rune) bool { return c == '/' }) + if len(elements) >= 1 { + t.Chain = elements[0] + } + slog.Info("iptable chain source factory", "table", t, "uri", u, "table", u.Hostname()) + return t + }) + +} + + +func (i *Iptable) Type() data.TypeName { return "iptable" } + +func (i *Iptable) Extract(sourceResource data.Resource, filter data.ElementSelector) (document data.Document, err error) { + + slog.Info("fan.Iptable.Extract()", "table", i) + + iptRules := make([]*resource.Iptable, 0, 100) + cmd := resource.NewIptableReadChainCommand() + if cmd == nil { + return document, fmt.Errorf("Iptable read chain: invalid command") + } + + var out []byte + if out, err = cmd.Execute(i); err == nil { + + if err = cmd.Extractor(out, &iptRules); err == nil { + document = folio.DocumentRegistry.NewDocument(folio.URI(sourceResource.URI())) + for _, rule := range iptRules { + if rule == nil { + rule = resource.NewIptable() + } + rule.Table = resource.IptableName(i.Table) + rule.Chain = resource.IptableChain(i.Chain) + slog.Info("iptable chain source Extract()", "rule", rule) + document.(*folio.Document).AddResourceDeclaration("iptable", rule) + } + } + } + slog.Info("fan.Iptable.Extract()", "output", out, "error", err) + return document, err +} + +func (i *Iptable) Emit(document data.Document, filter data.ElementSelector) (resourceTarget data.Resource, err error) { + return nil, nil +} + +func (i *Iptable) Close() error { + return nil +} diff --git a/internal/fan/jx.go b/internal/fan/jx.go new file mode 100644 index 0000000..84c7227 --- /dev/null +++ b/internal/fan/jx.go @@ -0,0 +1,271 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( + "context" +_ "encoding/json" + "fmt" +_ "gopkg.in/yaml.v3" + "net/url" + "path/filepath" + "decl/internal/codec" + "decl/internal/folio" + "decl/internal/data" +_ "os" + "io" + "errors" + "log/slog" + "strings" +) + +/* + Converts a file container an encoded (yaml, json, etc) JX document into a Document by using `Extract` or + `ExtractMany`. + Converts a JX Document structure into a yaml, json, etc encoded resource. +*/ +type JxFile struct { + Uri folio.URI `yaml:"uri,omitempty" json:"uri,omitempty"` + url *url.URL `yaml:"-" json:"-"` + + emitResource data.Resource `yaml:"-" json:"-"` + + Path string `yaml:"path" json:"path"` + Format codec.Format `yaml:"format,omitempty" json:"format,omitempty"` + reader io.ReadCloser `yaml:"-" json:"-"` + writer io.WriteCloser `yaml:"-" json:"-"` + decoder codec.Decoder `yaml:"-" json:"-"` + encoder codec.Encoder `yaml:"-" json:"-"` + closer func() error `yaml:"-" json:"-"` + index int `yaml:"-" json:"-"` +} + +func NewJxFile() *JxFile { + return &JxFile{ Format: codec.FormatYaml, index: 0, closer: func() error { return nil } } +} + +func init() { + folio.DocumentRegistry.ConverterTypes.Register([]string{"decl", "jx", "yaml", "yml", "json"}, func(u *url.URL) data.Converter { + j := NewJxFile() + j.SetURI(u) + return j + }) + + folio.DocumentRegistry.ConverterTypes.RegisterContentType([]string{"jx.yaml","jx.yml","jx.yaml.gz","jx.yml.gz", "jx.json", "jx.json.gz"}, func(u *url.URL) data.Converter { + j := NewJxFile() + slog.Info("JxFile.Factory", "jx", j) + j.SetURI(u) + slog.Info("JxFile.Factory", "jx", j) + return j + }) +} + +/* + Schemes: file, json, yaml, yml, decl, jx, http, https, other transport schemes? + Format: URL scheme name, `format` query param, file extension + + If the input url is a file + Detect Format +*/ +func (j *JxFile) SetURI(u *url.URL) { + slog.Info("JxFile.SetURI()", "jx", j) + if ! errors.Is(j.Format.Set(u.Scheme), codec.ErrInvalidFormat) { + u.Scheme = "file" + q := u.Query() + q.Set("format", string(j.Format)) + u.RawQuery = q.Encode() + } else { + if format,ok := u.Query()["format"]; ok { + _ = j.Format.Set(format[0]) + } + } + if u.Scheme == "file" { + if u.Path == "" || u.Path == "-" { + j.Path = "-" + } else { + fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.Path)) + j.Path = fileAbsolutePath + if _, err := u.Parse(j.Path); err != nil { + panic(err) + } + } + } else { + j.Path = filepath.Join(u.Hostname(), u.RequestURI()) + } + j.Uri.SetURL(u) + if j.Format == codec.FormatYaml { + exttype, ext := j.Uri.Extension() + if j.Format.Set(exttype) != nil { + _ = j.Format.Set(ext) + } + } +} + +func (j *JxFile) setencoder(target data.ContentIdentifier) { + if formatErr := j.Format.Set(target.ContentType()); formatErr != nil { + j.Format = codec.FormatYaml + if format,ok := j.url.Query()["format"]; ok { + if queryFormatErr := j.Format.Set(format[0]); queryFormatErr != nil { + j.Format = codec.FormatYaml + } + } + } + if j.encoder == nil { + j.encoder = codec.NewEncoder(j.writer, j.Format) + } +} + +func (j *JxFile) setdecoder(source data.ContentIdentifier) { + if j.decoder == nil { + for _,v := range strings.Split(source.ContentType(), ".") { + _ = j.Format.Set(v) + } + j.decoder = codec.NewDecoder(j.reader, j.Format) + } +} + +func (j *JxFile) Type() data.TypeName { return "jx" } + +func (j *JxFile) Extract(resourceSource data.Resource, filter data.ElementSelector) (doc data.Document, err error) { + if j.index == 0 { // XXX + if resourceSource == nil { + if len(j.Uri) > 0 { + resourceSource, err = j.Uri.NewResource(nil) + } else { + return nil, ErrInvalidSource + } + } + slog.Info("JxFile.Extract()", "source", resourceSource, "error", err) + var jxSourceFile data.FileResource = resourceSource.(data.FileResource) + j.reader, err = jxSourceFile.(data.ContentGetter).GetContent(nil) + slog.Info("JxFile.Extract()", "jxfile", j, "error", err) + if err != nil { + return + } + j.setdecoder(jxSourceFile.(data.ContentIdentifier)) + slog.Info("JxFile.Extract()", "jxfile", j) + } + + uri := resourceSource.URI() + if folio.DocumentRegistry.HasDocument(folio.URI(uri)) { + uri = fmt.Sprintf("%s?index=%d", uri, j.index) + } + doc = folio.DocumentRegistry.NewDocument(folio.URI(uri)) + err = j.decoder.Decode(doc) + slog.Info("JxFile.Extract()", "doc", doc, "jxfile", j, "error", err) + j.index++ + if err != nil { + return + } + if err = doc.Validate(); err != nil { + return + } + return +} + +func (j *JxFile) ExtractMany(resourceSource data.Resource, filter data.ElementSelector) (documents []data.Document, err error) { + documents = make([]data.Document, 0, 100) + defer j.Close() + + j.index = 0 + for { + var doc data.Document + if doc, err = j.Extract(resourceSource, filter); err == nil { + documents = append(documents, doc) + } else { + if errors.Is(err, io.EOF) { + err = nil + //documents = append(documents, doc) + } + break + } + } + slog.Info("JxFile.ExtractMany()", "jxfile", j, "error", err) + return +} + +func (j *JxFile) targetResource() (target data.Resource, err error) { + if j.emitResource == nil { + targetUrl := j.Uri.Parse() + targetUrl.Scheme = "file" + q := targetUrl.Query() + q.Set("format", string(j.Format)) + targetUrl.RawQuery = q.Encode() + j.Uri.SetURL(targetUrl) + slog.Info("JxFile.targetResource() SetURI", "uri", j.Uri, "targetUrl", targetUrl) + j.url = targetUrl + slog.Info("JxFile.targetResource()", "target", targetUrl, "jxfile", j) + + if j.emitResource, err = j.Uri.NewResource(nil); err != nil { + return nil, err + } + + var jxTargetFile data.FileResource = j.emitResource.(data.FileResource) + jxTargetFile.SetContentSourceRef(j.Uri.String()) + + slog.Info("JxFile.targetResource() SetContentSourceRef", "target", jxTargetFile, "uri", j.Uri.String()) + j.writer, err = jxTargetFile.(data.ContentReadWriter).ContentWriterStream() + j.setencoder(j.emitResource.(data.ContentIdentifier)) + } + target = j.emitResource + return +} + +func (j *JxFile) Emit(document data.Document, filter data.ElementSelector) (resourceTarget data.Resource, err error) { + ctx := context.Background() + + resourceTarget, err = j.targetResource() + + if err != nil { + return + } + + emitDoc := folio.DocumentRegistry.NewDocument("") + if err = document.Validate(); err != nil { + return + } + + slog.Info("JxFile.Emit()", "document", document, "context", ctx) + for _, declaration := range document.Filter(func (d data.Declaration) bool { + if filter != nil { + return filter(d.(*folio.Declaration).Attributes) + } + return true + }) { + //declaration.(*folio.Declaration).Resource().Read(ctx) // XXX added read here since it was removed from SetURI + emitDoc.ResourceDeclarations = append(emitDoc.ResourceDeclarations, declaration.(*folio.Declaration)) + } + + document.(*folio.Document).Format = j.Format + slog.Info("Emit", "target", j, "encoder", j.encoder, "emit", emitDoc) + if err = j.encoder.Encode(document); err != nil { + slog.Info("Emit", "err", err) + return + } + return +} + +func (j *JxFile) EmitMany(documents []data.Document, filter data.ElementSelector) (resourceTarget data.Resource, err error) { + for _, doc := range documents { + if resourceTarget, err = j.Emit(doc, filter); err != nil { + return + } + } + return +} + +func (j *JxFile) Close() (err error) { + if j.closer != nil { + err = j.closer() + } + if j.reader != nil { + j.reader.Close() + } + if j.encoder != nil { + j.encoder.Close() + } + if j.writer != nil { + j.writer.Close() + } + return +} diff --git a/internal/fan/jx_test.go b/internal/fan/jx_test.go new file mode 100644 index 0000000..f9ae511 --- /dev/null +++ b/internal/fan/jx_test.go @@ -0,0 +1,46 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( + "github.com/stretchr/testify/assert" + "testing" + "decl/internal/codec" + "decl/internal/folio" + "decl/internal/data" + "net/url" +) + +func TestNewJxSource(t *testing.T) { + s := NewJxFile() + assert.NotNil(t, s) +} + +func TestJxSetURI(t *testing.T) { + for _,v := range []struct{ url string; expectedformat codec.Format; expecteduri string }{ + { url: "file://foo", expectedformat: codec.FormatYaml, expecteduri: "file://foo" }, + { url: "json://foo", expectedformat: codec.FormatJson, expecteduri: "file://foo?format=json" }, + { url: "yaml://foo", expectedformat: codec.FormatYaml, expecteduri: "file://foo?format=yaml" }, + { url: "file://foo?format=json", expectedformat: codec.FormatJson, expecteduri: "file://foo?format=json" }, + { url: "file://foo.jx.json", expectedformat: codec.FormatJson, expecteduri: "file://foo.jx.json" }, + { url: "file://foo.jx.json.gz", expectedformat: codec.FormatJson, expecteduri: "file://foo.jx.json.gz" }, + { url: "https://foo.jx.json.gz", expectedformat: codec.FormatJson, expecteduri: "https://foo.jx.json.gz" }, + } { + j := NewJxFile() + assert.NotNil(t, j) + u,_ := url.Parse(v.url) + j.SetURI(u) + assert.Equal(t, v.expectedformat, j.Format) + assert.Equal(t, v.expecteduri, string(j.Uri)) + } +} + +func TestJxFactory(t *testing.T) { + converter, err := folio.DocumentRegistry.ConverterTypes.New("json://-") + assert.Nil(t, err) + assert.NotNil(t, converter) + assert.Equal(t, data.TypeName("jx"), converter.Type()) + jxfile := converter.(*JxFile) + assert.Equal(t, "-", jxfile.Path) + assert.Equal(t, codec.FormatJson, jxfile.Format) +} diff --git a/internal/fan/package.go b/internal/fan/package.go new file mode 100644 index 0000000..6cc1ec1 --- /dev/null +++ b/internal/fan/package.go @@ -0,0 +1,76 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( +_ "context" +_ "encoding/json" + "fmt" +_ "gopkg.in/yaml.v3" + "net/url" +_ "path/filepath" + "decl/internal/data" + "decl/internal/resource" + "decl/internal/folio" +_ "os" +_ "io" + "log/slog" +) + +type Package struct { + PackageType resource.PackageType `yaml:"type" json:"type"` +} + +func NewPackage() *Package { + return &Package{ PackageType: resource.SystemPackageType } +} + +func init() { + folio.DocumentRegistry.ConverterTypes.Register([]string{"package"}, func(u *url.URL) data.Converter { + p := NewPackage() + packageType := u.Query().Get("type") + if len(packageType) > 0 { + p.PackageType = resource.PackageType(packageType) + } + return p + }) + +} + +func (p *Package) Type() data.TypeName { return "package" } + +func (p *Package) Extract(sourceResource data.Resource, filter data.ElementSelector) (document data.Document, err error) { + slog.Info("fan.Package.Extract()", "package", p) + + installedPackages := make([]*resource.Package, 0, 100) + cmd := p.PackageType.NewReadPackagesCommand() + if cmd == nil { + return document, fmt.Errorf("%w: %s", resource.ErrUnsupportedPackageType, p.PackageType) + } + + var out []byte + if out, err = cmd.Execute(p); err == nil { + slog.Info("fan.Package.Extract()", "output", out) + if err = cmd.Extractor(out, &installedPackages); err == nil { + document = folio.DocumentRegistry.NewDocument("file://-") + for _, pkg := range installedPackages { + if pkg == nil { + pkg = resource.NewPackage() + } + if _, err = document.NewResource(pkg.URI()); err != nil { + return + } + } + } + } + slog.Info("fan.Package.Extract()", "output", out, "error", err) + return +} + +func (p *Package) Emit(document data.Document, filter data.ElementSelector) (resourceTarget data.Resource, err error) { + return nil, data.ErrUnsupportedConversion +} + +func (p *Package) Close() error { + return nil +} diff --git a/internal/source/package_test.go b/internal/fan/package_test.go similarity index 79% rename from internal/source/package_test.go rename to internal/fan/package_test.go index 334468a..c0b6c84 100644 --- a/internal/source/package_test.go +++ b/internal/fan/package_test.go @@ -1,6 +1,6 @@ // Copyright 2024 Matthew Rich . All rights reserved. -package source +package fan import ( "github.com/stretchr/testify/assert" @@ -16,8 +16,8 @@ func TestExtractPackages(t *testing.T) { p := NewPackage() assert.NotNil(t, p) - document, err := p.ExtractResources(nil) + document, err := p.Extract(nil, nil) assert.Nil(t, err) assert.NotNil(t, document) - assert.Greater(t, len(document), 0) + assert.Greater(t, document.Len(), 0) } diff --git a/internal/fan/tar.go b/internal/fan/tar.go new file mode 100644 index 0000000..6b8b736 --- /dev/null +++ b/internal/fan/tar.go @@ -0,0 +1,217 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( +_ "context" +_ "encoding/json" + "fmt" +_ "gopkg.in/yaml.v3" + "net/url" + "decl/internal/transport" + "decl/internal/data" + "decl/internal/folio" + "archive/tar" +_ "regexp" + "io" + "io/fs" + "log" + "log/slog" + "path/filepath" +) + +type Tar struct { + Uri folio.URI `yaml:"uri" json:"uri"` + parsedURI *url.URL `yaml:"-" json:"-"` + emitResource data.Resource `yaml:"-" json:"-"` + reader io.ReadCloser `yaml:"-" json:"-"` + writer io.WriteCloser `yaml:"-" json:"-"` + targetArchive *tar.Writer `yaml:"-" json:"-"` +} + +func NewTar() *Tar { + return &Tar{} +} + +func init() { + folio.DocumentRegistry.ConverterTypes.Register([]string{"tar"}, func(u *url.URL) data.Converter { + t := NewTar() + t.SetURI(u) + return t + }) + + folio.DocumentRegistry.ConverterTypes.RegisterContentType([]string{"tar", "tar.gz", "tgz"}, func(u *url.URL) data.Converter { + t := NewTar() + t.SetURI(u) + return t + }) + +} + +func (t *Tar) Type() data.TypeName { return "tar" } + +func (t *Tar) SetURI(u *url.URL) { + slog.Info("Tar.SetURI()", "tar", t) + u.Scheme = "file" + if u.Path == "" || u.Path == "-" { + } else { + fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.Path)) + u.Path = fileAbsolutePath + } + t.Uri.SetURL(u) + t.parsedURI = u + exttype, _ := t.Uri.Extension() + if exttype == "tgz" { + q := u.Query() + q.Set("gzip", string("true")) + u.RawQuery = q.Encode() + } +} + +func (t *Tar) targetResource() (target data.Resource, err error) { + if t.emitResource == nil { + + if t.emitResource, err = t.Uri.NewResource(nil); err != nil { + return nil, err + } + + var tarTargetFile data.FileResource = t.emitResource.(data.FileResource) + tarTargetFile.SetContentSourceRef(t.Uri.String()) + + t.writer, err = tarTargetFile.(data.ContentReadWriter).ContentWriterStream() + if err == io.EOF { + slog.Info("Tar.targetResource() ContentWriterStream", "target", tarTargetFile, "tar", t.writer.(*transport.Writer), "error", err) + panic(err) + } + t.targetArchive = tar.NewWriter(t.writer) + slog.Info("Tar.targetResource() SetContentSourceRef", "target", tarTargetFile, "uri", t.Uri.String(), "tar", t.targetArchive, "error", err) + } + target = t.emitResource + return +} + +// Convert a document of file resources to a tar file resource +func (t *Tar) Emit(document data.Document, filter data.ElementSelector) (resourceTarget data.Resource, err error) { + + resourceTarget, err = t.targetResource() + + slog.Info("Tar.Emit()", "writer", t.writer.(*transport.Writer), "error", err) + +/* + tarFile := resource.NewFile() + resourceTarget = tarFile + + tarFile.Path = t.Path + tarFile.ContentSourceRef = folio.ResourceReference(t.Path) + t.writer, err = tarFile.ContentSourceRef.ContentWriterStream() + targetArchive := tar.NewWriter(t.writer) + + defer t.writer.Close() +*/ + + for _,res := range document.Filter(func(d data.Declaration) bool { + return d.ResourceType() == "file" + }) { + + var f data.FileResource = res.(*folio.Declaration).Attributes.(data.FileResource) + + //f.PathNormalization(true) + //err = f.NormalizePath() + + fileInfo := f.FileInfo() + slog.Info("Tar.Emit() FileInfo", "fileinfo", fileInfo, "size", fileInfo.Size(), "file", f) + if fileInfo.Size() < 1 { + if len(f.GetContentSourceRef()) > 0 { + rs, _ := f.(data.ContentReader).ContentReaderStream() + info, _ := rs.Stat() + err = f.SetFileInfo(info) + slog.Info("Tar.Emit() Set FileInfo from ContentSourceRef", "fileinfo", f.FileInfo(), "file", f) + rs.Close() + } else { + if err = f.(data.Info).ReadStat(); err != nil { + return + } + } + } + + slog.Info("Tar.Emit", "file", f, "size", fileInfo.Size(), "error", err) + hdr, fiErr := tar.FileInfoHeader(fileInfo, "") + + if fileInfo.Mode() & fs.ModeSymlink != 0 { + hdr.Linkname = f.GetTarget() + } + + slog.Info("Tar.Emit", "header", hdr, "size", fileInfo.Size(), "err", fiErr) + if err := t.targetArchive.WriteHeader(hdr); err != nil { + slog.Error("Tar.Emit() WriteHeader", "target", t.targetArchive, "header", hdr, "resource", f, "fileinfo", fileInfo, "error", err) + log.Fatal(err) + } + + if fileInfo.IsDir() { + continue + } + + slog.Info("Tar.Emit - writing resource to target archive", "target", t.targetArchive, "resource", f, "err", err) + if _, err := f.GetContent(t.targetArchive); err != nil { + slog.Error("Tar.Emit() Content", "target", t.targetArchive, "resource", f, "fileinfo", fileInfo, "error", err) + log.Fatal(err) + } + slog.Info("Tar.Emit - wrote", "resource", f, "err", err) + } + + return +} + +// Convert a tar file resource to a document of file resources +func (t *Tar) Extract(resourceSource data.Resource, filter data.ElementSelector) (document data.Document, err error) { + document = folio.DocumentRegistry.NewDocument("") + var tarSourceFile data.FileResource = resourceSource.(data.FileResource) + //tarSourceFile := resourceSource.(*resource.File) + + t.reader, err = tarSourceFile.GetContent(nil) + sourceArchive := tar.NewReader(t.reader) + + defer t.reader.Close() + + for { + var hdr *tar.Header + hdr, err = sourceArchive.Next() + if err == io.EOF { + err = nil + break + } + if err != nil { + return + } + + var fileResource data.Resource + uri := fmt.Sprintf("file://%s", hdr.Name) + if fileResource, err = document.(*folio.Document).NewResource(uri); err != nil { + return + } + var f data.FileResource = fileResource.(data.FileResource) + + if err = f.SetFileInfo(hdr.FileInfo()); err != nil { + return + } + err = f.SetContent(sourceArchive) + if err != nil { + return + } + } + return +} + +func (t *Tar) Close() (err error) { + if t.reader != nil { + if err = t.reader.Close(); err != nil { + return + } + } + if err = t.targetArchive.Close(); err == nil { + if t.writer != nil { + err = t.writer.Close() + } + } + return +} diff --git a/internal/fan/tar_test.go b/internal/fan/tar_test.go new file mode 100644 index 0000000..7e833a2 --- /dev/null +++ b/internal/fan/tar_test.go @@ -0,0 +1,105 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package fan + +import ( + "github.com/stretchr/testify/assert" + "testing" + "bytes" + "archive/tar" + "decl/internal/data" + "decl/internal/folio" + "decl/internal/resource" + "path/filepath" + "strings" + "io" + "fmt" + "log/slog" +) + +var tarArchiveBuffer bytes.Buffer + +func TarArchive() (err error) { + tw := tar.NewWriter(&tarArchiveBuffer) + defer tw.Close() + + fileContent := "test file content" + + if err = tw.WriteHeader(&tar.Header{ + Name: "testfile", + Mode: 0600, + Size: int64(len(fileContent)), + }); err == nil { + _, err = tw.Write([]byte(fileContent)) + } + return +} + +func TestNewTar(t *testing.T) { + a := NewTar() + assert.NotNil(t, a) +} + +func TestExtractFiles(t *testing.T) { + a := NewTar() + assert.NotNil(t, a) + e := TarArchive() + assert.Nil(t, e) + assert.Greater(t, tarArchiveBuffer.Len(), 0) + + d := folio.NewDeclaration() + d.ResourceTypes = folio.DocumentRegistry.ResourceTypes + slog.Info("TestExtractFiles", "resourcetypes", folio.DocumentRegistry.ResourceTypes, "declarationtypes", d.ResourceTypes, "resource.ResourceTypes", resource.ResourceTypes) + d.Type = "file" + assert.Nil(t, d.NewResource(nil)) + + var sourceResource data.FileResource = d.Attributes.(data.FileResource) + assert.Nil(t, sourceResource.SetContent(&tarArchiveBuffer)) + + exDoc, err := a.Extract(d.Attributes, nil) + assert.Nil(t, err) + assert.NotNil(t, exDoc) + document := exDoc.(*folio.Document) + assert.Greater(t, document.Len(), 0) + + assert.Equal(t, folio.TypeName("file"), document.ResourceDeclarations[0].Type) + f := document.ResourceDeclarations[0].Resource().(data.FileResource) + assert.Equal(t, "testfile", f.FilePath()) +} + +func TestEmitFiles(t *testing.T) { + expected := "some test data" + + a := NewTar() + assert.NotNil(t, a) + + a.Uri = folio.URI(fmt.Sprintf("file://%s", filepath.Join(TempDir, "testemitfiles.tar"))) + + doc := folio.DocumentRegistry.NewDocument("") + + uri := fmt.Sprintf("file://%s", filepath.Join(TempDir, "foo.txt")) + res, resErr := doc.NewResource(uri) + assert.Nil(t, resErr) + assert.NotNil(t, res) + + assert.Equal(t, res, doc.GetResource(uri).Resource()) + f := doc.GetResource(uri).Attributes.(data.FileResource) + + assert.Nil(t, f.SetContent(strings.NewReader(expected))) + + target, emitErr := a.Emit(doc, nil) + assert.Nil(t, emitErr) + assert.Equal(t, folio.URI(fmt.Sprintf("file://%s", target.(data.FileResource).FilePath())), a.Uri) + + tarArchiveBuffer.Reset() + _, contentErr := target.(data.FileResource).GetContent(&tarArchiveBuffer) + assert.Nil(t, contentErr) + tr := tar.NewReader(&tarArchiveBuffer) + hdr, err := tr.Next() + assert.NotEqual(t, io.EOF, err) + assert.NotNil(t, hdr) + assert.Equal(t, f.FilePath(), hdr.Name) + data, err := io.ReadAll(tr) + assert.Nil(t, err) + assert.Equal(t, expected, string(data)) +} diff --git a/internal/source/user.go b/internal/fan/user.go similarity index 55% rename from internal/source/user.go rename to internal/fan/user.go index 2ed1a6d..835648d 100644 --- a/internal/source/user.go +++ b/internal/fan/user.go @@ -1,6 +1,6 @@ // Copyright 2024 Matthew Rich . All rights reserved. -package source +package fan import ( _ "context" @@ -10,6 +10,8 @@ _ "gopkg.in/yaml.v3" "net/url" _ "path/filepath" "decl/internal/resource" + "decl/internal/data" + "decl/internal/folio" _ "os" _ "io" "log/slog" @@ -24,7 +26,7 @@ func NewUser() *User { } func init() { - SourceTypes.Register([]string{"user"}, func(u *url.URL) DocSource { + folio.DocumentRegistry.ConverterTypes.Register([]string{"user"}, func(u *url.URL) data.Converter { userSource := NewUser() userType := u.Query().Get("type") if len(userType) > 0 { @@ -35,23 +37,21 @@ func init() { } -func (u *User) Type() string { return "user" } +func (u *User) Type() data.TypeName { return "user" } -func (u *User) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { - documents := make([]*resource.Document, 0, 100) - - slog.Info("user source ExtractResources()", "user", u) +func (u *User) Extract(sourceResource data.Resource, filter data.ElementSelector) (document data.Document, err error) { + slog.Info("user source Extract()", "user", u) Users := make([]*resource.User, 0, 100) cmd := u.UserType.NewReadUsersCommand() if cmd == nil { - return documents, resource.ErrUnsupportedUserType + return document, resource.ErrUnsupportedUserType } if out, err := cmd.Execute(u); err == nil { slog.Info("user source ExtractResources()", "output", out) if exErr := cmd.Extractor(out, &Users); exErr != nil { - return documents, exErr + return document, exErr } - document := resource.NewDocument() + document = folio.DocumentRegistry.NewDocument("user://-") for _, usr := range Users { if usr == nil { usr = resource.NewUser() @@ -59,10 +59,17 @@ func (u *User) ExtractResources(filter ResourceSelector) ([]*resource.Document, usr.UserType = u.UserType document.AddResourceDeclaration("user", usr) } - documents = append(documents, document) } else { - slog.Info("user source ExtractResources()", "output", out, "error", err) - return documents, err + slog.Info("user source Extract()", "output", out, "error", err) + return document, err } - return documents, nil + return document, nil +} + +func (u *User) Emit(document data.Document, filter data.ElementSelector) (resourceTarget data.Resource, err error) { + return nil, data.ErrUnsupportedConversion +} + +func (u *User) Close() error { + return nil } diff --git a/internal/source/user_test.go b/internal/fan/user_test.go similarity index 78% rename from internal/source/user_test.go rename to internal/fan/user_test.go index 404393e..45b0a54 100644 --- a/internal/source/user_test.go +++ b/internal/fan/user_test.go @@ -1,6 +1,6 @@ // Copyright 2024 Matthew Rich . All rights reserved. -package source +package fan import ( "github.com/stretchr/testify/assert" @@ -16,8 +16,8 @@ func TestExtractUsers(t *testing.T) { u := NewUser() assert.NotNil(t, u) - document, err := u.ExtractResources(nil) + document, err := u.Extract(nil, nil) assert.Nil(t, err) assert.NotNil(t, document) - assert.Greater(t, len(document), 0) + assert.Greater(t, document.Len(), 0) } diff --git a/internal/source/decl.go b/internal/source/decl.go deleted file mode 100644 index c55b723..0000000 --- a/internal/source/decl.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" - "net/url" - "path/filepath" - "decl/internal/resource" - "decl/internal/transport" - "decl/internal/codec" - "regexp" -_ "os" - "io" - "compress/gzip" - "errors" - "log/slog" -) - -type DeclFile struct { - Path string `yaml:"path" json:"path"` - transport *transport.Reader `yaml:"-" json:"-"` -} - -func NewDeclFile() *DeclFile { - return &DeclFile{} -} - -func init() { - SourceTypes.Register([]string{"decl"}, func(u *url.URL) DocSource { - t := NewDeclFile() - t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - t.transport,_ = transport.NewReader(u) - return t - }) - - SourceTypes.Register([]string{"yaml","yml","yaml.gz","yml.gz"}, func(u *url.URL) DocSource { - t := NewDeclFile() - if u.Scheme == "file" { - fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - t.Path = fileAbsolutePath - } else { - t.Path = filepath.Join(u.Hostname(), u.Path) - } - t.transport,_ = transport.NewReader(u) - return t - }) - -} - - -func (d *DeclFile) Type() string { return "decl" } - -func (d *DeclFile) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { - documents := make([]*resource.Document, 0, 100) - - GzipFileName := regexp.MustCompile(`^.*\.gz$`) - - defer d.transport.Close() - - var fileReader io.Reader - - if GzipFileName.FindString(d.Path) == d.Path { - slog.Info("decompressing gzip", "path", d.Path) - zr, err := gzip.NewReader(d.transport) - if err != nil { - return documents, err - } - fileReader = zr - } else { - fileReader = d.transport - } - - decoder := codec.NewYAMLDecoder(fileReader) - slog.Info("ExtractResources()", "documents", documents) - index := 0 - for { - doc := resource.NewDocument() - e := decoder.Decode(doc) - slog.Info("ExtractResources().Decode()", "document", doc, "error", e) - if errors.Is(e, io.EOF) { - break - } - if e != nil { - return documents, e - } - slog.Info("ExtractResources()", "res", doc.ResourceDecls[0].Attributes) - if validationErr := doc.Validate(); validationErr != nil { - return documents, validationErr - } - documents = append(documents, doc) - index++ - } - return documents, nil -} diff --git a/internal/source/dir.go b/internal/source/dir.go deleted file mode 100644 index 1dd6a29..0000000 --- a/internal/source/dir.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" - "net/url" - "path/filepath" - "decl/internal/resource" - "os" - "io" -) - -type Dir struct { - Path string `yaml:"path" json:"path"` - subDirsStack []string `yaml:"-" json:"-"` -} - -func NewDir() *Dir { - return &Dir{ - subDirsStack: make([]string, 0, 100), - } -} - -func init() { - SourceTypes.Register([]string{"file"}, func(u *url.URL) DocSource { - t := NewDir() - t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - return t - }) - -} - - -func (d *Dir) Type() string { return "dir" } - -func (d *Dir) ExtractDirectory(path string) (*resource.Document, error) { - document := resource.NewDocument() - files, err := os.ReadDir(path) - if err != nil { - return nil, err - } - - for _,file := range files { - f := resource.NewFile() - f.Path = filepath.Join(path, file.Name()) - info, infoErr := file.Info() - if infoErr != nil { - return document, infoErr - } - - if fiErr := f.UpdateAttributesFromFileInfo(info); fiErr != nil { - return document, fiErr - } - - f.FileType.SetMode(file.Type()) - - if file.IsDir() { - d.subDirsStack = append(d.subDirsStack, f.Path) - } else { - fileReader, fileReaderErr := os.Open(f.Path) - if fileReaderErr != nil { - return document, fileReaderErr - } - - readFileData, readErr := io.ReadAll(fileReader) - if readErr != nil { - return document, readErr - } - f.Content = string(readFileData) - f.UpdateContentAttributes() - } - - document.AddResourceDeclaration("file", f) - } - return document, nil -} - -func (d *Dir) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { - documents := make([]*resource.Document, 0, 100) - - d.subDirsStack = append(d.subDirsStack, d.Path) - - for { - if len(d.subDirsStack) == 0 { - break - } - var dirPath string - dirPath, d.subDirsStack = d.subDirsStack[len(d.subDirsStack) - 1], d.subDirsStack[:len(d.subDirsStack) - 1] - document, dirErr := d.ExtractDirectory(dirPath) - if dirErr != nil { - return documents, dirErr - } - - documents = append(documents, document) - } - return documents, nil -} diff --git a/internal/source/dir_test.go b/internal/source/dir_test.go deleted file mode 100644 index 19f8686..0000000 --- a/internal/source/dir_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestNewDirSource(t *testing.T) { - s := NewDir() - assert.NotNil(t, s) -} - -func TestExtractDirectory(t *testing.T) { - s := NewDir() - assert.NotNil(t, s) - - document, err := s.ExtractDirectory(TempDir) - assert.Nil(t, err) - assert.NotNil(t, document) - -} diff --git a/internal/source/docsource.go b/internal/source/docsource.go deleted file mode 100644 index bf715a0..0000000 --- a/internal/source/docsource.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" -_ "net/url" -_ "regexp" -_ "strings" -_ "os" -_ "io" -_ "compress/gzip" -_ "archive/tar" -_ "errors" -_ "path/filepath" - "decl/internal/resource" -_ "decl/internal/codec" -) - -type ResourceSelector func(r resource.Resource) bool - -type DocSource interface { - Type() string - - ExtractResources(filter ResourceSelector) ([]*resource.Document, error) -} - -func NewDocSource(uri string) DocSource { - s, e := SourceTypes.New(uri) - if e == nil { - return s - } - return nil -} diff --git a/internal/source/docsource_test.go b/internal/source/docsource_test.go deleted file mode 100644 index f75b0da..0000000 --- a/internal/source/docsource_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" -_ "fmt" - "github.com/stretchr/testify/assert" -_ "log" - "testing" - "os" - "log" -) - - -var TempDir string - -func TestMain(m *testing.M) { - var err error - TempDir, err = os.MkdirTemp("", "testdocsourcefile") - if err != nil || TempDir == "" { - log.Fatal(err) - } - - rc := m.Run() - - os.RemoveAll(TempDir) - os.Exit(rc) -} - -func TestNewDocSource(t *testing.T) { - resourceUri := "tar://foo" - testFile := NewDocSource(resourceUri) - assert.NotNil(t, testFile) -} - -/* -func TestResolveId(t *testing.T) { - testFile := NewResource("file://../../README.md") - assert.NotNil(t, testFile) - - absolutePath, e := filepath.Abs("../../README.md") - assert.Nil(t, e) - - testFile.ResolveId(context.Background()) - assert.Equal(t, absolutePath, testFile.(*File).Path) -} -*/ diff --git a/internal/source/http.go b/internal/source/http.go deleted file mode 100644 index 8d0dec5..0000000 --- a/internal/source/http.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" - "net/url" -_ "net/http" -_ "path/filepath" - "decl/internal/resource" - "decl/internal/iofilter" - "decl/internal/signature" - "decl/internal/transport" - "decl/internal/codec" -_ "os" - "io" - "errors" - "crypto/sha256" -) - -type HTTP struct { - Endpoint string `yaml:"endpoint" json:"endpoint"` - transport *transport.Reader `yaml:"-" json:"-"` -} - -func NewHTTP() *HTTP { - return &HTTP{} -} - -func init() { - SourceTypes.Register([]string{"http","https"}, func(u *url.URL) DocSource { - t := NewHTTP() - t.Endpoint = u.String() - t.transport,_ = transport.NewReader(u) - return t - }) -} - -func (d *HTTP) Type() string { return "http" } - -func (h *HTTP) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { - documents := make([]*resource.Document, 0, 100) - - defer h.transport.Close() - documentSignature := h.transport.Signature() - - hash := sha256.New() - sumReadData := iofilter.NewReader(h.transport, func(p []byte, readn int, readerr error) (n int, err error) { - hash.Write(p) - return - }) - - decoder := codec.NewYAMLDecoder(sumReadData) - index := 0 - for { - doc := resource.NewDocument() - e := decoder.Decode(doc) - if errors.Is(e, io.EOF) { - break - } - if e != nil { - return documents, e - } - if validationErr := doc.Validate(); validationErr != nil { - return documents, validationErr - } - documents = append(documents, doc) - index++ - } - - if documentSignature != "" { - sig := &signature.Ident{} - sigErr := sig.VerifySum(hash.Sum(nil), []byte(documentSignature)) - if sigErr != nil { - return documents, sigErr - } - } - - return documents, nil -} diff --git a/internal/source/iptable.go b/internal/source/iptable.go deleted file mode 100644 index f498815..0000000 --- a/internal/source/iptable.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" - "net/url" -_ "path/filepath" - "decl/internal/resource" -_ "os" -_ "io" - "strings" - "log/slog" -) - -type Iptable struct { - Table string `yaml:"table" json:"table"` - Chain string `yaml:"chain" json:"chain"` -} - -func NewIptable() *Iptable { - return &Iptable{} -} - -func init() { - SourceTypes.Register([]string{"iptable"}, func(u *url.URL) DocSource { - t := NewIptable() - t.Table = u.Hostname() - t.Chain = strings.Split(u.RequestURI(), "/")[1] - slog.Info("iptable chain source factory", "table", t, "uri", u, "table", u.Hostname()) - return t - }) - -} - - -func (i *Iptable) Type() string { return "iptable" } - -func (i *Iptable) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { - documents := make([]*resource.Document, 0, 100) - - slog.Info("iptable chain source ExtractResources()", "table", i) - iptRules := make([]*resource.Iptable, 0, 100) - cmd := resource.NewIptableReadChainCommand() - if out, err := cmd.Execute(i); err == nil { - slog.Info("iptable chain source ExtractResources()", "output", out) - if exErr := cmd.Extractor(out, &iptRules); exErr != nil { - return documents, exErr - } - document := resource.NewDocument() - for _, rule := range iptRules { - if rule == nil { - rule = resource.NewIptable() - } - rule.Table = resource.IptableName(i.Table) - rule.Chain = resource.IptableChain(i.Chain) - - document.AddResourceDeclaration("iptable", rule) - } - documents = append(documents, document) - } else { - slog.Info("iptable chain source ExtractResources()", "output", out, "error", err) - return documents, err - } - return documents, nil -} diff --git a/internal/source/package.go b/internal/source/package.go deleted file mode 100644 index fc1f716..0000000 --- a/internal/source/package.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" - "net/url" -_ "path/filepath" - "decl/internal/resource" -_ "os" -_ "io" - "log/slog" -) - -type Package struct { - PackageType resource.PackageType `yaml:"type" json:"type"` -} - -func NewPackage() *Package { - return &Package{ PackageType: resource.SystemPackageType } -} - -func init() { - SourceTypes.Register([]string{"package"}, func(u *url.URL) DocSource { - p := NewPackage() - packageType := u.Query().Get("type") - if len(packageType) > 0 { - p.PackageType = resource.PackageType(packageType) - } - return p - }) - -} - -func (p *Package) Type() string { return "package" } - -func (p *Package) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { - documents := make([]*resource.Document, 0, 100) - - slog.Info("package source ExtractResources()", "package", p) - installedPackages := make([]*resource.Package, 0, 100) - cmd := p.PackageType.NewReadPackagesCommand() - if cmd == nil { - return documents, resource.ErrUnsupportedPackageType - } - if out, err := cmd.Execute(p); err == nil { - slog.Info("package source ExtractResources()", "output", out) - if exErr := cmd.Extractor(out, &installedPackages); exErr != nil { - return documents, exErr - } - document := resource.NewDocument() - for _, pkg := range installedPackages { - if pkg == nil { - pkg = resource.NewPackage() - } - pkg.PackageType = p.PackageType - document.AddResourceDeclaration("package", pkg) - } - documents = append(documents, document) - } else { - slog.Info("package source ExtractResources()", "output", out, "error", err) - return documents, err - } - return documents, nil -} diff --git a/internal/source/tar.go b/internal/source/tar.go deleted file mode 100644 index f69d4cd..0000000 --- a/internal/source/tar.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" - "net/url" - "path/filepath" - "decl/internal/resource" - "decl/internal/transport" - "compress/gzip" - "archive/tar" - "regexp" -_ "os" - "io" -) - -type Tar struct { - Path string `yaml:"path" json:"path"` - transport *transport.Reader `yaml:"-" json:"-"` -} - -func NewTar() *Tar { - return &Tar{} -} - -func init() { - SourceTypes.Register([]string{"tar"}, func(u *url.URL) DocSource { - t := NewTar() - t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - t.transport,_ = transport.NewReader(u) - return t - }) - - SourceTypes.Register([]string{"tar.gz", "tgz"}, func(u *url.URL) DocSource { - t := NewTar() - if u.Scheme == "file" { - fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - t.Path = fileAbsolutePath - } else { - t.Path = filepath.Join(u.Hostname(), u.Path) - } - t.transport,_ = transport.NewReader(u) - return t - }) - -} - - -func (t *Tar) Type() string { return "tar" } - -func (t *Tar) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { - documents := make([]*resource.Document, 0, 100) - d := resource.NewDocument() - documents = append(documents, d) - - TarGzipFileName := regexp.MustCompile(`^.*\.(tar\.gz|tgz)$`) - TarFileName := regexp.MustCompile(`^.*\.tar$`) - - defer t.transport.Close() - - var gzipReader io.Reader - switch t.Path { - case TarGzipFileName.FindString(t.Path): - zr, err := gzip.NewReader(t.transport) - if err != nil { - return documents, err - } - gzipReader = zr - fallthrough - case TarFileName.FindString(t.Path): - var fileReader io.Reader - if gzipReader == nil { - fileReader = t.transport - } else { - fileReader = gzipReader - } - tarReader := tar.NewReader(fileReader) - for { - hdr, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return documents, err - } - f := resource.NewFile() - f.Path = hdr.Name - if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil { - return documents, fiErr - } - readErr := f.SetContent(tarReader) - if readErr != nil { - return documents, readErr - } - d.AddResourceDeclaration("file", f) - } - } - return documents, nil -} diff --git a/internal/source/tar_test.go b/internal/source/tar_test.go deleted file mode 100644 index decc0f2..0000000 --- a/internal/source/tar_test.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestNewTarSource(t *testing.T) { - s := NewTar() - assert.NotNil(t, s) -} - diff --git a/internal/source/types.go b/internal/source/types.go deleted file mode 100644 index caf7830..0000000 --- a/internal/source/types.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( - "errors" - "fmt" -_ "net/url" - "strings" -_ "path/filepath" - "decl/internal/types" -) - -var ( - ErrUnknownSourceType = errors.New("Unknown source type") - SourceTypes *types.Types[DocSource] = types.New[DocSource]() -) - -type TypeName string //`json:"type"` - -func (n *TypeName) UnmarshalJSON(b []byte) error { - SourceTypeName := strings.Trim(string(b), "\"") - if SourceTypes.Has(SourceTypeName) { - *n = TypeName(SourceTypeName) - return nil - } - return fmt.Errorf("%w: %s", ErrUnknownSourceType, SourceTypeName) -} diff --git a/internal/source/types_test.go b/internal/source/types_test.go deleted file mode 100644 index 353a99f..0000000 --- a/internal/source/types_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package source - -import ( -_ "context" - "encoding/json" - "github.com/stretchr/testify/assert" - "net/url" - "testing" - "decl/internal/resource" -) - -type MockDocSource struct { - InjectType func() string - InjectExtractResources func(filter ResourceSelector) ([]*resource.Document, error) -} - -func (m *MockDocSource) Type() string { return m.InjectType() } -func (m *MockDocSource) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { return m.InjectExtractResources(filter) } - -func NewFooDocSource() DocSource { - return &MockDocSource{ - InjectType: func() string { return "foo" }, - InjectExtractResources: func(filter ResourceSelector) ([]*resource.Document, error) { return nil,nil }, - } -} - -func NewFileDocSource() DocSource { - return &MockDocSource{ - InjectType: func() string { return "file" }, - InjectExtractResources: func(filter ResourceSelector) ([]*resource.Document, error) { return nil,nil }, - } -} - -func TestDocSourceTypeName(t *testing.T) { - SourceTypes.Register([]string{"file"}, func(*url.URL) DocSource { return NewFileDocSource() }) - - type fDocSourceName struct { - Name TypeName `json:"type"` - } - fTypeName := &fDocSourceName{} - jsonType := `{ "type": "file" }` - e := json.Unmarshal([]byte(jsonType), &fTypeName) - assert.Nil(t, e) - assert.Equal(t, "file", string(fTypeName.Name)) -} diff --git a/internal/target/decl.go b/internal/target/decl.go deleted file mode 100644 index 5e98029..0000000 --- a/internal/target/decl.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package target - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" - "net/url" - "path/filepath" - "decl/internal/resource" - "decl/internal/codec" - "os" - "compress/gzip" - "io" -_ "errors" - "log/slog" -) - -const ( - FormatYaml = "yaml" - FormatJson = "json" -) - -type DeclFile struct { - Path string `yaml:"path" json:"path"` - Gzip bool `yaml:"gzip,omitempty" json:"gzip,omitempty"` - Format string `yaml:"format,omitempty" json:"format,omitempty"` - encoder codec.Encoder `yaml:"-" json:"-"` - closer func() error `yaml:"-" json:"-"` -} - -func NewDeclFile() *DeclFile { - return &DeclFile{ Gzip: false, closer: func() error { return nil } } -} - -func NewFileDocTarget(u *url.URL, format string, gzip bool, fileUri bool) DocTarget { - t := NewDeclFile() - t.Format = format - t.Gzip = gzip - if fileUri { - fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - t.Path = fileAbsolutePath - } else { - t.Path = filepath.Join(u.Hostname(), u.Path) - } - if e := t.Open(); e != nil { - return nil - } - return t -} - -func init() { - TargetTypes.Register([]string{"decl", "file"}, func(u *url.URL) DocTarget { - t := NewDeclFile() - if u.Path != "-" { - t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - } else { - t.Path = "-" - } - if _,ok := u.Query()["gzip"]; ok { - t.Gzip = true - } - if format,ok := u.Query()["format"]; ok { - switch format[0] { - case string(FormatYaml): - t.Format = FormatYaml - case string(FormatJson): - t.Format = FormatJson - } - } - if e := t.Open(); e != nil { - return nil - } - return t - }) - - TargetTypes.Register([]string{"yaml.gz","yml.gz"}, func(u *url.URL) DocTarget { - switch u.Scheme { - case "yaml", "yml", "file": - return NewFileDocTarget(u, FormatYaml, true, false) - } - return NewFileDocTarget(u, FormatYaml, true, false) - }) - - TargetTypes.Register([]string{"json.gz"}, func(u *url.URL) DocTarget { - switch u.Scheme { - case "json", "file": - return NewFileDocTarget(u, FormatJson, true, false) - } - return NewFileDocTarget(u, FormatJson, true, false) - }) - - TargetTypes.Register([]string{"yaml","yml"}, func(u *url.URL) DocTarget { - switch u.Scheme { - case "yaml", "yml", "file": - return NewFileDocTarget(u, FormatYaml, false, false) - } - return NewFileDocTarget(u, FormatYaml, false, false) - }) - - TargetTypes.Register([]string{"json"}, func(u *url.URL) DocTarget { - switch u.Scheme { - case "json", "file": - return NewFileDocTarget(u, FormatJson, false, false) - } - return NewFileDocTarget(u, FormatJson, false, false) - }) - -} - -func (d *DeclFile) Open() error { - var file *os.File - var fileErr error - var fileWriter io.WriteCloser - if d.Path == "" || d.Path == "-" { - file = os.Stdout - } else { - file, fileErr = os.Open(d.Path) - if fileErr != nil { - return fileErr - } - d.closer = func() error { - d.encoder.Close() - fileWriter.Close() - if file != fileWriter { - file.Close() - } - return nil - } - } - - if d.Gzip { - fileWriter = gzip.NewWriter(file) - } else { - fileWriter = file - } - - switch d.Format { - case FormatJson: - d.encoder = codec.NewJSONEncoder(fileWriter) - case FormatYaml: - fallthrough - default: - d.encoder = codec.NewYAMLEncoder(fileWriter) - } - - return nil -} - -func (d *DeclFile) Close() error { - return d.closer() -} - -func (d *DeclFile) Type() string { return "decl" } - -func (d *DeclFile) EmitResources(documents []*resource.Document, filter resource.ResourceSelector) (error) { - for _, doc := range documents { - emitDoc := resource.NewDocument() - if validationErr := doc.Validate(); validationErr != nil { - return validationErr - } - for _, declaration := range doc.Filter(filter) { - emitDoc.ResourceDecls = append(emitDoc.ResourceDecls, *declaration) - } - slog.Info("EmitResources", "doctarget", d, "encoder", d.encoder, "emit", emitDoc) - if documentErr := d.encoder.Encode(emitDoc); documentErr != nil { - slog.Info("EmitResources", "err", documentErr) - return documentErr - } - } - return nil -} diff --git a/internal/target/doctarget.go b/internal/target/doctarget.go deleted file mode 100644 index fb77e30..0000000 --- a/internal/target/doctarget.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package target - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" -_ "net/url" -_ "regexp" -_ "strings" -_ "os" -_ "io" - "decl/internal/resource" -) - -// convert a document into some other container type - -// move selector to resource pkg -// type ResourceSelector func(r resource.Resource) bool - -type DocTarget interface { - Type() string - - EmitResources(documents []*resource.Document, filter resource.ResourceSelector) error - Close() error -} - -func NewDocTarget(uri string) DocTarget { - s, e := TargetTypes.New(uri) - if e == nil { - return s - } - return nil -} diff --git a/internal/target/tar.go b/internal/target/tar.go deleted file mode 100644 index 66c1090..0000000 --- a/internal/target/tar.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package target - -import ( -_ "context" -_ "encoding/json" -_ "fmt" -_ "gopkg.in/yaml.v3" - "net/url" - "path/filepath" - "decl/internal/resource" - "compress/gzip" - "archive/tar" -_ "regexp" - "os" - "io" - "log" - "log/slog" -) - -type Tar struct { - Path string `yaml:"path" json:"path"` - Gzip bool `yaml:"gzip" json:"gzip"` - writer *tar.Writer `yaml:"-" json:"-"` - closer func() error `yaml:"-" json:"-"` -} - -func NewTar() *Tar { - return &Tar{ Gzip: false, closer: func() error { return nil } } -} - -func init() { - TargetTypes.Register([]string{"tar"}, func(u *url.URL) DocTarget { - t := NewTar() - t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - if e := t.Open(); e != nil { - return nil - } - return t - }) - - TargetTypes.Register([]string{"tar.gz", "tgz"}, func(u *url.URL) DocTarget { - t := NewTar() - if u.Scheme == "file" { - fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.Path)) - t.Path = fileAbsolutePath - } else { - t.Path = filepath.Join(u.Hostname(), u.Path) - } - t.Gzip = true - if e := t.Open(); e != nil { - return nil - } - return t - }) - -} - -func (t *Tar) Open() error { - file, fileErr := os.Create(t.Path) - if fileErr != nil { - return fileErr - } - var fileWriter io.WriteCloser - if t.Gzip { - fileWriter = gzip.NewWriter(file) - } else { - fileWriter = file - } - - t.writer = tar.NewWriter(fileWriter) - t.closer = func() error { - t.writer.Close() - fileWriter.Close() - return file.Close() - } - return nil -} - -func (t *Tar) Close() error { - return t.closer() -} - -func (t *Tar) Type() string { return "tar" } - -func (t *Tar) EmitResources(documents []*resource.Document, filter resource.ResourceSelector) error { - for _,document := range documents { - for _,res := range document.Filter(func(d *resource.Declaration) bool { - return d.Type == "file" - }) { - var f *resource.File = res.Attributes.(*resource.File) - slog.Info("Tar.EmitResources", "file", f) - hdr, fiErr := tar.FileInfoHeader(f.FileInfo(), "") - slog.Info("Tar.EmitResources", "header", hdr, "err", fiErr) - if err := t.writer.WriteHeader(hdr); err != nil { - log.Fatal(err) - } - if _, err := t.writer.Write([]byte(f.Content)); err != nil { - log.Fatal(err) - } - } - } - return nil -} diff --git a/internal/target/tar_test.go b/internal/target/tar_test.go deleted file mode 100644 index 5bed971..0000000 --- a/internal/target/tar_test.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package target - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestNewTarSource(t *testing.T) { - s := NewTar() - assert.NotNil(t, s) -} - diff --git a/internal/target/types.go b/internal/target/types.go deleted file mode 100644 index 3e05646..0000000 --- a/internal/target/types.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package target - -import ( - "errors" - "fmt" - "net/url" - "strings" - "path/filepath" - "log/slog" -) - -var ( - ErrUnknownTargetType = errors.New("Unknown target type") - TargetTypes *Types = NewTypes() -) - -type TypeName string //`json:"type"` - -type TypeFactory func(*url.URL) DocTarget - -type Types struct { - registry map[string]TypeFactory -} - -func NewTypes() *Types { - return &Types{registry: make(map[string]TypeFactory)} -} - -func (t *Types) Register(names []string, factory TypeFactory) { - for _,name := range names { - t.registry[name] = factory - } -} - -func (t *Types) FromExtension(path string) (TypeFactory, error) { - elements := strings.Split(path, ".") - numberOfElements := len(elements) - if numberOfElements > 2 { - if src := t.Get(strings.Join(elements[numberOfElements - 2: numberOfElements - 1], ".")); src != nil { - return src, nil - } - } - if src := t.Get(elements[numberOfElements - 1]); src != nil { - return src, nil - } - return nil, fmt.Errorf("%w: %s", ErrUnknownTargetType, path) -} - -func (t *Types) New(uri string) (DocTarget, error) { - if uri == "" { - uri = "file://-" - } - - u, e := url.Parse(uri) - if u == nil || e != nil { - return nil, fmt.Errorf("%w: %s", ErrUnknownTargetType, e) - } - - if u.Scheme == "" { - u.Scheme = "file" - } - - path := filepath.Join(u.Hostname(), u.Path) - if d, lookupErr := t.FromExtension(path); d != nil { - slog.Info("Target.New", "target", t, "err", lookupErr) - return d(u), lookupErr - } else { - slog.Info("Target.New", "target", t, "err", lookupErr) - } - - if r, ok := t.registry[u.Scheme]; ok { - return r(u), nil - } - return nil, fmt.Errorf("%w: %s", ErrUnknownTargetType, u.Scheme) -} - -func (t *Types) Has(typename string) bool { - if _, ok := t.registry[typename]; ok { - return true - } - return false -} - -func (t *Types) Get(typename string) TypeFactory { - if d, ok := t.registry[typename]; ok { - return d - } - return nil -} - -func (n *TypeName) UnmarshalJSON(b []byte) error { - TargetTypeName := strings.Trim(string(b), "\"") - if TargetTypes.Has(TargetTypeName) { - *n = TypeName(TargetTypeName) - return nil - } - return fmt.Errorf("%w: %s", ErrUnknownTargetType, TargetTypeName) -} diff --git a/internal/target/types_test.go b/internal/target/types_test.go deleted file mode 100644 index 370b928..0000000 --- a/internal/target/types_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2024 Matthew Rich . All rights reserved. - -package target - -import ( -_ "context" - "encoding/json" - "github.com/stretchr/testify/assert" - "net/url" - "testing" - "decl/internal/resource" -) - -type MockDocTarget struct { - InjectType func() string - InjectEmitResources func(documents []*resource.Document, filter resource.ResourceSelector) error -} - -func (m *MockDocTarget) Type() string { return m.InjectType() } -func (m *MockDocTarget) Close() error { return nil } -func (m *MockDocTarget) EmitResources(documents []*resource.Document, filter resource.ResourceSelector) error { return m.InjectEmitResources(documents, filter) } - -func NewFooDocTarget() DocTarget { - return &MockDocTarget{ - InjectType: func() string { return "foo" }, - InjectEmitResources: func(documents []*resource.Document, filter resource.ResourceSelector) error { return nil }, - } -} - -func NewMockFileDocTarget() DocTarget { - return &MockDocTarget{ - InjectType: func() string { return "file" }, - InjectEmitResources: func(documents []*resource.Document, filter resource.ResourceSelector) error { return nil }, - } -} - -func TestNewTargetTypes(t *testing.T) { - targetTypes := NewTypes() - assert.NotNil(t, targetTypes) -} - -func TestNewTargetTypesRegister(t *testing.T) { - m := NewFooDocTarget() - - targetTypes := NewTypes() - assert.NotNil(t, targetTypes) - - targetTypes.Register([]string{"foo"}, func(*url.URL) DocTarget { return m }) - - r, e := targetTypes.New("foo://") - assert.Nil(t, e) - assert.Equal(t, m, r) -} - -func TestResourceTypesFromURI(t *testing.T) { - m := NewFooDocTarget() - - targetTypes := NewTypes() - assert.NotNil(t, targetTypes) - - targetTypes.Register([]string{"foo"}, func(*url.URL) DocTarget { return m }) - - r, e := targetTypes.New("foo://bar") - assert.Nil(t, e) - assert.Equal(t, m, r) -} - -func TestResourceTypesHasType(t *testing.T) { - m := NewFooDocTarget() - - targetTypes := NewTypes() - assert.NotNil(t, targetTypes) - - targetTypes.Register([]string{"foo"}, func(*url.URL) DocTarget { return m }) - - assert.True(t, targetTypes.Has("foo")) -} - -func TestDocTargetTypeName(t *testing.T) { - TargetTypes.Register([]string{"file"}, func(*url.URL) DocTarget { return NewMockFileDocTarget() }) - - type fDocTargetName struct { - Name TypeName `json:"type"` - } - fTypeName := &fDocTargetName{} - jsonType := `{ "type": "file" }` - e := json.Unmarshal([]byte(jsonType), &fTypeName) - assert.Nil(t, e) - assert.Equal(t, "file", string(fTypeName.Name)) -}