From 976f654c60a475c36ea127367d027e07c789d330 Mon Sep 17 00:00:00 2001 From: Matthew Rich Date: Wed, 17 Apr 2024 17:07:12 -0700 Subject: [PATCH] handle numeric user ids --- go.mod | 10 ++- go.sum | 20 +++++ internal/resource/os.go | 43 +++++++++-- internal/resource/os_test.go | 14 ++-- internal/source/decl.go | 100 ++++++++++++++++++++++++ internal/source/dir.go | 100 ++++++++++++++++++++++++ internal/source/dir_test.go | 23 ++++++ internal/source/docsource.go | 121 ++++++++++++++++++++++++++++++ internal/source/docsource_test.go | 30 ++++++++ internal/source/http.go | 88 ++++++++++++++++++++++ internal/source/http_test.go | 13 ++++ internal/source/tar.go | 102 +++++++++++++++++++++++++ internal/source/tar_test.go | 14 ++++ internal/source/types.go | 92 +++++++++++++++++++++++ internal/source/types_test.go | 89 ++++++++++++++++++++++ 15 files changed, 844 insertions(+), 15 deletions(-) create mode 100644 internal/source/decl.go create mode 100644 internal/source/dir.go create mode 100644 internal/source/dir_test.go create mode 100644 internal/source/docsource.go create mode 100644 internal/source/docsource_test.go create mode 100644 internal/source/http.go create mode 100644 internal/source/http_test.go create mode 100644 internal/source/tar.go create mode 100644 internal/source/tar_test.go create mode 100644 internal/source/types.go create mode 100644 internal/source/types_test.go diff --git a/go.mod b/go.mod index e36190a..5e96dcf 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module decl go 1.21.1 -require github.com/stretchr/testify v1.8.4 +require github.com/stretchr/testify v1.9.0 require ( github.com/Microsoft/go-winio v0.4.14 // indirect @@ -11,14 +11,19 @@ require ( github.com/docker/docker v25.0.5+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-yaml v1.11.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sters/yaml-diff v1.3.2 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -26,6 +31,7 @@ require ( go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/sys v0.1.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6a23ffc..7bf3367 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -20,11 +22,18 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I= +github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -36,12 +45,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sters/yaml-diff v1.3.2 h1:99Ke50QYFQYZjKMOiePxwyuQ+WeCvNy6cRooqdLs/ZE= +github.com/sters/yaml-diff v1.3.2/go.mod h1:86usbNZiUqke5wYjMxDVEjmvGjmY2FkMwOwe0A5zf68= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -75,8 +88,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -87,7 +104,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/resource/os.go b/internal/resource/os.go index 1089db9..29d0cc7 100644 --- a/internal/resource/os.go +++ b/internal/resource/os.go @@ -5,8 +5,11 @@ package resource import ( "os/user" "strconv" + "regexp" ) +var MatchId *regexp.Regexp = regexp.MustCompile(`^[0-9]+$`) + func LookupUIDString(userName string) string { user, userLookupErr := user.Lookup(userName) if userLookupErr != nil { @@ -16,12 +19,24 @@ func LookupUIDString(userName string) string { } func LookupUID(userName string) (int, error) { - user, userLookupErr := user.Lookup(userName) - if userLookupErr != nil { - return -1, userLookupErr + var UID string + if MatchId.MatchString(userName) { + user, userLookupErr := user.LookupId(userName) + if userLookupErr != nil { + //return -1, userLookupErr + UID = userName + } else { + UID = user.Uid + } + } else { + user, userLookupErr := user.Lookup(userName) + if userLookupErr != nil { + return -1, userLookupErr + } + UID = user.Uid } - uid, uidErr := strconv.Atoi(user.Uid) + uid, uidErr := strconv.Atoi(UID) if uidErr != nil { return -1, uidErr } @@ -30,12 +45,24 @@ func LookupUID(userName string) (int, error) { } func LookupGID(groupName string) (int, error) { - group, groupLookupErr := user.LookupGroup(groupName) - if groupLookupErr != nil { - return -1, groupLookupErr + var GID string + if MatchId.MatchString(groupName) { + group, groupLookupErr := user.LookupGroupId(groupName) + if groupLookupErr != nil { + //return -1, groupLookupErr + GID = groupName + } else { + GID = group.Gid + } + } else { + group, groupLookupErr := user.LookupGroup(groupName) + if groupLookupErr != nil { + return -1, groupLookupErr + } + GID = group.Gid } - gid, gidErr := strconv.Atoi(group.Gid) + gid, gidErr := strconv.Atoi(GID) if gidErr != nil { return -1, gidErr } diff --git a/internal/resource/os_test.go b/internal/resource/os_test.go index 62a874d..a727350 100644 --- a/internal/resource/os_test.go +++ b/internal/resource/os_test.go @@ -17,17 +17,21 @@ import ( func TestLookupUID(t *testing.T) { uid, e := LookupUID("nobody") - assert.Equal(t, nil, e) + assert.Nil(t, e) assert.Equal(t, 65534, uid) + + nuid, ne := LookupUID("1001") + assert.Nil(t, ne) + assert.Equal(t, 1001, nuid) } func TestLookupGID(t *testing.T) { gid, e := LookupGID("nobody") - assert.Equal(t, nil, e) + assert.Nil(t, e) assert.Equal(t, 65534, gid) -} - -func TestExecCommand(t *testing.T) { + ngid, ne := LookupGID("1001") + assert.Nil(t, ne) + assert.Equal(t, 1001, ngid) } diff --git a/internal/source/decl.go b/internal/source/decl.go new file mode 100644 index 0000000..8008372 --- /dev/null +++ b/internal/source/decl.go @@ -0,0 +1,100 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( +_ "context" +_ "encoding/json" +_ "fmt" +_ "gopkg.in/yaml.v3" + "net/url" + "path/filepath" + "decl/internal/resource" + "regexp" + "os" + "io" + "compress/gzip" + "errors" + "log/slog" +) + +type DeclFile struct { + Path string `yaml:"path" json:"path"` +} + +func NewDeclFile() *DeclFile { + return &DeclFile{} +} + +func init() { + SourceTypes.Register([]string{"decl"}, func(u *url.URL) DocSource { + t := NewDeclFile() + t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI())) + return t + }) + + SourceTypes.Register([]string{"yaml","yml","yaml.gz","yml.gz"}, func(u *url.URL) DocSource { + t := NewDeclFile() + if u.Scheme == "file" { + fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI())) + t.Path = fileAbsolutePath + } else { + t.Path = filepath.Join(u.Hostname(), u.Path) + } + return t + }) + +} + + +func (d *DeclFile) Type() string { return "decl" } + +func (d *DeclFile) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { + documents := make([]*resource.Document, 0, 100) + documents = append(documents, resource.NewDocument()) + + GzipFileName := regexp.MustCompile(`^.*\.gz$`) + + file, fileErr := os.Open(d.Path) + if fileErr != nil { + return documents, fileErr + } + var fileReader io.Reader + + if GzipFileName.FindString(d.Path) == d.Path { + slog.Info("decompressing gzip", "path", d.Path) + zr, err := gzip.NewReader(file) + if err != nil { + return documents, err + } + fileReader = zr + } else { + fileReader = file + } + decoder := resource.NewYAMLDecoder(fileReader) + index := 0 + for { + doc := documents[index] + e := decoder.Decode(doc) + if errors.Is(e, io.EOF) { + if len(documents) > 1 { + documents[index] = nil + } + break + } + if e != nil { + return documents, e + } + if validationErr := doc.Validate(); validationErr != nil { + return documents, validationErr + } +/* + if applyErr := doc.Apply(); applyErr != nil { + return documents, applyErr + } +*/ + documents = append(documents, resource.NewDocument()) + index++ + } + return documents, nil +} diff --git a/internal/source/dir.go b/internal/source/dir.go new file mode 100644 index 0000000..2da88fe --- /dev/null +++ b/internal/source/dir.go @@ -0,0 +1,100 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( +_ "context" +_ "encoding/json" +_ "fmt" +_ "gopkg.in/yaml.v3" + "net/url" + "path/filepath" + "decl/internal/resource" + "os" + "io" +) + +type Dir struct { + Path string `yaml:"path" json:"path"` + subDirsStack []string `yaml:"-" json:"-"` +} + +func NewDir() *Dir { + return &Dir{ + subDirsStack: make([]string, 0, 100), + } +} + +func init() { + SourceTypes.Register([]string{"file"}, func(u *url.URL) DocSource { + t := NewDir() + t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI())) + return t + }) + +} + + +func (d *Dir) Type() string { return "dir" } + +func (d *Dir) ExtractDirectory(path string) (*resource.Document, error) { + document := resource.NewDocument() + files, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + for _,file := range files { + f := resource.NewFile() + f.Path = filepath.Join(path, file.Name()) + info, infoErr := file.Info() + if infoErr != nil { + return document, infoErr + } + + if fiErr := f.UpdateAttributesFromFileInfo(info); fiErr != nil { + return document, fiErr + } + + f.FileType.SetMode(file.Type()) + + if file.IsDir() { + d.subDirsStack = append(d.subDirsStack, f.Path) + } else { + fileReader, fileReaderErr := os.Open(f.Path) + if fileReaderErr != nil { + return document, fileReaderErr + } + + readFileData, readErr := io.ReadAll(fileReader) + if readErr != nil { + return document, readErr + } + f.Content = string(readFileData) + } + + document.AddResourceDeclaration("file", f) + } + return document, nil +} + +func (d *Dir) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { + documents := make([]*resource.Document, 0, 100) + + d.subDirsStack = append(d.subDirsStack, d.Path) + + for { + if len(d.subDirsStack) == 0 { + break + } + var dirPath string + dirPath, d.subDirsStack = d.subDirsStack[len(d.subDirsStack) - 1], d.subDirsStack[:len(d.subDirsStack) - 1] + document, dirErr := d.ExtractDirectory(dirPath) + if dirErr != nil { + return documents, dirErr + } + + documents = append(documents, document) + } + return documents, nil +} diff --git a/internal/source/dir_test.go b/internal/source/dir_test.go new file mode 100644 index 0000000..19f8686 --- /dev/null +++ b/internal/source/dir_test.go @@ -0,0 +1,23 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewDirSource(t *testing.T) { + s := NewDir() + assert.NotNil(t, s) +} + +func TestExtractDirectory(t *testing.T) { + s := NewDir() + assert.NotNil(t, s) + + document, err := s.ExtractDirectory(TempDir) + assert.Nil(t, err) + assert.NotNil(t, document) + +} diff --git a/internal/source/docsource.go b/internal/source/docsource.go new file mode 100644 index 0000000..41197c2 --- /dev/null +++ b/internal/source/docsource.go @@ -0,0 +1,121 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( +_ "context" +_ "encoding/json" +_ "fmt" +_ "gopkg.in/yaml.v3" + "net/url" + "regexp" +_ "strings" + "os" + "io" + "compress/gzip" + "archive/tar" + "errors" + "path/filepath" + "decl/internal/resource" +) + +type ResourceSelector func(r resource.Resource) bool + +type DocSource interface { + Type() string + + ExtractResources(filter ResourceSelector) ([]*resource.Document, error) +} + +func NewDocSource(uri string) DocSource { + s, e := SourceTypes.New(uri) + if e == nil { + return s + } + return nil +} + +func ExtractResources(uri string, filter ResourceSelector) ([]*resource.Document, error) { + documents := make([]*resource.Document, 0, 100) + d := resource.NewDocument() + documents = append(documents, d) + + TarGzipFileName := regexp.MustCompile(`^.*\.(tar\.gz|tgz)$`) + TarFileName := regexp.MustCompile(`^.*\.tar$`) + + u,e := url.Parse(uri) + if e != nil { + return nil, e + } + switch u.Scheme { + case "file": + fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI())) + file, fileErr := os.Open(fileAbsolutePath) + if fileErr != nil { + return documents, fileErr + } + var gzipReader io.Reader + switch u.Path { + case TarGzipFileName.FindString(u.Path): + zr, err := gzip.NewReader(file) + if err != nil { + return documents, err + } + gzipReader = zr + fallthrough + case TarFileName.FindString(u.Path): + var fileReader io.Reader + if gzipReader == nil { + fileReader = file + } else { + fileReader = gzipReader + } + tarReader := tar.NewReader(fileReader) + for { + hdr, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return documents, err + } + f := resource.NewFile() + if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil { + return documents, fiErr + } + readFileData, readErr := io.ReadAll(tarReader) + if readErr != nil { + return documents, readErr + } + f.Content = string(readFileData) + d.AddResourceDeclaration("file", f) + } + default: + decoder := resource.NewYAMLDecoder(file) + index := 0 + for { + doc := documents[index] + e := decoder.Decode(doc) + if errors.Is(e, io.EOF) { + if len(documents) > 1 { + documents[index] = nil + } + break + } + if e != nil { + return documents, e + } + if validationErr := doc.Validate(); validationErr != nil { + return documents, validationErr + } + if applyErr := doc.Apply(); applyErr != nil { + return documents, applyErr + } + documents = append(documents, resource.NewDocument()) + index++ + } + + } + } + return documents, nil +} diff --git a/internal/source/docsource_test.go b/internal/source/docsource_test.go new file mode 100644 index 0000000..20083ef --- /dev/null +++ b/internal/source/docsource_test.go @@ -0,0 +1,30 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( +_ "context" +_ "fmt" + "github.com/stretchr/testify/assert" +_ "log" + "testing" +) + +func TestNewDocSource(t *testing.T) { + resourceUri := "tar://foo" + testFile := NewDocSource(resourceUri) + assert.NotNil(t, testFile) +} + +/* +func TestResolveId(t *testing.T) { + testFile := NewResource("file://../../README.md") + assert.NotNil(t, testFile) + + absolutePath, e := filepath.Abs("../../README.md") + assert.Nil(t, e) + + testFile.ResolveId(context.Background()) + assert.Equal(t, absolutePath, testFile.(*File).Path) +} +*/ diff --git a/internal/source/http.go b/internal/source/http.go new file mode 100644 index 0000000..6378dcf --- /dev/null +++ b/internal/source/http.go @@ -0,0 +1,88 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( +_ "context" +_ "encoding/json" +_ "fmt" +_ "gopkg.in/yaml.v3" + "net/url" + "net/http" +_ "path/filepath" + "decl/internal/resource" + "decl/internal/iofilter" + "decl/internal/signature" +_ "os" + "io" + "errors" + "crypto/sha256" +) + +type HTTP struct { + Endpoint string `yaml:"endpoint" json:"endpoint"` +} + +func NewHTTP() *HTTP { + return &HTTP{} +} + +func init() { + SourceTypes.Register([]string{"http","https"}, func(u *url.URL) DocSource { + t := NewHTTP() + t.Endpoint = u.String() + return t + }) +} + +func (d *HTTP) Type() string { return "http" } + +func (h *HTTP) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { + documents := make([]*resource.Document, 0, 100) + documents = append(documents, resource.NewDocument()) + + resp, err := http.Get(h.Endpoint) + if err != nil { + return documents, err + } + defer resp.Body.Close() + signature := resp.Header.Get("Signature") + hash := sha256.New() + sumReadData := iofilter.New(resp.Body, func(p []byte, readn int, readerr error) (n int, err error) { + hash.Write(p) + return + }) + + decoder := resource.NewYAMLDecoder(sumReadData) + index := 0 + for { + doc := documents[index] + e := decoder.Decode(doc) + if errors.Is(e, io.EOF) { + if len(documents) > 1 { + documents[index] = nil + } + break + } + if e != nil { + return documents, e + } + if validationErr := doc.Validate(); validationErr != nil { + return documents, validationErr + } +/* + if applyErr := doc.Apply(); applyErr != nil { + return documents, applyErr + } +*/ + documents = append(documents, resource.NewDocument()) + index++ + } + + if signature != "" { + sig := &signature.&Ident{} + sig.VerifySum(hash.Sum(nil), signature) + } + + return documents, nil +} diff --git a/internal/source/http_test.go b/internal/source/http_test.go new file mode 100644 index 0000000..4d3d02f --- /dev/null +++ b/internal/source/http_test.go @@ -0,0 +1,13 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewHTTPSource(t *testing.T) { + h := NewHTTP() + assert.NotNil(t, h) +} diff --git a/internal/source/tar.go b/internal/source/tar.go new file mode 100644 index 0000000..3769504 --- /dev/null +++ b/internal/source/tar.go @@ -0,0 +1,102 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( +_ "context" +_ "encoding/json" +_ "fmt" +_ "gopkg.in/yaml.v3" + "net/url" + "path/filepath" + "decl/internal/resource" + "compress/gzip" + "archive/tar" + "regexp" + "os" + "io" +) + +type Tar struct { + Path string `yaml:"path" json:"path"` +} + +func NewTar() *Tar { + return &Tar{} +} + +func init() { + SourceTypes.Register([]string{"tar"}, func(u *url.URL) DocSource { + t := NewTar() + t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path)) + return t + }) + + SourceTypes.Register([]string{"tar.gz", "tgz"}, func(u *url.URL) DocSource { + t := NewTar() + if u.Scheme == "file" { + fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI())) + t.Path = fileAbsolutePath + } else { + t.Path = filepath.Join(u.Hostname(), u.Path) + } + return t + }) + +} + + +func (t *Tar) Type() string { return "tar" } + +func (t *Tar) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) { + documents := make([]*resource.Document, 0, 100) + d := resource.NewDocument() + documents = append(documents, d) + + TarGzipFileName := regexp.MustCompile(`^.*\.(tar\.gz|tgz)$`) + TarFileName := regexp.MustCompile(`^.*\.tar$`) + + file, fileErr := os.Open(t.Path) + if fileErr != nil { + return documents, fileErr + } + var gzipReader io.Reader + switch t.Path { + case TarGzipFileName.FindString(t.Path): + zr, err := gzip.NewReader(file) + if err != nil { + return documents, err + } + gzipReader = zr + fallthrough + case TarFileName.FindString(t.Path): + var fileReader io.Reader + if gzipReader == nil { + fileReader = file + } else { + fileReader = gzipReader + } + tarReader := tar.NewReader(fileReader) + for { + hdr, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return documents, err + } + f := resource.NewFile() + f.Path = hdr.Name + if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil { + return documents, fiErr + } + readFileData, readErr := io.ReadAll(tarReader) + if readErr != nil { + return documents, readErr + } + f.Content = string(readFileData) + d.AddResourceDeclaration("file", f) + } + } + return documents, nil +} diff --git a/internal/source/tar_test.go b/internal/source/tar_test.go new file mode 100644 index 0000000..decc0f2 --- /dev/null +++ b/internal/source/tar_test.go @@ -0,0 +1,14 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewTarSource(t *testing.T) { + s := NewTar() + assert.NotNil(t, s) +} + diff --git a/internal/source/types.go b/internal/source/types.go new file mode 100644 index 0000000..e206934 --- /dev/null +++ b/internal/source/types.go @@ -0,0 +1,92 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( + "errors" + "fmt" + "net/url" + "strings" + "path/filepath" +) + +var ( + ErrUnknownSourceType = errors.New("Unknown source type") + SourceTypes *Types = NewTypes() +) + +type TypeName string //`json:"type"` + +type TypeFactory func(*url.URL) DocSource + +type Types struct { + registry map[string]TypeFactory +} + +func NewTypes() *Types { + return &Types{registry: make(map[string]TypeFactory)} +} + +func (t *Types) Register(names []string, factory TypeFactory) { + for _,name := range names { + t.registry[name] = factory + } +} + +func (t *Types) FromExtension(path string) (TypeFactory, error) { + elements := strings.Split(path, ".") + numberOfElements := len(elements) + if numberOfElements > 2 { + if src := t.Get(strings.Join(elements[numberOfElements - 2: numberOfElements - 1], ".")); src != nil { + return src, nil + } + } + if src := t.Get(elements[numberOfElements - 1]); src != nil { + return src, nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, path) +} + +func (t *Types) New(uri string) (DocSource, error) { + u, e := url.Parse(uri) + if u == nil || e != nil { + return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, e) + } + + if u.Scheme == "" { + u.Scheme = "file" + } + + path := filepath.Join(u.Hostname(), u.Path) + if d, lookupErr := t.FromExtension(path); d != nil { + return d(u), lookupErr + } + + if r, ok := t.registry[u.Scheme]; ok { + return r(u), nil + } + return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, u.Scheme) +} + +func (t *Types) Has(typename string) bool { + if _, ok := t.registry[typename]; ok { + return true + } + return false +} + +func (t *Types) Get(typename string) TypeFactory { + if d, ok := t.registry[typename]; ok { + return d + } + return nil +} + +func (n *TypeName) UnmarshalJSON(b []byte) error { + SourceTypeName := strings.Trim(string(b), "\"") + if SourceTypes.Has(SourceTypeName) { + *n = TypeName(SourceTypeName) + return nil + } + return fmt.Errorf("%w: %s", ErrUnknownSourceType, SourceTypeName) +} diff --git a/internal/source/types_test.go b/internal/source/types_test.go new file mode 100644 index 0000000..d5b5727 --- /dev/null +++ b/internal/source/types_test.go @@ -0,0 +1,89 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package source + +import ( +_ "context" + "encoding/json" + "github.com/stretchr/testify/assert" + "net/url" + "testing" + "decl/internal/resource" +) + +type MockDocSource struct { + InjectType func() string + InjectExtractResources func(uri string, filter ResourceSelector) ([]*resource.Document, error) +} + +func (m *MockDocSource) Type() string { return m.InjectType() } +func (m *MockDocSource) ExtractResources(uri string, filter ResourceSelector) ([]*resource.Document, error) { return m.InjectExtractResources(uri, filter) } + +func NewFooDocSource() DocSource { + return &MockDocSource{ + InjectType: func() string { return "foo" }, + InjectExtractResources: func(uri string, filter ResourceSelector) ([]*resource.Document, error) { return nil,nil }, + } +} + +func NewFileDocSource() DocSource { + return &MockDocSource{ + InjectType: func() string { return "file" }, + InjectExtractResources: func(uri string, filter ResourceSelector) ([]*resource.Document, error) { return nil,nil }, + } +} + +func TestNewSourceTypes(t *testing.T) { + sourceTypes := NewTypes() + assert.NotNil(t, sourceTypes) +} + +func TestNewSourceTypesRegister(t *testing.T) { + m := NewFooDocSource() + + sourceTypes := NewTypes() + assert.NotNil(t, sourceTypes) + + sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m }) + + r, e := sourceTypes.New("foo://") + assert.Nil(t, e) + assert.Equal(t, m, r) +} + +func TestResourceTypesFromURI(t *testing.T) { + m := NewFooDocSource() + + sourceTypes := NewTypes() + assert.NotNil(t, sourceTypes) + + sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m }) + + r, e := sourceTypes.New("foo://bar") + assert.Nil(t, e) + assert.Equal(t, m, r) +} + +func TestResourceTypesHasType(t *testing.T) { + m := NewFooDocSource() + + sourceTypes := NewTypes() + assert.NotNil(t, sourceTypes) + + sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m }) + + assert.True(t, sourceTypes.Has("foo")) +} + +func TestDocSourceTypeName(t *testing.T) { + SourceTypes.Register([]string{"file"}, func(*url.URL) DocSource { return NewFileDocSource() }) + + type fDocSourceName struct { + Name TypeName `json:"type"` + } + fTypeName := &fDocSourceName{} + jsonType := `{ "type": "file" }` + e := json.Unmarshal([]byte(jsonType), &fTypeName) + assert.Nil(t, e) + assert.Equal(t, "file", string(fTypeName.Name)) +}