doublejynx/jx
doublejynx/jx
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases 7 Wiki Activity

handle numeric user ids
Some checks failed
Lint / golangci-lint (push) Failing after 9m47s
Details
Declarative Tests / test (push) Failing after 10s
Details

Browse Source
This commit is contained in:
Matthew Rich 2024-04-17 17:07:12 -07:00
parent e859ef136c
commit 976f654c60
15 changed files with 844 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
go.mod
View File

@ -2,7 +2,7 @@ module decl
go 1.21.1
require github.com/stretchr/testify v1.8.4
require github.com/stretchr/testify v1.9.0
require (
github.com/Microsoft/go-winio v0.4.14 // indirect
@ -11,14 +11,19 @@ require (
github.com/docker/docker v25.0.5+incompatible // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/goccy/go-yaml v1.11.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sters/yaml-diff v1.3.2 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@ -26,6 +31,7 @@ require (
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

20
go.sum
View File

@ -13,6 +13,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@ -20,11 +22,18 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@ -36,12 +45,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sters/yaml-diff v1.3.2 h1:99Ke50QYFQYZjKMOiePxwyuQ+WeCvNy6cRooqdLs/ZE=
github.com/sters/yaml-diff v1.3.2/go.mod h1:86usbNZiUqke5wYjMxDVEjmvGjmY2FkMwOwe0A5zf68=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
@ -75,8 +88,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -87,7 +104,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

43
internal/resource/os.go
View File

@ -5,8 +5,11 @@ package resource
import (
"os/user"
"strconv"
"regexp"
)
var MatchId *regexp.Regexp = regexp.MustCompile(`^[0-9]+$`)
func LookupUIDString(userName string) string {
user, userLookupErr := user.Lookup(userName)
if userLookupErr != nil {
@ -16,12 +19,24 @@ func LookupUIDString(userName string) string {
}
func LookupUID(userName string) (int, error) {
user, userLookupErr := user.Lookup(userName)
if userLookupErr != nil {
return -1, userLookupErr
var UID string
if MatchId.MatchString(userName) {
user, userLookupErr := user.LookupId(userName)
if userLookupErr != nil {
//return -1, userLookupErr
UID = userName
} else {
UID = user.Uid
}
} else {
user, userLookupErr := user.Lookup(userName)
if userLookupErr != nil {
return -1, userLookupErr
}
UID = user.Uid
}
uid, uidErr := strconv.Atoi(user.Uid)
uid, uidErr := strconv.Atoi(UID)
if uidErr != nil {
return -1, uidErr
}
@ -30,12 +45,24 @@ func LookupUID(userName string) (int, error) {
}
func LookupGID(groupName string) (int, error) {
group, groupLookupErr := user.LookupGroup(groupName)
if groupLookupErr != nil {
return -1, groupLookupErr
var GID string
if MatchId.MatchString(groupName) {
group, groupLookupErr := user.LookupGroupId(groupName)
if groupLookupErr != nil {
//return -1, groupLookupErr
GID = groupName
} else {
GID = group.Gid
}
} else {
group, groupLookupErr := user.LookupGroup(groupName)
if groupLookupErr != nil {
return -1, groupLookupErr
}
GID = group.Gid
}
gid, gidErr := strconv.Atoi(group.Gid)
gid, gidErr := strconv.Atoi(GID)
if gidErr != nil {
return -1, gidErr
}

14
internal/resource/os_test.go
View File

@ -17,17 +17,21 @@ import (
func TestLookupUID(t *testing.T) {
uid, e := LookupUID("nobody")
assert.Equal(t, nil, e)
assert.Nil(t, e)
assert.Equal(t, 65534, uid)
nuid, ne := LookupUID("1001")
assert.Nil(t, ne)
assert.Equal(t, 1001, nuid)
}
func TestLookupGID(t *testing.T) {
gid, e := LookupGID("nobody")
assert.Equal(t, nil, e)
assert.Nil(t, e)
assert.Equal(t, 65534, gid)
}
func TestExecCommand(t *testing.T) {
ngid, ne := LookupGID("1001")
assert.Nil(t, ne)
assert.Equal(t, 1001, ngid)
}

100
internal/source/decl.go Normal file
View File

@ -0,0 +1,100 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
_ "context"
_ "encoding/json"
_ "fmt"
_ "gopkg.in/yaml.v3"
"net/url"
"path/filepath"
"decl/internal/resource"
"regexp"
"os"
"io"
"compress/gzip"
"errors"
"log/slog"
)
type DeclFile struct {
Path string `yaml:"path" json:"path"`
}
func NewDeclFile() *DeclFile {
return &DeclFile{}
}
func init() {
SourceTypes.Register([]string{"decl"}, func(u *url.URL) DocSource {
t := NewDeclFile()
t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
return t
})
SourceTypes.Register([]string{"yaml","yml","yaml.gz","yml.gz"}, func(u *url.URL) DocSource {
t := NewDeclFile()
if u.Scheme == "file" {
fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
t.Path = fileAbsolutePath
} else {
t.Path = filepath.Join(u.Hostname(), u.Path)
}
return t
})
}
func (d *DeclFile) Type() string { return "decl" }
func (d *DeclFile) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) {
documents := make([]*resource.Document, 0, 100)
documents = append(documents, resource.NewDocument())
GzipFileName := regexp.MustCompile(`^.*\.gz$`)
file, fileErr := os.Open(d.Path)
if fileErr != nil {
return documents, fileErr
}
var fileReader io.Reader
if GzipFileName.FindString(d.Path) == d.Path {
slog.Info("decompressing gzip", "path", d.Path)
zr, err := gzip.NewReader(file)
if err != nil {
return documents, err
}
fileReader = zr
} else {
fileReader = file
}
decoder := resource.NewYAMLDecoder(fileReader)
index := 0
for {
doc := documents[index]
e := decoder.Decode(doc)
if errors.Is(e, io.EOF) {
if len(documents) > 1 {
documents[index] = nil
}
break
}
if e != nil {
return documents, e
}
if validationErr := doc.Validate(); validationErr != nil {
return documents, validationErr
}
/*
if applyErr := doc.Apply(); applyErr != nil {
return documents, applyErr
}
*/
documents = append(documents, resource.NewDocument())
index++
}
return documents, nil
}

100
internal/source/dir.go Normal file
View File

@ -0,0 +1,100 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
_ "context"
_ "encoding/json"
_ "fmt"
_ "gopkg.in/yaml.v3"
"net/url"
"path/filepath"
"decl/internal/resource"
"os"
"io"
)
type Dir struct {
Path string `yaml:"path" json:"path"`
subDirsStack []string `yaml:"-" json:"-"`
}
func NewDir() *Dir {
return &Dir{
subDirsStack: make([]string, 0, 100),
}
}
func init() {
SourceTypes.Register([]string{"file"}, func(u *url.URL) DocSource {
t := NewDir()
t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
return t
})
}
func (d *Dir) Type() string { return "dir" }
func (d *Dir) ExtractDirectory(path string) (*resource.Document, error) {
document := resource.NewDocument()
files, err := os.ReadDir(path)
if err != nil {
return nil, err
}
for _,file := range files {
f := resource.NewFile()
f.Path = filepath.Join(path, file.Name())
info, infoErr := file.Info()
if infoErr != nil {
return document, infoErr
}
if fiErr := f.UpdateAttributesFromFileInfo(info); fiErr != nil {
return document, fiErr
}
f.FileType.SetMode(file.Type())
if file.IsDir() {
d.subDirsStack = append(d.subDirsStack, f.Path)
} else {
fileReader, fileReaderErr := os.Open(f.Path)
if fileReaderErr != nil {
return document, fileReaderErr
}
readFileData, readErr := io.ReadAll(fileReader)
if readErr != nil {
return document, readErr
}
f.Content = string(readFileData)
}
document.AddResourceDeclaration("file", f)
}
return document, nil
}
func (d *Dir) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) {
documents := make([]*resource.Document, 0, 100)
d.subDirsStack = append(d.subDirsStack, d.Path)
for {
if len(d.subDirsStack) == 0 {
break
}
var dirPath string
dirPath, d.subDirsStack = d.subDirsStack[len(d.subDirsStack) - 1], d.subDirsStack[:len(d.subDirsStack) - 1]
document, dirErr := d.ExtractDirectory(dirPath)
if dirErr != nil {
return documents, dirErr
}
documents = append(documents, document)
}
return documents, nil
}

23
internal/source/dir_test.go Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestNewDirSource(t *testing.T) {
s := NewDir()
assert.NotNil(t, s)
}
func TestExtractDirectory(t *testing.T) {
s := NewDir()
assert.NotNil(t, s)
document, err := s.ExtractDirectory(TempDir)
assert.Nil(t, err)
assert.NotNil(t, document)
}

121
internal/source/docsource.go Normal file
View File

@ -0,0 +1,121 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
_ "context"
_ "encoding/json"
_ "fmt"
_ "gopkg.in/yaml.v3"
"net/url"
"regexp"
_ "strings"
"os"
"io"
"compress/gzip"
"archive/tar"
"errors"
"path/filepath"
"decl/internal/resource"
)
type ResourceSelector func(r resource.Resource) bool
type DocSource interface {
Type() string
ExtractResources(filter ResourceSelector) ([]*resource.Document, error)
}
func NewDocSource(uri string) DocSource {
s, e := SourceTypes.New(uri)
if e == nil {
return s
}
return nil
}
func ExtractResources(uri string, filter ResourceSelector) ([]*resource.Document, error) {
documents := make([]*resource.Document, 0, 100)
d := resource.NewDocument()
documents = append(documents, d)
TarGzipFileName := regexp.MustCompile(`^.*\.(tar\.gz|tgz)$`)
TarFileName := regexp.MustCompile(`^.*\.tar$`)
u,e := url.Parse(uri)
if e != nil {
return nil, e
}
switch u.Scheme {
case "file":
fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
file, fileErr := os.Open(fileAbsolutePath)
if fileErr != nil {
return documents, fileErr
}
var gzipReader io.Reader
switch u.Path {
case TarGzipFileName.FindString(u.Path):
zr, err := gzip.NewReader(file)
if err != nil {
return documents, err
}
gzipReader = zr
fallthrough
case TarFileName.FindString(u.Path):
var fileReader io.Reader
if gzipReader == nil {
fileReader = file
} else {
fileReader = gzipReader
}
tarReader := tar.NewReader(fileReader)
for {
hdr, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return documents, err
}
f := resource.NewFile()
if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil {
return documents, fiErr
}
readFileData, readErr := io.ReadAll(tarReader)
if readErr != nil {
return documents, readErr
}
f.Content = string(readFileData)
d.AddResourceDeclaration("file", f)
}
default:
decoder := resource.NewYAMLDecoder(file)
index := 0
for {
doc := documents[index]
e := decoder.Decode(doc)
if errors.Is(e, io.EOF) {
if len(documents) > 1 {
documents[index] = nil
}
break
}
if e != nil {
return documents, e
}
if validationErr := doc.Validate(); validationErr != nil {
return documents, validationErr
}
if applyErr := doc.Apply(); applyErr != nil {
return documents, applyErr
}
documents = append(documents, resource.NewDocument())
index++
}
}
}
return documents, nil
}

30
internal/source/docsource_test.go Normal file
View File

@ -0,0 +1,30 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
_ "context"
_ "fmt"
"github.com/stretchr/testify/assert"
_ "log"
"testing"
)
func TestNewDocSource(t *testing.T) {
resourceUri := "tar://foo"
testFile := NewDocSource(resourceUri)
assert.NotNil(t, testFile)
}
/*
func TestResolveId(t *testing.T) {
testFile := NewResource("file://../../README.md")
assert.NotNil(t, testFile)
absolutePath, e := filepath.Abs("../../README.md")
assert.Nil(t, e)
testFile.ResolveId(context.Background())
assert.Equal(t, absolutePath, testFile.(*File).Path)
}
*/

88
internal/source/http.go Normal file
View File

@ -0,0 +1,88 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
_ "context"
_ "encoding/json"
_ "fmt"
_ "gopkg.in/yaml.v3"
"net/url"
"net/http"
_ "path/filepath"
"decl/internal/resource"
"decl/internal/iofilter"
"decl/internal/signature"
_ "os"
"io"
"errors"
"crypto/sha256"
)
type HTTP struct {
Endpoint string `yaml:"endpoint" json:"endpoint"`
}
func NewHTTP() *HTTP {
return &HTTP{}
}
func init() {
SourceTypes.Register([]string{"http","https"}, func(u *url.URL) DocSource {
t := NewHTTP()
t.Endpoint = u.String()
return t
})
}
func (d *HTTP) Type() string { return "http" }
func (h *HTTP) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) {
documents := make([]*resource.Document, 0, 100)
documents = append(documents, resource.NewDocument())
resp, err := http.Get(h.Endpoint)
if err != nil {
return documents, err
}
defer resp.Body.Close()
signature := resp.Header.Get("Signature")
hash := sha256.New()
sumReadData := iofilter.New(resp.Body, func(p []byte, readn int, readerr error) (n int, err error) {
hash.Write(p)
return
})
decoder := resource.NewYAMLDecoder(sumReadData)
index := 0
for {
doc := documents[index]
e := decoder.Decode(doc)
if errors.Is(e, io.EOF) {
if len(documents) > 1 {
documents[index] = nil
}
break
}
if e != nil {
return documents, e
}
if validationErr := doc.Validate(); validationErr != nil {
return documents, validationErr
}
/*
if applyErr := doc.Apply(); applyErr != nil {
return documents, applyErr
}
*/
documents = append(documents, resource.NewDocument())
index++
}
if signature != "" {
sig := &signature.&Ident{}
sig.VerifySum(hash.Sum(nil), signature)
}
return documents, nil
}

13
internal/source/http_test.go Normal file
View File

@ -0,0 +1,13 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestNewHTTPSource(t *testing.T) {
h := NewHTTP()
assert.NotNil(t, h)
}

102
internal/source/tar.go Normal file
View File

@ -0,0 +1,102 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
_ "context"
_ "encoding/json"
_ "fmt"
_ "gopkg.in/yaml.v3"
"net/url"
"path/filepath"
"decl/internal/resource"
"compress/gzip"
"archive/tar"
"regexp"
"os"
"io"
)
type Tar struct {
Path string `yaml:"path" json:"path"`
}
func NewTar() *Tar {
return &Tar{}
}
func init() {
SourceTypes.Register([]string{"tar"}, func(u *url.URL) DocSource {
t := NewTar()
t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path))
return t
})
SourceTypes.Register([]string{"tar.gz", "tgz"}, func(u *url.URL) DocSource {
t := NewTar()
if u.Scheme == "file" {
fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
t.Path = fileAbsolutePath
} else {
t.Path = filepath.Join(u.Hostname(), u.Path)
}
return t
})
}
func (t *Tar) Type() string { return "tar" }
func (t *Tar) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) {
documents := make([]*resource.Document, 0, 100)
d := resource.NewDocument()
documents = append(documents, d)
TarGzipFileName := regexp.MustCompile(`^.*\.(tar\.gz|tgz)$`)
TarFileName := regexp.MustCompile(`^.*\.tar$`)
file, fileErr := os.Open(t.Path)
if fileErr != nil {
return documents, fileErr
}
var gzipReader io.Reader
switch t.Path {
case TarGzipFileName.FindString(t.Path):
zr, err := gzip.NewReader(file)
if err != nil {
return documents, err
}
gzipReader = zr
fallthrough
case TarFileName.FindString(t.Path):
var fileReader io.Reader
if gzipReader == nil {
fileReader = file
} else {
fileReader = gzipReader
}
tarReader := tar.NewReader(fileReader)
for {
hdr, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return documents, err
}
f := resource.NewFile()
f.Path = hdr.Name
if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil {
return documents, fiErr
}
readFileData, readErr := io.ReadAll(tarReader)
if readErr != nil {
return documents, readErr
}
f.Content = string(readFileData)
d.AddResourceDeclaration("file", f)
}
}
return documents, nil
}

14
internal/source/tar_test.go Normal file
View File

@ -0,0 +1,14 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestNewTarSource(t *testing.T) {
s := NewTar()
assert.NotNil(t, s)
}

92
internal/source/types.go Normal file
View File

@ -0,0 +1,92 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
"errors"
"fmt"
"net/url"
"strings"
"path/filepath"
)
var (
ErrUnknownSourceType = errors.New("Unknown source type")
SourceTypes *Types = NewTypes()
)
type TypeName string //`json:"type"`
type TypeFactory func(*url.URL) DocSource
type Types struct {
registry map[string]TypeFactory
}
func NewTypes() *Types {
return &Types{registry: make(map[string]TypeFactory)}
}
func (t *Types) Register(names []string, factory TypeFactory) {
for _,name := range names {
t.registry[name] = factory
}
}
func (t *Types) FromExtension(path string) (TypeFactory, error) {
elements := strings.Split(path, ".")
numberOfElements := len(elements)
if numberOfElements > 2 {
if src := t.Get(strings.Join(elements[numberOfElements - 2: numberOfElements - 1], ".")); src != nil {
return src, nil
}
}
if src := t.Get(elements[numberOfElements - 1]); src != nil {
return src, nil
}
return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, path)
}
func (t *Types) New(uri string) (DocSource, error) {
u, e := url.Parse(uri)
if u == nil || e != nil {
return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, e)
}
if u.Scheme == "" {
u.Scheme = "file"
}
path := filepath.Join(u.Hostname(), u.Path)
if d, lookupErr := t.FromExtension(path); d != nil {
return d(u), lookupErr
}
if r, ok := t.registry[u.Scheme]; ok {
return r(u), nil
}
return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, u.Scheme)
}
func (t *Types) Has(typename string) bool {
if _, ok := t.registry[typename]; ok {
return true
}
return false
}
func (t *Types) Get(typename string) TypeFactory {
if d, ok := t.registry[typename]; ok {
return d
}
return nil
}
func (n *TypeName) UnmarshalJSON(b []byte) error {
SourceTypeName := strings.Trim(string(b), "\"")
if SourceTypes.Has(SourceTypeName) {
*n = TypeName(SourceTypeName)
return nil
}
return fmt.Errorf("%w: %s", ErrUnknownSourceType, SourceTypeName)
}

89
internal/source/types_test.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package source
import (
_ "context"
"encoding/json"
"github.com/stretchr/testify/assert"
"net/url"
"testing"
"decl/internal/resource"
)
type MockDocSource struct {
InjectType func() string
InjectExtractResources func(uri string, filter ResourceSelector) ([]*resource.Document, error)
}
func (m *MockDocSource) Type() string { return m.InjectType() }
func (m *MockDocSource) ExtractResources(uri string, filter ResourceSelector) ([]*resource.Document, error) { return m.InjectExtractResources(uri, filter) }
func NewFooDocSource() DocSource {
return &MockDocSource{
InjectType: func() string { return "foo" },
InjectExtractResources: func(uri string, filter ResourceSelector) ([]*resource.Document, error) { return nil,nil },
}
}
func NewFileDocSource() DocSource {
return &MockDocSource{
InjectType: func() string { return "file" },
InjectExtractResources: func(uri string, filter ResourceSelector) ([]*resource.Document, error) { return nil,nil },
}
}
func TestNewSourceTypes(t *testing.T) {
sourceTypes := NewTypes()
assert.NotNil(t, sourceTypes)
}
func TestNewSourceTypesRegister(t *testing.T) {
m := NewFooDocSource()
sourceTypes := NewTypes()
assert.NotNil(t, sourceTypes)
sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m })
r, e := sourceTypes.New("foo://")
assert.Nil(t, e)
assert.Equal(t, m, r)
}
func TestResourceTypesFromURI(t *testing.T) {
m := NewFooDocSource()
sourceTypes := NewTypes()
assert.NotNil(t, sourceTypes)
sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m })
r, e := sourceTypes.New("foo://bar")
assert.Nil(t, e)
assert.Equal(t, m, r)
}
func TestResourceTypesHasType(t *testing.T) {
m := NewFooDocSource()
sourceTypes := NewTypes()
assert.NotNil(t, sourceTypes)
sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m })
assert.True(t, sourceTypes.Has("foo"))
}
func TestDocSourceTypeName(t *testing.T) {
SourceTypes.Register([]string{"file"}, func(*url.URL) DocSource { return NewFileDocSource() })
type fDocSourceName struct {
Name TypeName `json:"type"`
}
fTypeName := &fDocSourceName{}
jsonType := `{ "type": "file" }`
e := json.Unmarshal([]byte(jsonType), &fTypeName)
assert.Nil(t, e)
assert.Equal(t, "file", string(fTypeName.Name))
}