handle numeric user ids

go.mod

@@ -2,7 +2,7 @@ module decl
 
 go 1.21.1
 
-require github.com/stretchr/testify v1.8.4
+require github.com/stretchr/testify v1.9.0
 
 require (
 	github.com/Microsoft/go-winio v0.4.14 // indirect
@@ -11,14 +11,19 @@ require (
 	github.com/docker/docker v25.0.5+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/goccy/go-yaml v1.11.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sters/yaml-diff v1.3.2 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
@@ -26,6 +31,7 @@ require (
 	go.opentelemetry.io/otel v1.24.0 // indirect
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
-	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -13,6 +13,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -20,11 +22,18 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
+github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@@ -36,12 +45,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sters/yaml-diff v1.3.2 h1:99Ke50QYFQYZjKMOiePxwyuQ+WeCvNy6cRooqdLs/ZE=
+github.com/sters/yaml-diff v1.3.2/go.mod h1:86usbNZiUqke5wYjMxDVEjmvGjmY2FkMwOwe0A5zf68=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
@@ -75,8 +88,12 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -87,7 +104,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/resource/os.go

@@ -5,8 +5,11 @@ package resource
 import (
 	"os/user"
 	"strconv"
+	"regexp"
 )
 
+var MatchId	*regexp.Regexp = regexp.MustCompile(`^[0-9]+$`)
+
 func LookupUIDString(userName string) string {
 	user, userLookupErr := user.Lookup(userName)
 	if userLookupErr != nil {
@@ -16,12 +19,24 @@ func LookupUIDString(userName string) string {
 }
 
 func LookupUID(userName string) (int, error) {
-	user, userLookupErr := user.Lookup(userName)
+	var UID string
-	if userLookupErr != nil {
+	if MatchId.MatchString(userName) {
-		return -1, userLookupErr
+		user, userLookupErr := user.LookupId(userName)
+		if userLookupErr != nil {
+			//return -1, userLookupErr
+			UID = userName
+		} else {
+			UID = user.Uid
+		}
+	} else {
+		user, userLookupErr := user.Lookup(userName)
+		if userLookupErr != nil {
+			return -1, userLookupErr
+		}
+		UID = user.Uid
 	}
 
-	uid, uidErr := strconv.Atoi(user.Uid)
+	uid, uidErr := strconv.Atoi(UID)
 	if uidErr != nil {
 		return -1, uidErr
 	}
@@ -30,12 +45,24 @@ func LookupUID(userName string) (int, error) {
 }
 
 func LookupGID(groupName string) (int, error) {
-	group, groupLookupErr := user.LookupGroup(groupName)
+	var GID string
-	if groupLookupErr != nil {
+	if MatchId.MatchString(groupName) {
-		return -1, groupLookupErr
+		group, groupLookupErr := user.LookupGroupId(groupName)
+		if groupLookupErr != nil {
+			//return -1, groupLookupErr
+			GID = groupName
+		} else {
+			GID = group.Gid
+		}
+	} else {
+		group, groupLookupErr := user.LookupGroup(groupName)
+		if groupLookupErr != nil {
+			return -1, groupLookupErr
+		}
+		GID = group.Gid
 	}
 
-	gid, gidErr := strconv.Atoi(group.Gid)
+	gid, gidErr := strconv.Atoi(GID)
 	if gidErr != nil {
 		return -1, gidErr
 	}

internal/resource/os_test.go

@@ -17,17 +17,21 @@ import (
 func TestLookupUID(t *testing.T) {
 	uid, e := LookupUID("nobody")
 
-	assert.Equal(t, nil, e)
+	assert.Nil(t, e)
 	assert.Equal(t, 65534, uid)
+
+	nuid, ne := LookupUID("1001")
+	assert.Nil(t, ne)
+	assert.Equal(t, 1001, nuid)
 }
 
 func TestLookupGID(t *testing.T) {
 	gid, e := LookupGID("nobody")
 
-	assert.Equal(t, nil, e)
+	assert.Nil(t, e)
 	assert.Equal(t, 65534, gid)
-}
-
-func TestExecCommand(t *testing.T) {
 
+	ngid, ne := LookupGID("1001")
+	assert.Nil(t, ne)
+	assert.Equal(t, 1001, ngid)
 }

internal/source/decl.go

@@ -0,0 +1,100 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+_	"context"
+_	"encoding/json"
+_	"fmt"
+_	"gopkg.in/yaml.v3"
+	"net/url"
+	"path/filepath"
+	"decl/internal/resource"
+	"regexp"
+	"os"
+	"io"
+	"compress/gzip"
+	"errors"
+	"log/slog"
+)
+
+type DeclFile struct {
+	Path string `yaml:"path" json:"path"`
+}
+
+func NewDeclFile() *DeclFile {
+	return &DeclFile{}
+}
+
+func init() {
+	SourceTypes.Register([]string{"decl"}, func(u *url.URL) DocSource {
+		t := NewDeclFile()
+		t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
+		return t
+	})
+
+	SourceTypes.Register([]string{"yaml","yml","yaml.gz","yml.gz"}, func(u *url.URL) DocSource {
+		t := NewDeclFile()
+		if u.Scheme == "file" {
+			fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
+			t.Path = fileAbsolutePath
+		} else {
+			t.Path = filepath.Join(u.Hostname(), u.Path)
+		}
+		return t
+	})
+
+}
+
+
+func (d *DeclFile) Type() string { return "decl" }
+
+func (d *DeclFile) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) {
+	documents := make([]*resource.Document, 0, 100)
+	documents = append(documents, resource.NewDocument())
+
+	GzipFileName := regexp.MustCompile(`^.*\.gz$`)
+		
+	file, fileErr := os.Open(d.Path)
+	if fileErr != nil {
+		return documents, fileErr
+	}
+	var fileReader io.Reader
+
+	if GzipFileName.FindString(d.Path) == d.Path {
+		slog.Info("decompressing gzip", "path", d.Path)
+		zr, err := gzip.NewReader(file)
+		if err != nil {
+			return documents, err
+		}
+		fileReader = zr
+	} else {
+		fileReader = file
+	}
+	decoder := resource.NewYAMLDecoder(fileReader)
+	index := 0
+	for {
+		doc := documents[index]
+		e := decoder.Decode(doc)
+		if errors.Is(e, io.EOF) {
+			if len(documents) > 1 {
+				documents[index] = nil
+			}
+			break
+		}
+		if e != nil {
+			return documents, e
+		}
+		if validationErr := doc.Validate(); validationErr != nil {
+			return documents, validationErr
+		}
+/*
+		if applyErr := doc.Apply(); applyErr != nil {
+			return documents, applyErr
+		}
+*/
+		documents = append(documents, resource.NewDocument())
+		index++
+	}
+	return documents, nil
+}

internal/source/dir.go

@@ -0,0 +1,100 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+_	"context"
+_	"encoding/json"
+_	"fmt"
+_	"gopkg.in/yaml.v3"
+	"net/url"
+	"path/filepath"
+	"decl/internal/resource"
+	"os"
+	"io"
+)
+
+type Dir struct {
+	Path string `yaml:"path" json:"path"`
+	subDirsStack []string `yaml:"-" json:"-"`
+}
+
+func NewDir() *Dir {
+	return &Dir{
+		subDirsStack: make([]string, 0, 100),
+	}
+}
+
+func init() {
+	SourceTypes.Register([]string{"file"}, func(u *url.URL) DocSource {
+		t := NewDir()
+		t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
+		return t
+	})
+
+}
+
+
+func (d *Dir) Type() string { return "dir" }
+
+func (d *Dir) ExtractDirectory(path string) (*resource.Document, error) {
+	document := resource.NewDocument()
+	files, err := os.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+
+	for _,file := range files {
+		f := resource.NewFile()
+		f.Path = filepath.Join(path, file.Name())
+		info, infoErr := file.Info()
+		if infoErr != nil {
+			return document, infoErr
+		}
+
+		if fiErr := f.UpdateAttributesFromFileInfo(info); fiErr != nil {
+			return document, fiErr
+		}
+
+		f.FileType.SetMode(file.Type())
+
+		if file.IsDir() {
+			d.subDirsStack = append(d.subDirsStack, f.Path)
+		} else {
+			fileReader, fileReaderErr := os.Open(f.Path)
+			if fileReaderErr != nil {
+				return document, fileReaderErr
+			}
+
+			readFileData, readErr := io.ReadAll(fileReader)
+			if readErr != nil {
+				return document, readErr
+			}
+			f.Content = string(readFileData)
+		}
+
+		document.AddResourceDeclaration("file", f)
+	}
+	return document, nil
+}
+
+func (d *Dir) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) {
+	documents := make([]*resource.Document, 0, 100)
+
+	d.subDirsStack = append(d.subDirsStack, d.Path)
+
+	for {
+		if len(d.subDirsStack) == 0 {
+			break
+		}
+		var dirPath string
+		dirPath, d.subDirsStack = d.subDirsStack[len(d.subDirsStack) - 1], d.subDirsStack[:len(d.subDirsStack) - 1]
+		document, dirErr := d.ExtractDirectory(dirPath)
+		if dirErr != nil {
+			return documents, dirErr
+		}
+		
+		documents = append(documents, document)
+	}
+	return documents, nil
+}

internal/source/dir_test.go

@@ -0,0 +1,23 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestNewDirSource(t *testing.T) {
+	s := NewDir()
+	assert.NotNil(t, s)
+}
+
+func TestExtractDirectory(t *testing.T) {
+	s := NewDir()
+	assert.NotNil(t, s)
+
+	document, err := s.ExtractDirectory(TempDir)
+	assert.Nil(t, err)
+	assert.NotNil(t, document)
+
+}

internal/source/docsource.go

@@ -0,0 +1,121 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+_	"context"
+_	"encoding/json"
+_	"fmt"
+_	"gopkg.in/yaml.v3"
+	"net/url"
+	"regexp"
+_	"strings"
+	"os"
+	"io"
+	"compress/gzip"
+	"archive/tar"
+	"errors"
+	"path/filepath"
+	"decl/internal/resource"
+)
+
+type ResourceSelector func(r resource.Resource) bool
+
+type DocSource interface {
+	Type() string
+
+	ExtractResources(filter ResourceSelector) ([]*resource.Document, error)
+}
+
+func NewDocSource(uri string) DocSource {
+	s, e := SourceTypes.New(uri)
+	if e == nil {
+		return s
+	}
+	return nil
+}
+
+func ExtractResources(uri string, filter ResourceSelector) ([]*resource.Document, error) {
+	documents := make([]*resource.Document, 0, 100)
+	d := resource.NewDocument()
+	documents = append(documents, d)
+	
+	TarGzipFileName := regexp.MustCompile(`^.*\.(tar\.gz|tgz)$`)
+	TarFileName := regexp.MustCompile(`^.*\.tar$`)
+
+	u,e := url.Parse(uri)
+	if e != nil {
+		return nil, e
+	}
+	switch u.Scheme {
+	case "file":
+		fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
+		file, fileErr := os.Open(fileAbsolutePath)
+		if fileErr != nil {
+			return documents, fileErr
+		}
+		var gzipReader io.Reader
+		switch u.Path {
+		case TarGzipFileName.FindString(u.Path):
+			zr, err := gzip.NewReader(file)
+			if err != nil {
+				return documents, err
+			}
+			gzipReader = zr
+			fallthrough
+		case TarFileName.FindString(u.Path):
+			var fileReader io.Reader
+			if gzipReader == nil {
+				fileReader = file
+			} else {
+				fileReader = gzipReader
+			}
+			tarReader := tar.NewReader(fileReader)
+			for {
+				hdr, err := tarReader.Next()
+				if err == io.EOF {
+					break
+				}
+				if err != nil {
+					return documents, err
+				}
+				f := resource.NewFile()
+				if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil {
+					return documents, fiErr
+				}
+				readFileData, readErr := io.ReadAll(tarReader)
+				if readErr != nil {
+					return documents, readErr
+				}
+				f.Content = string(readFileData)
+				d.AddResourceDeclaration("file", f)
+			}
+		default:
+      decoder := resource.NewYAMLDecoder(file)
+      index := 0
+			for {
+				doc := documents[index]
+				e := decoder.Decode(doc)
+				if errors.Is(e, io.EOF) {
+					if len(documents) > 1 {
+						documents[index] = nil
+					}
+					break
+				}
+				if e != nil {
+					return documents, e
+				}
+				if validationErr := doc.Validate(); validationErr != nil {
+					return documents, validationErr
+				}
+				if applyErr := doc.Apply(); applyErr != nil {
+					return documents, applyErr
+				}
+				documents = append(documents, resource.NewDocument())
+				index++
+			}
+
+		}
+	}
+	return documents, nil
+}

internal/source/docsource_test.go

@@ -0,0 +1,30 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+_	"context"
+_	"fmt"
+	"github.com/stretchr/testify/assert"
+_	"log"
+	"testing"
+)
+
+func TestNewDocSource(t *testing.T) {
+	resourceUri := "tar://foo"
+	testFile := NewDocSource(resourceUri)
+	assert.NotNil(t, testFile)
+}
+
+/*
+func TestResolveId(t *testing.T) {
+	testFile := NewResource("file://../../README.md")
+	assert.NotNil(t, testFile)
+
+	absolutePath, e := filepath.Abs("../../README.md")
+	assert.Nil(t, e)
+
+	testFile.ResolveId(context.Background())
+	assert.Equal(t, absolutePath, testFile.(*File).Path)
+}
+*/

internal/source/http.go

@@ -0,0 +1,88 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+_	"context"
+_	"encoding/json"
+_	"fmt"
+_	"gopkg.in/yaml.v3"
+	"net/url"
+	"net/http"
+_	"path/filepath"
+	"decl/internal/resource"
+	"decl/internal/iofilter"
+	"decl/internal/signature"
+_	"os"
+	"io"
+	"errors"
+	"crypto/sha256"
+)
+
+type HTTP struct {
+	Endpoint string `yaml:"endpoint" json:"endpoint"`
+}
+
+func NewHTTP() *HTTP {
+	return &HTTP{}
+}
+
+func init() {
+	SourceTypes.Register([]string{"http","https"}, func(u *url.URL) DocSource {
+		t := NewHTTP()
+		t.Endpoint = u.String()
+		return t
+	})
+}
+
+func (d *HTTP) Type() string { return "http" }
+
+func (h *HTTP) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) {
+	documents := make([]*resource.Document, 0, 100)
+	documents = append(documents, resource.NewDocument())
+
+	resp, err := http.Get(h.Endpoint)
+	if err != nil {
+		return documents, err
+	}
+	defer resp.Body.Close()
+	signature := resp.Header.Get("Signature")
+	hash := sha256.New()
+	sumReadData := iofilter.New(resp.Body, func(p []byte, readn int, readerr error) (n int, err error) {
+		hash.Write(p)
+		return 
+	})
+
+	decoder := resource.NewYAMLDecoder(sumReadData)
+	index := 0
+	for {
+		doc := documents[index]
+		e := decoder.Decode(doc)
+		if errors.Is(e, io.EOF) {
+			if len(documents) > 1 {
+				documents[index] = nil
+			}
+			break
+		}
+		if e != nil {
+			return documents, e
+		}
+		if validationErr := doc.Validate(); validationErr != nil {
+			return documents, validationErr
+		}
+/*
+		if applyErr := doc.Apply(); applyErr != nil {
+			return documents, applyErr
+		}
+*/
+		documents = append(documents, resource.NewDocument())
+		index++
+	}
+
+	if signature != "" {
+		sig := &signature.&Ident{}
+		sig.VerifySum(hash.Sum(nil), signature)
+	}
+
+	return documents, nil
+}

internal/source/http_test.go

@@ -0,0 +1,13 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestNewHTTPSource(t *testing.T) {
+	h := NewHTTP()
+	assert.NotNil(t, h)
+}

internal/source/tar.go

@@ -0,0 +1,102 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+_	"context"
+_	"encoding/json"
+_	"fmt"
+_	"gopkg.in/yaml.v3"
+	"net/url"
+	"path/filepath"
+	"decl/internal/resource"
+	"compress/gzip"
+	"archive/tar"
+	"regexp"
+	"os"
+	"io"
+)
+
+type Tar struct {
+	Path string `yaml:"path" json:"path"`
+}
+
+func NewTar() *Tar {
+	return &Tar{}
+}
+
+func init() {
+	SourceTypes.Register([]string{"tar"}, func(u *url.URL) DocSource {
+		t := NewTar()
+		t.Path,_ = filepath.Abs(filepath.Join(u.Hostname(), u.Path))
+		return t
+	})
+
+	SourceTypes.Register([]string{"tar.gz", "tgz"}, func(u *url.URL) DocSource {
+		t := NewTar()
+		if u.Scheme == "file" {
+			fileAbsolutePath, _ := filepath.Abs(filepath.Join(u.Hostname(), u.RequestURI()))
+			t.Path = fileAbsolutePath
+		} else {
+			t.Path = filepath.Join(u.Hostname(), u.Path)
+		}
+		return t
+	})
+
+}
+
+
+func (t *Tar) Type() string { return "tar" }
+
+func (t *Tar) ExtractResources(filter ResourceSelector) ([]*resource.Document, error) {
+	documents := make([]*resource.Document, 0, 100)
+	d := resource.NewDocument()
+	documents = append(documents, d)
+
+	TarGzipFileName := regexp.MustCompile(`^.*\.(tar\.gz|tgz)$`)
+	TarFileName := regexp.MustCompile(`^.*\.tar$`)
+
+	file, fileErr := os.Open(t.Path)
+	if fileErr != nil {
+		return documents, fileErr
+	}
+	var gzipReader io.Reader
+	switch t.Path {
+	case TarGzipFileName.FindString(t.Path):
+		zr, err := gzip.NewReader(file)
+		if err != nil {
+			return documents, err
+		}
+		gzipReader = zr
+		fallthrough
+	case TarFileName.FindString(t.Path):
+		var fileReader io.Reader
+		if gzipReader == nil {
+			fileReader = file
+		} else {
+			fileReader = gzipReader
+		}
+		tarReader := tar.NewReader(fileReader)
+		for {
+			hdr, err := tarReader.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return documents, err
+			}
+			f := resource.NewFile()
+			f.Path = hdr.Name
+			if fiErr := f.UpdateAttributesFromFileInfo(hdr.FileInfo()); fiErr != nil {
+				return documents, fiErr
+			}
+			readFileData, readErr := io.ReadAll(tarReader)
+			if readErr != nil {
+				return documents, readErr
+			}
+			f.Content = string(readFileData)
+			d.AddResourceDeclaration("file", f)
+		}
+	}
+	return documents, nil
+}

internal/source/tar_test.go

@@ -0,0 +1,14 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestNewTarSource(t *testing.T) {
+	s := NewTar()
+	assert.NotNil(t, s)
+}
+

internal/source/types.go

@@ -0,0 +1,92 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"path/filepath"
+)
+
+var (
+	ErrUnknownSourceType        = errors.New("Unknown source type")
+	SourceTypes          *Types = NewTypes()
+)
+
+type TypeName string //`json:"type"`
+
+type TypeFactory func(*url.URL) DocSource
+
+type Types struct {
+	registry map[string]TypeFactory
+}
+
+func NewTypes() *Types {
+	return &Types{registry: make(map[string]TypeFactory)}
+}
+
+func (t *Types) Register(names []string, factory TypeFactory) {
+	for _,name := range names {
+		t.registry[name] = factory
+	}
+}
+
+func (t *Types) FromExtension(path string) (TypeFactory, error) {
+	elements := strings.Split(path, ".")
+	numberOfElements := len(elements)
+	if numberOfElements > 2 {
+		if src := t.Get(strings.Join(elements[numberOfElements - 2: numberOfElements - 1], ".")); src != nil {
+			return src, nil
+		}
+	}
+	if src := t.Get(elements[numberOfElements - 1]); src != nil {
+		return src, nil
+	}
+	return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, path)
+}
+
+func (t *Types) New(uri string) (DocSource, error) {
+	u, e := url.Parse(uri)
+	if u == nil || e != nil {
+		return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, e)
+	}
+
+	if u.Scheme == "" {
+		u.Scheme = "file"
+	}
+
+	path := filepath.Join(u.Hostname(), u.Path)
+	if d, lookupErr := t.FromExtension(path); d != nil {
+		return d(u), lookupErr
+	}
+
+	if r, ok := t.registry[u.Scheme]; ok {
+		return r(u), nil
+	}
+	return nil, fmt.Errorf("%w: %s", ErrUnknownSourceType, u.Scheme)
+}
+
+func (t *Types) Has(typename string) bool {
+	if _, ok := t.registry[typename]; ok {
+		return true
+	}
+	return false
+}
+
+func (t *Types) Get(typename string) TypeFactory {
+	if d, ok := t.registry[typename]; ok {
+		return d
+	}
+	return nil
+}
+
+func (n *TypeName) UnmarshalJSON(b []byte) error {
+	SourceTypeName := strings.Trim(string(b), "\"")
+	if SourceTypes.Has(SourceTypeName) {
+		*n = TypeName(SourceTypeName)
+		return nil
+	}
+	return fmt.Errorf("%w: %s", ErrUnknownSourceType, SourceTypeName)
+}

internal/source/types_test.go

@@ -0,0 +1,89 @@
+// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
+
+package source
+
+import (
+_	"context"
+	"encoding/json"
+	"github.com/stretchr/testify/assert"
+	"net/url"
+	"testing"
+	"decl/internal/resource"
+)
+
+type MockDocSource struct {
+	InjectType func() string
+	InjectExtractResources func(uri string, filter ResourceSelector) ([]*resource.Document, error)
+}
+
+func (m *MockDocSource) Type() string { return m.InjectType() }
+func (m *MockDocSource) ExtractResources(uri string, filter ResourceSelector) ([]*resource.Document, error) { return m.InjectExtractResources(uri, filter) }
+
+func NewFooDocSource() DocSource {
+	return &MockDocSource{
+		InjectType: func() string { return "foo" },
+		InjectExtractResources: func(uri string, filter ResourceSelector) ([]*resource.Document, error) { return nil,nil },
+	}
+}
+
+func NewFileDocSource() DocSource {
+	return &MockDocSource{
+		InjectType: func() string { return "file" },
+		InjectExtractResources: func(uri string, filter ResourceSelector) ([]*resource.Document, error) { return nil,nil },
+	}
+}
+
+func TestNewSourceTypes(t *testing.T) {
+	sourceTypes := NewTypes()
+	assert.NotNil(t, sourceTypes)
+}
+
+func TestNewSourceTypesRegister(t *testing.T) {
+	m := NewFooDocSource()
+
+	sourceTypes := NewTypes()
+	assert.NotNil(t, sourceTypes)
+
+	sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m })
+
+	r, e := sourceTypes.New("foo://")
+	assert.Nil(t, e)
+	assert.Equal(t, m, r)
+}
+
+func TestResourceTypesFromURI(t *testing.T) {
+	m := NewFooDocSource()
+
+	sourceTypes := NewTypes()
+	assert.NotNil(t, sourceTypes)
+
+	sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m })
+
+	r, e := sourceTypes.New("foo://bar")
+	assert.Nil(t, e)
+	assert.Equal(t, m, r)
+}
+
+func TestResourceTypesHasType(t *testing.T) {
+	m := NewFooDocSource()
+
+	sourceTypes := NewTypes()
+	assert.NotNil(t, sourceTypes)
+
+	sourceTypes.Register([]string{"foo"}, func(*url.URL) DocSource { return m })
+
+	assert.True(t, sourceTypes.Has("foo"))
+}
+
+func TestDocSourceTypeName(t *testing.T) {
+	SourceTypes.Register([]string{"file"}, func(*url.URL) DocSource { return NewFileDocSource() })
+
+	type fDocSourceName struct {
+		Name TypeName `json:"type"`
+	}
+	fTypeName := &fDocSourceName{}
+	jsonType := `{ "type": "file" }`
+	e := json.Unmarshal([]byte(jsonType), &fTypeName)
+	assert.Nil(t, e)
+	assert.Equal(t, "file", string(fTypeName.Name))
+}