jx/internal/resource/file.go
Matthew Rich 07808c62fd
Some checks failed
Declarative Tests / test (push) Waiting to run
Lint / golangci-lint (push) Has been cancelled
add support for remote command execution
2024-11-10 10:24:06 -08:00

823 lines
20 KiB
Go

// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.
package resource
import (
"context"
"errors"
"fmt"
"log/slog"
"gopkg.in/yaml.v3"
"encoding/json"
"io"
"io/fs"
"net/url"
"os"
"os/user"
"strconv"
"syscall"
"time"
"crypto/sha256"
"gitea.rosskeen.house/rosskeen.house/machine"
"decl/internal/codec"
"decl/internal/iofilter"
"decl/internal/data"
"decl/internal/folio"
"decl/internal/transport"
"strings"
"embed"
"compress/gzip"
)
const (
FileTypeName TypeName = "file"
)
// Describes the type of file the resource represents
type FileType string
// Supported file types
const (
RegularFile FileType = "regular"
DirectoryFile FileType = "directory"
BlockDeviceFile FileType = "block"
CharacterDeviceFile FileType = "char"
NamedPipeFile FileType = "pipe"
SymbolicLinkFile FileType = "symlink"
SocketFile FileType = "socket"
)
var ErrInvalidFileInfo error = errors.New("Invalid FileInfo")
var ErrInvalidFileMode error = errors.New("Invalid Mode")
var ErrInvalidFileOwner error = errors.New("Unknown User")
var ErrInvalidFileGroup error = errors.New("Unknown Group")
type FileMode string
func init() {
folio.DocumentRegistry.ResourceTypes.Register([]string{"file"}, func(u *url.URL) (res data.Resource) {
f := NewFile()
slog.Info("FileFactory", "uri", u)
if u != nil {
if err := folio.CastParsedURI(u).ConstructResource(f); err != nil {
panic(err)
}
}
return f
})
}
/*
Manage the state of file system objects
The file content may be serialized directly in the `Content` field
or the `ContentSourceRef/sourceref` may be used to refer to the source
of the content from which to stream the content.
The `SerializeContent` the flag allows forcing the content to be serialized in the output.
*/
type File struct {
*Common `json:",inline" yaml:",inline"`
Filesystem fs.FS `json:"-" yaml:"-"`
stater machine.Stater `json:"-" yaml:"-"`
basePath int `json:"-" yaml:"-"`
Owner string `json:"owner" yaml:"owner"`
Group string `json:"group" yaml:"group"`
Mode FileMode `json:"mode" yaml:"mode"`
Atime time.Time `json:"atime,omitempty" yaml:"atime,omitempty"`
Ctime time.Time `json:"ctime,omitempty" yaml:"ctime,omitempty"`
Mtime time.Time `json:"mtime,omitempty" yaml:"mtime,omitempty"`
Content string `json:"content,omitempty" yaml:"content,omitempty"`
ContentSourceRef folio.ResourceReference `json:"sourceref,omitempty" yaml:"sourceref,omitempty"`
Sha256 string `json:"sha256,omitempty" yaml:"sha256,omitempty"`
Size int64 `json:"size,omitempty" yaml:"size,omitempty"`
Target string `json:"target,omitempty" yaml:"target,omitempty"`
FileType FileType `json:"filetype" yaml:"filetype"`
SerializeContent bool `json:"serializecontent,omitempty" yaml:"serializecontent,omitempty"`
GzipContent bool `json:"gzipcontent,omitempty" yaml:"gzipcontent,omitempty"`
Resources data.ResourceMapper `json:"-" yaml:"-"`
}
type ResourceFileInfo struct {
resource *File
}
func NewFile() *File {
currentUser, _ := user.Current()
group, _ := user.LookupGroupId(currentUser.Gid)
f := &File{
Common: NewCommon(FileTypeName, true),
Owner: currentUser.Username,
Group: group.Name,
Mode: "0644",
FileType: RegularFile,
SerializeContent: false,
}
f.PathNormalization(false)
slog.Info("NewFile()", "file", f)
return f
}
func NewNormalizedFile() *File {
f := NewFile()
f.PathNormalization(true)
return f
}
func (f *File) Init(u data.URIParser) error {
if u == nil {
u = folio.URI(f.URI()).Parse()
}
return f.SetParsedURI(u)
}
func (f *File) NormalizePath() error {
return f.Common.NormalizePath()
}
func (f *File) ContentType() string {
var ext strings.Builder
if f.parsedURI.Scheme != "file" {
return f.parsedURI.Scheme
}
if f.fileext == "" {
return f.exttype
}
ext.WriteString(f.exttype)
ext.WriteRune('.')
ext.WriteString(f.fileext)
return ext.String()
}
func (f *File) SetResourceMapper(resources data.ResourceMapper) {
f.Resources = resources
}
func (f *File) Clone() data.Resource {
return &File {
Common: f.Common.Clone(),
Owner: f.Owner,
Group: f.Group,
Mode: f.Mode,
Atime: f.Atime,
Ctime: f.Ctime,
Mtime: f.Mtime,
Content: f.Content,
Sha256: f.Sha256,
Size: f.Size,
Target: f.Target,
FileType: f.FileType,
}
}
func (f *File) StateMachine() machine.Stater {
if f.stater == nil {
f.stater = StorageMachine(f)
}
return f.stater
}
func (f *File) Notify(m *machine.EventMessage) {
ctx := context.Background()
slog.Info("Notify()", "file", f, "m", m)
switch m.On {
case machine.ENTERSTATEEVENT:
switch m.Dest {
case "start_stat":
if statErr := f.ReadStat(); statErr == nil {
if triggerErr := f.StateMachine().Trigger("exists"); triggerErr == nil {
return
}
} else {
if triggerErr := f.StateMachine().Trigger("notexists"); triggerErr == nil {
return
}
}
case "start_read":
if _,readErr := f.Read(ctx); readErr == nil {
if triggerErr := f.StateMachine().Trigger("state_read"); triggerErr == nil {
return
} else {
_ = f.AddError(triggerErr)
}
} else {
_ = f.AddError(readErr)
if f.IsResourceInconsistent() {
if triggerErr := f.StateMachine().Trigger("read-failed"); triggerErr == nil {
panic(readErr)
} else {
_ = f.AddError(triggerErr)
panic(fmt.Errorf("%w - %w", readErr, triggerErr))
}
}
_ = f.AddError(f.StateMachine().Trigger("notexists"))
}
case "start_create":
if createErr := f.Create(ctx); createErr == nil {
if triggerErr := f.StateMachine().Trigger("created"); triggerErr == nil {
return
} else {
_ = f.AddError(triggerErr)
}
} else {
_ = f.AddError(createErr)
if f.IsResourceInconsistent() {
if triggerErr := f.StateMachine().Trigger("create-failed"); triggerErr == nil {
panic(createErr)
} else {
_ = f.AddError(triggerErr)
panic(fmt.Errorf("%w - %w", createErr, triggerErr))
}
}
_ = f.StateMachine().Trigger("notexists")
panic(createErr)
}
case "start_update":
if updateErr := f.Update(ctx); updateErr == nil {
if triggerErr := f.stater.Trigger("updated"); triggerErr == nil {
return
} else {
_ = f.AddError(triggerErr)
}
} else {
_ = f.AddError(updateErr)
if f.IsResourceInconsistent() {
if triggerErr := f.StateMachine().Trigger("update-failed"); triggerErr == nil {
panic(updateErr)
} else {
panic(fmt.Errorf("%w - %w", updateErr, triggerErr))
}
}
_ = f.StateMachine().Trigger("notexists")
panic(updateErr)
}
case "start_delete":
if deleteErr := f.Delete(ctx); deleteErr == nil {
if triggerErr := f.StateMachine().Trigger("deleted"); triggerErr == nil {
return
} else {
f.Common.State = "present"
panic(triggerErr)
}
} else {
_ = f.StateMachine().Trigger("exists")
panic(deleteErr)
}
case "inconsistent":
f.Common.State = "inconsistent"
case "absent":
f.Common.State = "absent"
case "present", "created", "read":
f.Common.State = "present"
}
case machine.EXITSTATEEVENT:
switch m.Dest {
case "start_create":
slog.Info("File.Notify - EXITSTATE", "dest", m.Dest, "common.state", f.Common.State)
}
}
}
func (f *File) SetGzipContent(flag bool) {
f.GzipContent = flag
}
func (f *File) FilePath() string {
return f.Common.Path
}
func (f *File) SetFS(fsys fs.FS) {
f.Filesystem = fsys
}
func (f *File) URI() string {
return fmt.Sprintf("file://%s", f.Common.Path)
}
func (f *File) RelativePath() string {
return f.Common.Path[f.basePath:]
}
func (f *File) SetBasePath(index int) {
if index < len(f.Common.Path) {
f.basePath = index
}
}
func (f *File) DetectGzip() bool {
return (f.parsedURI.Query().Get("gzip") == "true" || f.fileext == "gz" || f.exttype == "tgz" || f.exttype == "gz" || f.fileext == "tgz" )
}
func (f *File) JSON() ([]byte, error) {
return json.Marshal(f)
}
func (f *File) Validate() (err error) {
var fileJson []byte
if fileJson, err = f.JSON(); err == nil {
s := NewSchema(f.Type())
err = s.Validate(string(fileJson))
}
return err
}
func (f *File) Apply() error {
ctx := context.Background()
switch f.Common.State {
case "absent":
return f.Delete(ctx)
case "present":
return f.Create(ctx)
}
return nil
}
func (f *File) Load(docData []byte, format codec.Format) (err error) {
err = format.StringDecoder(string(docData)).Decode(f)
if err == nil {
f.UpdateContentAttributes()
}
return
}
func (f *File) LoadReader(r io.ReadCloser, format codec.Format) (err error) {
err = format.Decoder(r).Decode(f)
if err == nil {
f.UpdateContentAttributes()
}
return
}
func (f *File) LoadString(docData string, format codec.Format) (err error) {
err = format.StringDecoder(docData).Decode(f)
if err == nil {
f.UpdateContentAttributes()
}
return
}
func (f *File) LoadDecl(yamlResourceDeclaration string) (err error) {
return f.LoadString(yamlResourceDeclaration, codec.FormatYaml)
}
func (f *File) ResolveId(ctx context.Context) string {
if e := f.NormalizePath(); e != nil {
panic(e)
}
return f.Common.Path
}
/*
func (f *File) NormalizePath() (err error) {
if f.config != nil {
if prefixPath, configErr := f.config.GetValue("prefix"); configErr == nil {
f.Path = filepath.Join(prefixPath.(string), f.Path)
}
}
if f.absPath, err = filepath.Abs(f.Path); err == nil && f.normalizePath {
f.Path = f.absPath
}
return
}
*/
func (f *File) GetContentSourceRef() string {
return string(f.ContentSourceRef)
}
func (f *File) SetContentSourceRef(uri string) {
f.Size = 0
f.ContentSourceRef = folio.ResourceReference(uri)
}
func (f *File) Stat() (info fs.FileInfo, err error) {
if _, ok := f.Filesystem.(embed.FS); ok {
info, err = fs.Stat(f.Filesystem, f.Common.Path)
} else {
info, err = os.Lstat(f.absPath)
}
return
}
func (f *File) FileInfo() fs.FileInfo {
return &ResourceFileInfo{ resource: f }
}
func (f *ResourceFileInfo) Name() string {
// return filepath.Base(f.resource.Path)
return f.resource.RelativePath()
}
func (f *ResourceFileInfo) Size() int64 {
return f.resource.Size
}
func (f *ResourceFileInfo) Mode() (mode os.FileMode) {
if fileMode, fileModeErr := f.resource.Mode.GetMode(); fileModeErr == nil {
mode |= fileMode
} else {
panic(fileModeErr)
}
mode |= f.resource.FileType.GetMode()
return
}
func (f *ResourceFileInfo) ModTime() time.Time {
return f.resource.Mtime
}
func (f *ResourceFileInfo) IsDir() bool {
return f.resource.FileType == DirectoryFile
}
func (f *ResourceFileInfo) Sys() any {
return nil
}
func (f *File) Create(ctx context.Context) error {
slog.Info("File.Create()", "file", f)
uid, uidErr := LookupUID(f.Owner)
if uidErr != nil {
return fmt.Errorf("%w: unkwnon user %d", ErrInvalidFileOwner, uid)
}
gid, gidErr := LookupGID(f.Group)
if gidErr != nil {
return fmt.Errorf("%w: unkwnon group %d", ErrInvalidFileGroup, gid)
}
mode, modeErr := f.Mode.GetMode()
if modeErr != nil {
return modeErr
}
switch f.FileType {
case SymbolicLinkFile:
linkErr := os.Symlink(f.Target, f.Common.Path)
if linkErr != nil {
return linkErr
}
case DirectoryFile:
if mkdirErr := os.MkdirAll(f.Common.Path, mode); mkdirErr != nil {
return mkdirErr
}
default:
fallthrough
case RegularFile:
copyBuffer := make([]byte, 32 * 1024)
hash := sha256.New()
f.Size = 0
var contentReader io.ReadCloser
if len(f.Content) == 0 && len(f.ContentSourceRef) != 0 {
if refReader, err := f.ContentSourceRef.Lookup(nil).ContentReaderStream(); err == nil {
contentReader = refReader
} else {
return err
}
} else {
contentReader = io.NopCloser(strings.NewReader(f.Content))
}
sumReadData := iofilter.NewReader(contentReader, func(p []byte, readn int, readerr error) (n int, err error) {
hash.Write(p[:readn])
f.Size += int64(readn)
return readn, readerr
})
var createdFileWriter io.WriteCloser
createdFile, fileErr := os.Create(f.Common.Path)
slog.Info("File.Create(): os.Create()", "path", f.Common.Path, "error", fileErr)
if fileErr != nil {
return fileErr
}
if f.GzipContent && f.DetectGzip() {
createdFileWriter = gzip.NewWriter(createdFile)
defer createdFileWriter.Close()
} else {
createdFileWriter = createdFile
}
defer createdFile.Close()
slog.Info("File.Create(): Chmod()", "path", f.Common.Path, "mode", mode)
if chmodErr := createdFile.Chmod(mode); chmodErr != nil {
return chmodErr
}
_, writeErr := io.CopyBuffer(createdFileWriter, sumReadData, copyBuffer)
if writeErr != nil {
return fmt.Errorf("File.Create(): CopyBuffer failed %v %v: %w", createdFileWriter, contentReader, writeErr)
}
f.Sha256 = fmt.Sprintf("%x", hash.Sum(nil))
if !f.Mtime.IsZero() && !f.Atime.IsZero() {
slog.Info("File.Create(): Chtimes()", "path", f.Common.Path, "atime", f.Atime, "mtime", f.Mtime)
if chtimesErr := os.Chtimes(f.Common.Path, f.Atime, f.Mtime); chtimesErr != nil {
return chtimesErr
}
} else {
slog.Info("File.Create(): Chtimes() SKIPPED", "path", f.Common.Path, "atime", f.Atime, "mtime", f.Mtime)
}
}
slog.Info("File.Create(): Chown()", "path", f.Common.Path, "uid", uid, "gid", gid)
if chownErr := os.Chown(f.Common.Path, uid, gid); chownErr != nil {
return chownErr
}
f.Common.State = "present"
return nil
}
func (f *File) Update(ctx context.Context) error {
return f.Create(ctx)
}
func (f *File) Delete(ctx context.Context) error {
return os.Remove(f.Common.Path)
}
func (f *File) UpdateContentAttributes() {
f.Size = int64(len(f.Content))
f.Sha256 = fmt.Sprintf("%x", sha256.Sum256([]byte(f.Content)))
}
func (f *File) SetFileInfo(info os.FileInfo) error {
return f.UpdateAttributesFromFileInfo(info)
}
func (f *File) UpdateAttributesFromFileInfo(info os.FileInfo) error {
if info != nil {
f.Mtime = info.ModTime()
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
f.Atime = time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec))
f.Ctime = time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec))
userId := strconv.Itoa(int(stat.Uid))
groupId := strconv.Itoa(int(stat.Gid))
fileUser, userErr := user.LookupId(userId)
if userErr != nil { //UnknownUserIdError
//panic(userErr)
f.Owner = userId
} else {
f.Owner = fileUser.Username
}
fileGroup, groupErr := user.LookupGroupId(groupId)
if groupErr != nil {
//panic(groupErr)
f.Group = groupId
} else {
f.Group = fileGroup.Name
}
}
f.Size = info.Size()
f.Mode = FileMode(fmt.Sprintf("%04o", info.Mode().Perm()))
f.FileType.SetMode(info.Mode())
return nil
}
return ErrInvalidFileInfo
}
func (f *File) ContentSourceRefStat() (info fs.FileInfo) {
if len(f.ContentSourceRef) > 0 {
rs, _ := f.ContentReaderStream()
info, _ = rs.Stat()
rs.Close()
}
return
}
func (f *File) ReadStat() (err error) {
var info fs.FileInfo
slog.Info("ReadStat()", "filesystem", f.Filesystem, "path", f.Common.Path)
info, err = f.Stat()
slog.Info("ReadStat()", "filesystem", f.Filesystem, "path", f.Common.Path, "info", info, "error", err)
if err == nil {
_ = f.SetFileInfo(info)
} else {
if refStat := f.ContentSourceRefStat(); refStat != nil {
_ = f.SetFileInfo(refStat)
//f.Size = refStat.Size()
err = nil
}
// XXX compare the mod times and set state to outdated
}
slog.Info("ReadStat()", "stat", info, "path", f.Common.Path)
if err != nil {
f.Common.State = "absent"
return
}
return
}
func (f *File) open() (file io.ReadCloser, err error) {
slog.Info("open()", "file", f.Common.Path, "fs", f.Filesystem)
if _, ok := f.Filesystem.(embed.FS); ok {
file, err = f.Filesystem.Open(f.Common.Path)
} else {
file, err = os.Open(f.Common.Path)
}
if f.GzipContent && f.DetectGzip() {
file, err = gzip.NewReader(file)
}
slog.Info("open()", "file", f.Common.Path, "error", err)
return
}
func (f *File) Read(ctx context.Context) ([]byte, error) {
/*
if normalizePathErr := f.NormalizePath(); normalizePathErr != nil {
return nil, normalizePathErr
}
*/
statErr := f.ReadStat()
if statErr != nil {
return nil, fmt.Errorf("%w - %w: %s", ErrResourceStateAbsent, statErr, f.Path)
}
switch f.FileType {
case RegularFile:
if len(f.ContentSourceRef) == 0 || f.SerializeContent {
//file, fileErr := os.Open(f.Path)
file, fileErr := f.open()
if fileErr != nil {
panic(fileErr)
}
fileContent, ioErr := io.ReadAll(file)
if ioErr != nil {
panic(ioErr)
}
f.Content = string(fileContent)
f.Sha256 = fmt.Sprintf("%x", sha256.Sum256(fileContent))
}
case SymbolicLinkFile:
linkTarget, pathErr := os.Readlink(f.Common.Path)
if pathErr != nil {
return nil, pathErr
}
f.Target = linkTarget
}
f.Common.State = "present"
return yaml.Marshal(f)
}
// set up reader for source content
func (f *File) readThru() (contentReader io.ReadCloser, err error) {
if len(f.ContentSourceRef) != 0 {
contentReader, err = f.ContentSourceRef.Lookup(nil).ContentReaderStream()
if f.GzipContent {
contentReader.(*transport.Reader).DetectGzip()
} else {
contentReader.(*transport.Reader).SetGzip(false)
}
slog.Info("File.readThru()", "reader", contentReader)
} else {
if len(f.Content) != 0 {
contentReader = io.NopCloser(strings.NewReader(f.Content))
} else {
//contentReader, err = os.Open(f.Path)
contentReader, err = f.open()
}
}
contentReader = f.UpdateContentAttributesFromReader(contentReader)
return
}
func (f *File) UpdateContentAttributesFromReader(reader io.ReadCloser) io.ReadCloser {
var content strings.Builder
hash := sha256.New()
f.Size = 0
f.Content = ""
f.Sha256 = ""
return iofilter.NewReader(reader, func(p []byte, readn int, readerr error) (n int, err error) {
hash.Write(p[:readn])
f.Sha256 = fmt.Sprintf("%x", hash.Sum(nil))
f.Size += int64(readn)
if len(f.ContentSourceRef) == 0 || f.SerializeContent {
content.Write(p[:readn])
f.Content = content.String()
}
return readn, readerr
})
}
func (f *File) SetContent(r io.Reader) error {
fileContent, ioErr := io.ReadAll(r)
f.Content = string(fileContent)
f.UpdateContentAttributes()
return ioErr
}
func (f *File) GetContent(w io.Writer) (contentReader io.ReadCloser, err error) {
slog.Info("File.GetContent()", "content", len(f.Content), "sourceref", f.ContentSourceRef)
switch f.FileType {
case RegularFile:
contentReader, err = f.readThru()
if w != nil {
copyBuffer := make([]byte, 32 * 1024)
_, writeErr := io.CopyBuffer(w, contentReader, copyBuffer)
if writeErr != nil {
return nil, fmt.Errorf("File.GetContent(): CopyBuffer failed %v %v: %w", w, contentReader, writeErr)
}
return nil, nil
}
}
return
}
func (f *File) ContentReaderStream() (*transport.Reader, error) {
if len(f.Content) == 0 && len(f.ContentSourceRef) != 0 {
return f.ContentSourceRef.Lookup(nil).ContentReaderStream()
}
return nil, fmt.Errorf("Cannot provide transport reader for string content")
}
// ContentWriterStream() would not provide a mechanism to keep the in-memory state up-to-date
func (f *File) ContentWriterStream() (*transport.Writer, error) {
if len(f.Content) == 0 && len(f.ContentSourceRef) != 0 {
return f.ContentSourceRef.Lookup(nil).ContentWriterStream()
}
return nil, fmt.Errorf("Cannot provide transport writer for string content")
}
func (f *File) GetTarget() string { return f.Target }
func (f *File) Type() string { return "file" }
func (f *FileType) UnmarshalYAML(value *yaml.Node) error {
var s string
if err := value.Decode(&s); err != nil {
return err
}
switch s {
case string(RegularFile), string(DirectoryFile), string(BlockDeviceFile), string(CharacterDeviceFile), string(NamedPipeFile), string(SymbolicLinkFile), string(SocketFile):
*f = FileType(s)
return nil
default:
return errors.New("invalid FileType value")
}
}
func (f *FileType) SetMode(mode os.FileMode) {
switch true {
case mode.IsRegular():
*f = RegularFile
case mode.IsDir():
*f = DirectoryFile
case mode&os.ModeSymlink != 0:
*f = SymbolicLinkFile
case mode&os.ModeNamedPipe != 0:
*f = NamedPipeFile
case mode&os.ModeSocket != 0:
*f = SocketFile
case mode&os.ModeCharDevice != 0:
*f = CharacterDeviceFile
case mode&os.ModeDevice != 0:
*f = BlockDeviceFile
}
}
func (f *FileType) GetMode() (mode os.FileMode) {
switch *f {
case RegularFile:
case DirectoryFile:
mode |= os.ModeDir
case SymbolicLinkFile:
mode |= os.ModeSymlink
case NamedPipeFile:
mode |= os.ModeNamedPipe
case SocketFile:
mode |= os.ModeSocket
case CharacterDeviceFile:
mode |= os.ModeCharDevice
case BlockDeviceFile:
mode |= os.ModeDevice
}
return
}
func (f *FileMode) GetMode() (os.FileMode, error) {
if mode, modeErr := strconv.ParseInt(string(*f), 8, 64); modeErr != nil {
return os.FileMode(mode), fmt.Errorf("%w: %s invalid mode %d - %w", ErrInvalidFileMode, *f, mode, modeErr)
} else {
return os.FileMode(mode), nil
}
}