// Copyright 2024 Matthew Rich . All rights reserved. package resource import ( "context" "errors" "fmt" "log/slog" "gopkg.in/yaml.v3" "encoding/json" "io" "io/fs" "net/url" "os" "os/user" "strconv" "syscall" "time" "crypto/sha256" "gitea.rosskeen.house/rosskeen.house/machine" "decl/internal/codec" "decl/internal/iofilter" "decl/internal/data" "decl/internal/folio" "decl/internal/transport" "strings" "embed" "compress/gzip" ) const ( FileTypeName TypeName = "file" ) // Describes the type of file the resource represents type FileType string // Supported file types const ( RegularFile FileType = "regular" DirectoryFile FileType = "directory" BlockDeviceFile FileType = "block" CharacterDeviceFile FileType = "char" NamedPipeFile FileType = "pipe" SymbolicLinkFile FileType = "symlink" SocketFile FileType = "socket" ) var ErrInvalidFileInfo error = errors.New("Invalid FileInfo") var ErrInvalidFileMode error = errors.New("Invalid Mode") var ErrInvalidFileOwner error = errors.New("Unknown User") var ErrInvalidFileGroup error = errors.New("Unknown Group") type FileMode string func init() { folio.DocumentRegistry.ResourceTypes.Register([]string{"file"}, func(u *url.URL) (res data.Resource) { f := NewFile() slog.Info("FileFactory", "uri", u) if u != nil { if err := folio.CastParsedURI(u).ConstructResource(f); err != nil { panic(err) } } return f }) } /* Manage the state of file system objects The file content may be serialized directly in the `Content` field or the `ContentSourceRef/sourceref` may be used to refer to the source of the content from which to stream the content. The `SerializeContent` the flag allows forcing the content to be serialized in the output. */ type File struct { *Common `json:",inline" yaml:",inline"` Filesystem fs.FS `json:"-" yaml:"-"` stater machine.Stater `json:"-" yaml:"-"` basePath int `json:"-" yaml:"-"` Owner string `json:"owner" yaml:"owner"` Group string `json:"group" yaml:"group"` Mode FileMode `json:"mode" yaml:"mode"` Atime time.Time `json:"atime,omitempty" yaml:"atime,omitempty"` Ctime time.Time `json:"ctime,omitempty" yaml:"ctime,omitempty"` Mtime time.Time `json:"mtime,omitempty" yaml:"mtime,omitempty"` Content string `json:"content,omitempty" yaml:"content,omitempty"` ContentSourceRef folio.ResourceReference `json:"sourceref,omitempty" yaml:"sourceref,omitempty"` Sha256 string `json:"sha256,omitempty" yaml:"sha256,omitempty"` Size int64 `json:"size,omitempty" yaml:"size,omitempty"` Target string `json:"target,omitempty" yaml:"target,omitempty"` FileType FileType `json:"filetype" yaml:"filetype"` SerializeContent bool `json:"serializecontent,omitempty" yaml:"serializecontent,omitempty"` GzipContent bool `json:"gzipcontent,omitempty" yaml:"gzipcontent,omitempty"` Resources data.ResourceMapper `json:"-" yaml:"-"` } type ResourceFileInfo struct { resource *File } func NewFile() *File { currentUser, _ := user.Current() group, _ := user.LookupGroupId(currentUser.Gid) f := &File{ Common: NewCommon(FileTypeName, true), Owner: currentUser.Username, Group: group.Name, Mode: "0644", FileType: RegularFile, SerializeContent: false, } f.PathNormalization(false) slog.Info("NewFile()", "file", f) return f } func NewNormalizedFile() *File { f := NewFile() f.PathNormalization(true) return f } func (f *File) Init(u data.URIParser) error { if u == nil { u = folio.URI(f.URI()).Parse() } return f.SetParsedURI(u) } func (f *File) NormalizePath() error { return f.Common.NormalizePath() } func (f *File) ContentType() string { var ext strings.Builder if f.parsedURI.Scheme != "file" { return f.parsedURI.Scheme } if f.fileext == "" { return f.exttype } ext.WriteString(f.exttype) ext.WriteRune('.') ext.WriteString(f.fileext) return ext.String() } func (f *File) SetResourceMapper(resources data.ResourceMapper) { f.Resources = resources } func (f *File) Clone() data.Resource { return &File { Common: f.Common.Clone(), Owner: f.Owner, Group: f.Group, Mode: f.Mode, Atime: f.Atime, Ctime: f.Ctime, Mtime: f.Mtime, Content: f.Content, Sha256: f.Sha256, Size: f.Size, Target: f.Target, FileType: f.FileType, } } func (f *File) StateMachine() machine.Stater { if f.stater == nil { f.stater = StorageMachine(f) } return f.stater } func (f *File) Notify(m *machine.EventMessage) { ctx := context.Background() slog.Info("Notify()", "file", f, "m", m) switch m.On { case machine.ENTERSTATEEVENT: switch m.Dest { case "start_stat": if statErr := f.ReadStat(); statErr == nil { if triggerErr := f.StateMachine().Trigger("exists"); triggerErr == nil { return } } else { if triggerErr := f.StateMachine().Trigger("notexists"); triggerErr == nil { return } } case "start_read": if _,readErr := f.Read(ctx); readErr == nil { if triggerErr := f.StateMachine().Trigger("state_read"); triggerErr == nil { return } else { _ = f.AddError(triggerErr) } } else { _ = f.AddError(readErr) if f.IsResourceInconsistent() { if triggerErr := f.StateMachine().Trigger("read-failed"); triggerErr == nil { panic(readErr) } else { _ = f.AddError(triggerErr) panic(fmt.Errorf("%w - %w", readErr, triggerErr)) } } _ = f.AddError(f.StateMachine().Trigger("notexists")) } case "start_create": if createErr := f.Create(ctx); createErr == nil { if triggerErr := f.StateMachine().Trigger("created"); triggerErr == nil { return } else { _ = f.AddError(triggerErr) } } else { _ = f.AddError(createErr) if f.IsResourceInconsistent() { if triggerErr := f.StateMachine().Trigger("create-failed"); triggerErr == nil { panic(createErr) } else { _ = f.AddError(triggerErr) panic(fmt.Errorf("%w - %w", createErr, triggerErr)) } } _ = f.StateMachine().Trigger("notexists") panic(createErr) } case "start_update": if updateErr := f.Update(ctx); updateErr == nil { if triggerErr := f.stater.Trigger("updated"); triggerErr == nil { return } else { _ = f.AddError(triggerErr) } } else { _ = f.AddError(updateErr) if f.IsResourceInconsistent() { if triggerErr := f.StateMachine().Trigger("update-failed"); triggerErr == nil { panic(updateErr) } else { panic(fmt.Errorf("%w - %w", updateErr, triggerErr)) } } _ = f.StateMachine().Trigger("notexists") panic(updateErr) } case "start_delete": if deleteErr := f.Delete(ctx); deleteErr == nil { if triggerErr := f.StateMachine().Trigger("deleted"); triggerErr == nil { return } else { f.Common.State = "present" panic(triggerErr) } } else { _ = f.StateMachine().Trigger("exists") panic(deleteErr) } case "inconsistent": f.Common.State = "inconsistent" case "absent": f.Common.State = "absent" case "present", "created", "read": f.Common.State = "present" } case machine.EXITSTATEEVENT: switch m.Dest { case "start_create": slog.Info("File.Notify - EXITSTATE", "dest", m.Dest, "common.state", f.Common.State) } } } func (f *File) SetGzipContent(flag bool) { f.GzipContent = flag } func (f *File) FilePath() string { return f.Common.Path } func (f *File) SetFS(fsys fs.FS) { f.Filesystem = fsys } func (f *File) URI() string { return fmt.Sprintf("file://%s", f.Common.Path) } func (f *File) RelativePath() string { return f.Common.Path[f.basePath:] } func (f *File) SetBasePath(index int) { if index < len(f.Common.Path) { f.basePath = index } } func (f *File) DetectGzip() bool { return (f.parsedURI.Query().Get("gzip") == "true" || f.fileext == "gz" || f.exttype == "tgz" || f.exttype == "gz" || f.fileext == "tgz" ) } func (f *File) JSON() ([]byte, error) { return json.Marshal(f) } func (f *File) Validate() (err error) { var fileJson []byte if fileJson, err = f.JSON(); err == nil { s := NewSchema(f.Type()) err = s.Validate(string(fileJson)) } return err } func (f *File) Apply() error { ctx := context.Background() switch f.Common.State { case "absent": return f.Delete(ctx) case "present": return f.Create(ctx) } return nil } func (f *File) Load(docData []byte, format codec.Format) (err error) { err = format.StringDecoder(string(docData)).Decode(f) if err == nil { f.UpdateContentAttributes() } return } func (f *File) LoadReader(r io.ReadCloser, format codec.Format) (err error) { err = format.Decoder(r).Decode(f) if err == nil { f.UpdateContentAttributes() } return } func (f *File) LoadString(docData string, format codec.Format) (err error) { err = format.StringDecoder(docData).Decode(f) if err == nil { f.UpdateContentAttributes() } return } func (f *File) LoadDecl(yamlResourceDeclaration string) (err error) { return f.LoadString(yamlResourceDeclaration, codec.FormatYaml) } func (f *File) ResolveId(ctx context.Context) string { if e := f.NormalizePath(); e != nil { panic(e) } return f.Common.Path } /* func (f *File) NormalizePath() (err error) { if f.config != nil { if prefixPath, configErr := f.config.GetValue("prefix"); configErr == nil { f.Path = filepath.Join(prefixPath.(string), f.Path) } } if f.absPath, err = filepath.Abs(f.Path); err == nil && f.normalizePath { f.Path = f.absPath } return } */ func (f *File) GetContentSourceRef() string { return string(f.ContentSourceRef) } func (f *File) SetContentSourceRef(uri string) { f.Size = 0 f.ContentSourceRef = folio.ResourceReference(uri) } func (f *File) Stat() (info fs.FileInfo, err error) { if _, ok := f.Filesystem.(embed.FS); ok { info, err = fs.Stat(f.Filesystem, f.Common.Path) } else { info, err = os.Lstat(f.absPath) } return } func (f *File) FileInfo() fs.FileInfo { return &ResourceFileInfo{ resource: f } } func (f *ResourceFileInfo) Name() string { // return filepath.Base(f.resource.Path) return f.resource.RelativePath() } func (f *ResourceFileInfo) Size() int64 { return f.resource.Size } func (f *ResourceFileInfo) Mode() (mode os.FileMode) { if fileMode, fileModeErr := f.resource.Mode.GetMode(); fileModeErr == nil { mode |= fileMode } else { panic(fileModeErr) } mode |= f.resource.FileType.GetMode() return } func (f *ResourceFileInfo) ModTime() time.Time { return f.resource.Mtime } func (f *ResourceFileInfo) IsDir() bool { return f.resource.FileType == DirectoryFile } func (f *ResourceFileInfo) Sys() any { return nil } func (f *File) Create(ctx context.Context) error { slog.Info("File.Create()", "file", f) uid, uidErr := LookupUID(f.Owner) if uidErr != nil { return fmt.Errorf("%w: unkwnon user %d", ErrInvalidFileOwner, uid) } gid, gidErr := LookupGID(f.Group) if gidErr != nil { return fmt.Errorf("%w: unkwnon group %d", ErrInvalidFileGroup, gid) } mode, modeErr := f.Mode.GetMode() if modeErr != nil { return modeErr } switch f.FileType { case SymbolicLinkFile: linkErr := os.Symlink(f.Target, f.Common.Path) if linkErr != nil { return linkErr } case DirectoryFile: if mkdirErr := os.MkdirAll(f.Common.Path, mode); mkdirErr != nil { return mkdirErr } default: fallthrough case RegularFile: copyBuffer := make([]byte, 32 * 1024) hash := sha256.New() f.Size = 0 var contentReader io.ReadCloser if len(f.Content) == 0 && len(f.ContentSourceRef) != 0 { if refReader, err := f.ContentSourceRef.Lookup(nil).ContentReaderStream(); err == nil { contentReader = refReader } else { return err } } else { contentReader = io.NopCloser(strings.NewReader(f.Content)) } sumReadData := iofilter.NewReader(contentReader, func(p []byte, readn int, readerr error) (n int, err error) { hash.Write(p[:readn]) f.Size += int64(readn) return readn, readerr }) var createdFileWriter io.WriteCloser createdFile, fileErr := os.Create(f.Common.Path) slog.Info("File.Create(): os.Create()", "path", f.Common.Path, "error", fileErr) if fileErr != nil { return fileErr } if f.GzipContent && f.DetectGzip() { createdFileWriter = gzip.NewWriter(createdFile) defer createdFileWriter.Close() } else { createdFileWriter = createdFile } defer createdFile.Close() slog.Info("File.Create(): Chmod()", "path", f.Common.Path, "mode", mode) if chmodErr := createdFile.Chmod(mode); chmodErr != nil { return chmodErr } _, writeErr := io.CopyBuffer(createdFileWriter, sumReadData, copyBuffer) if writeErr != nil { return fmt.Errorf("File.Create(): CopyBuffer failed %v %v: %w", createdFileWriter, contentReader, writeErr) } f.Sha256 = fmt.Sprintf("%x", hash.Sum(nil)) if !f.Mtime.IsZero() && !f.Atime.IsZero() { slog.Info("File.Create(): Chtimes()", "path", f.Common.Path, "atime", f.Atime, "mtime", f.Mtime) if chtimesErr := os.Chtimes(f.Common.Path, f.Atime, f.Mtime); chtimesErr != nil { return chtimesErr } } else { slog.Info("File.Create(): Chtimes() SKIPPED", "path", f.Common.Path, "atime", f.Atime, "mtime", f.Mtime) } } slog.Info("File.Create(): Chown()", "path", f.Common.Path, "uid", uid, "gid", gid) if chownErr := os.Chown(f.Common.Path, uid, gid); chownErr != nil { return chownErr } f.Common.State = "present" return nil } func (f *File) Update(ctx context.Context) error { return f.Create(ctx) } func (f *File) Delete(ctx context.Context) error { return os.Remove(f.Common.Path) } func (f *File) UpdateContentAttributes() { f.Size = int64(len(f.Content)) f.Sha256 = fmt.Sprintf("%x", sha256.Sum256([]byte(f.Content))) } func (f *File) SetFileInfo(info os.FileInfo) error { return f.UpdateAttributesFromFileInfo(info) } func (f *File) UpdateAttributesFromFileInfo(info os.FileInfo) error { if info != nil { f.Mtime = info.ModTime() if stat, ok := info.Sys().(*syscall.Stat_t); ok { f.Atime = time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec)) f.Ctime = time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)) userId := strconv.Itoa(int(stat.Uid)) groupId := strconv.Itoa(int(stat.Gid)) fileUser, userErr := user.LookupId(userId) if userErr != nil { //UnknownUserIdError //panic(userErr) f.Owner = userId } else { f.Owner = fileUser.Username } fileGroup, groupErr := user.LookupGroupId(groupId) if groupErr != nil { //panic(groupErr) f.Group = groupId } else { f.Group = fileGroup.Name } } f.Size = info.Size() f.Mode = FileMode(fmt.Sprintf("%04o", info.Mode().Perm())) f.FileType.SetMode(info.Mode()) return nil } return ErrInvalidFileInfo } func (f *File) ContentSourceRefStat() (info fs.FileInfo) { if len(f.ContentSourceRef) > 0 { rs, _ := f.ContentReaderStream() info, _ = rs.Stat() rs.Close() } return } func (f *File) ReadStat() (err error) { var info fs.FileInfo slog.Info("ReadStat()", "filesystem", f.Filesystem, "path", f.Common.Path) info, err = f.Stat() slog.Info("ReadStat()", "filesystem", f.Filesystem, "path", f.Common.Path, "info", info, "error", err) if err == nil { _ = f.SetFileInfo(info) } else { if refStat := f.ContentSourceRefStat(); refStat != nil { _ = f.SetFileInfo(refStat) //f.Size = refStat.Size() err = nil } // XXX compare the mod times and set state to outdated } slog.Info("ReadStat()", "stat", info, "path", f.Common.Path) if err != nil { f.Common.State = "absent" return } return } func (f *File) open() (file io.ReadCloser, err error) { slog.Info("open()", "file", f.Common.Path, "fs", f.Filesystem) if _, ok := f.Filesystem.(embed.FS); ok { file, err = f.Filesystem.Open(f.Common.Path) } else { file, err = os.Open(f.Common.Path) } if f.GzipContent && f.DetectGzip() { file, err = gzip.NewReader(file) } slog.Info("open()", "file", f.Common.Path, "error", err) return } func (f *File) Read(ctx context.Context) ([]byte, error) { /* if normalizePathErr := f.NormalizePath(); normalizePathErr != nil { return nil, normalizePathErr } */ statErr := f.ReadStat() if statErr != nil { return nil, fmt.Errorf("%w - %w: %s", ErrResourceStateAbsent, statErr, f.Path) } switch f.FileType { case RegularFile: if len(f.ContentSourceRef) == 0 || f.SerializeContent { //file, fileErr := os.Open(f.Path) file, fileErr := f.open() if fileErr != nil { panic(fileErr) } fileContent, ioErr := io.ReadAll(file) if ioErr != nil { panic(ioErr) } f.Content = string(fileContent) f.Sha256 = fmt.Sprintf("%x", sha256.Sum256(fileContent)) } case SymbolicLinkFile: linkTarget, pathErr := os.Readlink(f.Common.Path) if pathErr != nil { return nil, pathErr } f.Target = linkTarget } f.Common.State = "present" return yaml.Marshal(f) } // set up reader for source content func (f *File) readThru() (contentReader io.ReadCloser, err error) { if len(f.ContentSourceRef) != 0 { contentReader, err = f.ContentSourceRef.Lookup(nil).ContentReaderStream() if f.GzipContent { contentReader.(*transport.Reader).DetectGzip() } else { contentReader.(*transport.Reader).SetGzip(false) } slog.Info("File.readThru()", "reader", contentReader) } else { if len(f.Content) != 0 { contentReader = io.NopCloser(strings.NewReader(f.Content)) } else { //contentReader, err = os.Open(f.Path) contentReader, err = f.open() } } contentReader = f.UpdateContentAttributesFromReader(contentReader) return } func (f *File) UpdateContentAttributesFromReader(reader io.ReadCloser) io.ReadCloser { var content strings.Builder hash := sha256.New() f.Size = 0 f.Content = "" f.Sha256 = "" return iofilter.NewReader(reader, func(p []byte, readn int, readerr error) (n int, err error) { hash.Write(p[:readn]) f.Sha256 = fmt.Sprintf("%x", hash.Sum(nil)) f.Size += int64(readn) if len(f.ContentSourceRef) == 0 || f.SerializeContent { content.Write(p[:readn]) f.Content = content.String() } return readn, readerr }) } func (f *File) SetContent(r io.Reader) error { fileContent, ioErr := io.ReadAll(r) f.Content = string(fileContent) f.UpdateContentAttributes() return ioErr } func (f *File) GetContent(w io.Writer) (contentReader io.ReadCloser, err error) { slog.Info("File.GetContent()", "content", len(f.Content), "sourceref", f.ContentSourceRef) switch f.FileType { case RegularFile: contentReader, err = f.readThru() if w != nil { copyBuffer := make([]byte, 32 * 1024) _, writeErr := io.CopyBuffer(w, contentReader, copyBuffer) if writeErr != nil { return nil, fmt.Errorf("File.GetContent(): CopyBuffer failed %v %v: %w", w, contentReader, writeErr) } return nil, nil } } return } func (f *File) ContentReaderStream() (*transport.Reader, error) { if len(f.Content) == 0 && len(f.ContentSourceRef) != 0 { return f.ContentSourceRef.Lookup(nil).ContentReaderStream() } return nil, fmt.Errorf("Cannot provide transport reader for string content") } // ContentWriterStream() would not provide a mechanism to keep the in-memory state up-to-date func (f *File) ContentWriterStream() (*transport.Writer, error) { if len(f.Content) == 0 && len(f.ContentSourceRef) != 0 { return f.ContentSourceRef.Lookup(nil).ContentWriterStream() } return nil, fmt.Errorf("Cannot provide transport writer for string content") } func (f *File) GetTarget() string { return f.Target } func (f *File) Type() string { return "file" } func (f *FileType) UnmarshalYAML(value *yaml.Node) error { var s string if err := value.Decode(&s); err != nil { return err } switch s { case string(RegularFile), string(DirectoryFile), string(BlockDeviceFile), string(CharacterDeviceFile), string(NamedPipeFile), string(SymbolicLinkFile), string(SocketFile): *f = FileType(s) return nil default: return errors.New("invalid FileType value") } } func (f *FileType) SetMode(mode os.FileMode) { switch true { case mode.IsRegular(): *f = RegularFile case mode.IsDir(): *f = DirectoryFile case mode&os.ModeSymlink != 0: *f = SymbolicLinkFile case mode&os.ModeNamedPipe != 0: *f = NamedPipeFile case mode&os.ModeSocket != 0: *f = SocketFile case mode&os.ModeCharDevice != 0: *f = CharacterDeviceFile case mode&os.ModeDevice != 0: *f = BlockDeviceFile } } func (f *FileType) GetMode() (mode os.FileMode) { switch *f { case RegularFile: case DirectoryFile: mode |= os.ModeDir case SymbolicLinkFile: mode |= os.ModeSymlink case NamedPipeFile: mode |= os.ModeNamedPipe case SocketFile: mode |= os.ModeSocket case CharacterDeviceFile: mode |= os.ModeCharDevice case BlockDeviceFile: mode |= os.ModeDevice } return } func (f *FileMode) GetMode() (os.FileMode, error) { if mode, modeErr := strconv.ParseInt(string(*f), 8, 64); modeErr != nil { return os.FileMode(mode), fmt.Errorf("%w: %s invalid mode %d - %w", ErrInvalidFileMode, *f, mode, modeErr) } else { return os.FileMode(mode), nil } }