jx/internal/resource/file.go

// Copyright 2024 Matthew Rich <matthewrich.conf@gmail.com>. All rights reserved.

package resource

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"gopkg.in/yaml.v3"
	"encoding/json"
	"io"
	"io/fs"
	"net/url"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"syscall"
	"time"
	"crypto/sha256"
	"gitea.rosskeen.house/rosskeen.house/machine"
	"decl/internal/codec"
	"decl/internal/iofilter"
	"decl/internal/data"
	"decl/internal/folio"
	"decl/internal/transport"
	"strings"
	"embed"
	"compress/gzip"
)

const (
	FileTypeName TypeName = "file"
)

// Describes the type of file the resource represents
type FileType string

// Supported file types
const (
	RegularFile         FileType = "regular"
	DirectoryFile       FileType = "directory"
	BlockDeviceFile     FileType = "block"
	CharacterDeviceFile FileType = "char"
	NamedPipeFile       FileType = "pipe"
	SymbolicLinkFile    FileType = "symlink"
	SocketFile          FileType = "socket"
)

var ErrInvalidFileInfo error = errors.New("Invalid FileInfo")
var ErrInvalidFileMode error = errors.New("Invalid Mode")
var ErrInvalidFileOwner error = errors.New("Unknown User")
var ErrInvalidFileGroup error = errors.New("Unknown Group")

type FileMode string

func init() {
	folio.DocumentRegistry.ResourceTypes.Register([]string{"file"}, func(u *url.URL) data.Resource {
		f := NewFile()
		f.parsedURI = u
		//f.Uri.SetURL(u)
		f.Path = filepath.Join(u.Hostname(), u.Path)
		f.exttype, f.fileext = f.Uri.Extension()

		slog.Info("folio.DocumentRegistry.ResourceTypes.Register()()", "url", u, "file", f)
/*
		if absPath, err := filepath.Abs(f.Path); err == nil {
			f.Filesystem = os.DirFS(filepath.Dir(absPath))
		}
*/
		return f
	})
}

/*

Manage the state of file system objects
The file content may be serialized directly in the `Content` field
or the `ContentSourceRef/sourceref` may be used to refer to the source
of the content from which to stream the content.
The `SerializeContent` the flag allows forcing the content to be serialized in the output.

*/
type File struct {
	Uri folio.URI `json:"uri,omitempty" yaml:"uri,omitempty"`
	parsedURI *url.URL `json:"-" yaml:"-"`
	Filesystem fs.FS `json:"-" yaml:"-"`

	exttype string `json:"-" yaml:"-"`
	fileext string `json:"-" yaml:"-"`
	stater machine.Stater `json:"-" yaml:"-"`
	normalizePath bool `json:"-" yaml:"-"`
	absPath string `json:"-" yaml:"-"`
	basePath int `json:"-" yaml:"-"`

	Path   string `json:"path" yaml:"path"`
	Owner  string `json:"owner" yaml:"owner"`
	Group  string `json:"group" yaml:"group"`
	Mode   FileMode `json:"mode" yaml:"mode"`

	Atime time.Time `json:"atime,omitempty" yaml:"atime,omitempty"`
	Ctime time.Time `json:"ctime,omitempty" yaml:"ctime,omitempty"`
	Mtime time.Time `json:"mtime,omitempty" yaml:"mtime,omitempty"`

	Content			string		`json:"content,omitempty" yaml:"content,omitempty"`
	ContentSourceRef	folio.ResourceReference   `json:"sourceref,omitempty" yaml:"sourceref,omitempty"`
	Sha256			string		`json:"sha256,omitempty" yaml:"sha256,omitempty"`
	Size			int64		`json:"size,omitempty" yaml:"size,omitempty"`
	Target			string		`json:"target,omitempty" yaml:"target,omitempty"`
	FileType		FileType	`json:"filetype" yaml:"filetype"`
	State			string		`json:"state,omitempty" yaml:"state,omitempty"`
	SerializeContent	bool		`json:"serializecontent,omitempty" yaml:"serializecontent,omitempty"`
	GzipContent		bool		`json:"gzipcontent,omitempty" yaml:"gzipcontent,omitempty"`
	config			data.ConfigurationValueGetter
	Resources		data.ResourceMapper `json:"-" yaml:"-"`
}

type ResourceFileInfo struct {
	resource *File
}

func NewFile() *File {
	currentUser, _ := user.Current()
	group, _ := user.LookupGroupId(currentUser.Gid)
	f := &File{ normalizePath: false, Owner: currentUser.Username, Group: group.Name, Mode: "0644", FileType: RegularFile, SerializeContent: false }
	slog.Info("NewFile()", "file", f)
	return f
}

func NewNormalizedFile() *File {
	f := NewFile()
	f.normalizePath = true
	return f
}

func (f *File) ContentType() string {
	if f.parsedURI.Scheme != "file" {
		return f.parsedURI.Scheme
	}
	return f.exttype
}

func (f *File) SetResourceMapper(resources data.ResourceMapper) {
	f.Resources = resources
}

func (f *File) Clone() data.Resource {
	return &File {
		Uri: f.Uri,
		parsedURI: f.parsedURI,
		exttype: f.exttype,
		fileext: f.fileext,
		normalizePath: f.normalizePath,
		absPath: f.absPath,
		Path: f.Path,
		Owner: f.Owner,
		Group: f.Group,
		Mode: f.Mode,
		Atime: f.Atime,
		Ctime: f.Ctime,
		Mtime: f.Mtime,
		Content: f.Content,
		Sha256: f.Sha256,
		Size: f.Size,
		Target: f.Target,
		FileType: f.FileType,
		State: f.State,
	}
}

func (f *File) StateMachine() machine.Stater {
	if f.stater == nil {
		f.stater = StorageMachine(f)
	}
	return f.stater
}

func (f *File) Notify(m *machine.EventMessage) {
	ctx := context.Background()
	slog.Info("Notify()", "file", f, "m", m)
	switch m.On {
	case machine.ENTERSTATEEVENT:
		switch m.Dest {
			case "start_stat":
				if statErr := f.ReadStat(); statErr == nil {
					if triggerErr := f.StateMachine().Trigger("exists"); triggerErr == nil {
						return
					}
				} else {
					if triggerErr := f.StateMachine().Trigger("notexists"); triggerErr == nil {
						return
					}
				}
			case "start_read":
				if _,readErr := f.Read(ctx); readErr == nil {
					if triggerErr := f.StateMachine().Trigger("state_read"); triggerErr == nil {
						return
					} else {
						f.State = "absent"
						panic(triggerErr)
					}
			} else {
				f.State = "absent"
				if ! errors.Is(readErr, ErrResourceStateAbsent) {
					panic(readErr)
				}
			}
		case "start_create":
			if e := f.Create(ctx); e == nil {
				if triggerErr := f.StateMachine().Trigger("created"); triggerErr == nil {
					return
				}
			} else {
				f.State = "absent"
				panic(e)
			}
		case "start_delete":
			if deleteErr := f.Delete(ctx); deleteErr == nil {
				if triggerErr := f.StateMachine().Trigger("deleted"); triggerErr == nil {
					return
				} else {
					f.State = "present"
					panic(triggerErr)
				}
			} else {
				f.State = "present"
				panic(deleteErr)
			}
		case "absent":
			f.State = "absent"
		case "present", "created", "read":
			f.State = "present"
		}
	case machine.EXITSTATEEVENT:
	}
}

func (f *File) SetGzipContent(flag bool) {
	f.GzipContent = flag
}

func (f *File) PathNormalization(flag bool) {
	f.normalizePath = flag
}

func (f *File) FilePath() string {
	return f.Path
}

func (f *File) SetFS(fsys fs.FS) {
	f.Filesystem = fsys
}

func (f *File) URI() string {
	return fmt.Sprintf("file://%s", f.Path)
}

func (f *File) RelativePath() string {
	return f.Path[f.basePath:]
}

func (f *File) SetBasePath(index int) {
	if index < len(f.Path) {
		f.basePath = index
	}
}

func (f *File) SetURI(uri string) (err error) {
	slog.Info("File.SetURI()", "uri", uri, "file", f, "parsed", f.parsedURI)
	f.SetURIFromString(uri)
	err = f.SetParsedURI(f.Uri.Parse())
	return
}

func (f *File) DetectGzip() bool {
	return (f.parsedURI.Query().Get("gzip") == "true" || f.fileext == "gz" || f.exttype == "tgz" || f.exttype == "gz" || f.fileext == "tgz" ) 
}

func (f *File) SetURIFromString(uri string) {
	f.Uri = folio.URI(uri)
	f.exttype, f.fileext = f.Uri.Extension()
}

func (f *File) SetParsedURI(u *url.URL) (err error) {
	if u != nil {
		if u.Scheme == "" {
			u.Scheme = "file"
			f.Uri = ""
		}
		if f.Uri.IsEmpty()  {
			f.SetURIFromString(u.String())
		}
		slog.Info("File.SetParsedURI()", "parsed", u, "path", f.Path)
		f.parsedURI = u
		if f.parsedURI.Scheme == "file" {
			f.Path = filepath.Join(f.parsedURI.Hostname(), f.parsedURI.Path)
			slog.Info("File.SetParsedURI()", "path", f.Path)
			if err = f.NormalizePath(); err != nil {
				return
			}
			return
		}
	}
	err = fmt.Errorf("%w: %s is not a file", ErrInvalidResourceURI, f.Uri)
	return
}

func (f *File) UseConfig(config data.ConfigurationValueGetter) {
	f.config = config
}

func (f *File) JSON() ([]byte, error) {
	return json.Marshal(f)
}

func (f *File) Validate() (err error) {
	var fileJson []byte
	if fileJson, err = f.JSON(); err == nil {
		s := NewSchema(f.Type())
		err = s.Validate(string(fileJson))
	}
	return err
}

func (f *File) Apply() error {
	ctx := context.Background()
	switch f.State {
	case "absent":
		return f.Delete(ctx)
	case "present":
		return f.Create(ctx)
	}

	return nil
}

func (f *File) Load(docData []byte, format codec.Format) (err error) {
	err = format.StringDecoder(string(docData)).Decode(f)
	if err == nil {
		f.UpdateContentAttributes()
	}
	return
}

func (f *File) LoadReader(r io.ReadCloser, format codec.Format) (err error) {
	err = format.Decoder(r).Decode(f)
	if err == nil {
		f.UpdateContentAttributes()
	}
	return
}

func (f *File) LoadString(docData string, format codec.Format) (err error) {
	err = format.StringDecoder(docData).Decode(f)
	if err == nil {
		f.UpdateContentAttributes()
	}
	return
}

func (f *File) LoadDecl(yamlResourceDeclaration string) (err error) {
	return f.LoadString(yamlResourceDeclaration, codec.FormatYaml)
}

func (f *File) ResolveId(ctx context.Context) string {
	if e := f.NormalizePath(); e != nil {
		panic(e)
	}
	return f.Path
}

func (f *File) NormalizePath() (err error) {
	if f.config != nil {
		if prefixPath, configErr := f.config.GetValue("prefix"); configErr == nil {
			f.Path = filepath.Join(prefixPath.(string), f.Path)
		}
	}
	if f.absPath, err = filepath.Abs(f.Path); err == nil && f.normalizePath {
		f.Path = f.absPath
	}
	return
}

func (f *File) GetContentSourceRef() string {
	return string(f.ContentSourceRef)
}

func (f *File) SetContentSourceRef(uri string) {
	f.Size = 0
	f.ContentSourceRef = folio.ResourceReference(uri)
}

func (f *File) Stat() (info fs.FileInfo, err error) {
	if _, ok := f.Filesystem.(embed.FS); ok {
		info, err = fs.Stat(f.Filesystem, f.Path)
	} else {
		info, err = os.Lstat(f.absPath)
	}
	return
}

func (f *File) FileInfo() fs.FileInfo {
	return &ResourceFileInfo{ resource: f }
}

func (f *ResourceFileInfo) Name() string {
//	return filepath.Base(f.resource.Path)
	return f.resource.RelativePath()
}

func (f *ResourceFileInfo) Size() int64 {
	return f.resource.Size
}

func (f *ResourceFileInfo) Mode() (mode os.FileMode) {
	if fileMode, fileModeErr := f.resource.Mode.GetMode(); fileModeErr == nil {
		mode |= fileMode
	} else {
		panic(fileModeErr)
	}
	mode |= f.resource.FileType.GetMode()
	return
} 

func (f *ResourceFileInfo) ModTime() time.Time {
	return f.resource.Mtime
}

func (f *ResourceFileInfo) IsDir() bool {
	return f.resource.FileType == DirectoryFile
}

func (f *ResourceFileInfo) Sys() any {
	return nil
}

func (f *File) Create(ctx context.Context) error {
	slog.Info("File.Create()", "file", f)
	uid, uidErr := LookupUID(f.Owner)
	if uidErr != nil {
		return fmt.Errorf("%w: unkwnon user %d", ErrInvalidFileOwner, uid)
	}
	gid, gidErr := LookupGID(f.Group)
	if gidErr != nil {
		return fmt.Errorf("%w: unkwnon group %d", ErrInvalidFileGroup, gid)
	}

	mode, modeErr := f.Mode.GetMode()
	if modeErr != nil {
		return modeErr
	}

	switch f.FileType {
	case SymbolicLinkFile:
		linkErr := os.Symlink(f.Target, f.Path)
		if linkErr != nil {
			return linkErr
		}
	case DirectoryFile:
		if mkdirErr := os.MkdirAll(f.Path, mode); mkdirErr != nil {
			return mkdirErr
		}
	default:
		fallthrough
	case RegularFile:
		copyBuffer := make([]byte, 32 * 1024)

		hash := sha256.New()
		f.Size = 0
		var contentReader io.ReadCloser
		if len(f.Content) == 0  && len(f.ContentSourceRef) != 0 {
			if refReader, err := f.ContentSourceRef.Lookup(nil).ContentReaderStream(); err == nil {
				contentReader = refReader
			} else {
				return err
			}
		} else {
			contentReader = io.NopCloser(strings.NewReader(f.Content))
		}

		sumReadData := iofilter.NewReader(contentReader, func(p []byte, readn int, readerr error) (n int, err error) {
			hash.Write(p[:readn])
			f.Size += int64(readn)
			return readn, readerr
		})

		var createdFileWriter io.WriteCloser
		createdFile, fileErr := os.Create(f.Path)
		if fileErr != nil {
			return fileErr
		}

		if f.GzipContent && f.DetectGzip() {
			createdFileWriter = gzip.NewWriter(createdFile)
			defer createdFileWriter.Close()
		} else {
			createdFileWriter = createdFile
		}

		defer createdFile.Close()

		if chmodErr := createdFile.Chmod(mode); chmodErr != nil {
			return chmodErr
		}

		_, writeErr := io.CopyBuffer(createdFileWriter, sumReadData, copyBuffer)
		if writeErr != nil {
			return fmt.Errorf("File.Create(): CopyBuffer failed %v %v: %w", createdFileWriter, contentReader, writeErr)
		}

		f.Sha256 = fmt.Sprintf("%x", hash.Sum(nil))
		if !f.Mtime.IsZero() && !f.Atime.IsZero() {
			if chtimesErr := os.Chtimes(f.Path, f.Atime, f.Mtime); chtimesErr != nil {
				return chtimesErr
			}
		}
	}

	if chownErr := os.Chown(f.Path, uid, gid); chownErr != nil {
		return chownErr
	}
	f.State = "present"
	return nil
}

func (f *File) Update(ctx context.Context) error {
	return f.Create(ctx)
}

func (f *File) Delete(ctx context.Context) error {
	return os.Remove(f.Path)
}

func (f *File) UpdateContentAttributes() {
	f.Size = int64(len(f.Content))
	f.Sha256 = fmt.Sprintf("%x", sha256.Sum256([]byte(f.Content)))
}

func (f *File) SetFileInfo(info os.FileInfo) error {
	return f.UpdateAttributesFromFileInfo(info)
}

func (f *File) UpdateAttributesFromFileInfo(info os.FileInfo) error {
	if info != nil {
		f.Mtime = info.ModTime()
		if stat, ok := info.Sys().(*syscall.Stat_t); ok {
			f.Atime = time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec))
			f.Ctime = time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec))

			userId := strconv.Itoa(int(stat.Uid))
			groupId := strconv.Itoa(int(stat.Gid))
			fileUser, userErr := user.LookupId(userId)
			if userErr != nil { //UnknownUserIdError
				//panic(userErr)
				f.Owner = userId
			} else {
				f.Owner = fileUser.Username
			}
			fileGroup, groupErr := user.LookupGroupId(groupId)
			if groupErr != nil {
				//panic(groupErr)
				f.Group = groupId
			} else {
				f.Group = fileGroup.Name
			}
		}
		f.Size = info.Size()
		f.Mode = FileMode(fmt.Sprintf("%04o", info.Mode().Perm()))
		f.FileType.SetMode(info.Mode())
		return nil
	}
	return ErrInvalidFileInfo
}


func (f *File) ContentSourceRefStat() (info fs.FileInfo) {
	if len(f.ContentSourceRef) > 0 {
		rs, _ := f.ContentReaderStream()
		info, _ = rs.Stat()
		rs.Close()
	}
	return
}

func (f *File) ReadStat() (err error) {
	var info fs.FileInfo
	slog.Info("ReadStat()", "filesystem", f.Filesystem, "path", f.Path)

	info, err = f.Stat()

	slog.Info("ReadStat()", "filesystem", f.Filesystem, "path", f.Path, "info", info, "error", err)

	if err == nil {
		_ = f.SetFileInfo(info)
	} else {
		if refStat := f.ContentSourceRefStat(); refStat != nil {
			_ = f.SetFileInfo(refStat)
			//f.Size = refStat.Size()
			err = nil
		}
		// XXX compare the mod times and set state to outdated
	}


	slog.Info("ReadStat()", "stat", info, "path", f.Path)
	if err != nil {
		f.State = "absent"
		return
	}

	return
}

func (f *File) open() (file io.ReadCloser, err error) {
	slog.Info("open()", "file", f.Path, "fs", f.Filesystem)
	if _, ok := f.Filesystem.(embed.FS); ok {
		file, err = f.Filesystem.Open(f.Path)
	} else {
		file, err = os.Open(f.Path)
	}
	if f.GzipContent && f.DetectGzip() {
		file, err = gzip.NewReader(file)
	}
	slog.Info("open()", "file", f.Path, "error", err)
	return
}

func (f *File) Read(ctx context.Context) ([]byte, error) {
	if normalizePathErr := f.NormalizePath(); normalizePathErr != nil {
		return nil, normalizePathErr
	}
	
	statErr := f.ReadStat()
	if statErr != nil {
		return nil, fmt.Errorf("%w - %w", ErrResourceStateAbsent, statErr)
	}

	switch f.FileType {
	case RegularFile:
		if len(f.ContentSourceRef) == 0 || f.SerializeContent {
			//file, fileErr := os.Open(f.Path)
			file, fileErr := f.open()
			if fileErr != nil {
				panic(fileErr)
			}

			fileContent, ioErr := io.ReadAll(file)
			if ioErr != nil {
				panic(ioErr)
			}
			f.Content = string(fileContent)
			f.Sha256 = fmt.Sprintf("%x", sha256.Sum256(fileContent))
		}
	case SymbolicLinkFile:
		linkTarget, pathErr := os.Readlink(f.Path)
		if pathErr != nil {
			return nil, pathErr
		}
		f.Target = linkTarget
	}
	f.State = "present"
	return yaml.Marshal(f)
}

// set up reader for source content
func (f *File) readThru() (contentReader io.ReadCloser, err error) {
	if len(f.ContentSourceRef) != 0 {
		contentReader, err = f.ContentSourceRef.Lookup(nil).ContentReaderStream()
		if f.GzipContent {
			contentReader.(*transport.Reader).DetectGzip()
		} else {
			contentReader.(*transport.Reader).SetGzip(false)
		}
		slog.Info("File.readThru()", "reader", contentReader)
	} else {
		if len(f.Content) != 0 {
			contentReader = io.NopCloser(strings.NewReader(f.Content))
		} else {
			//contentReader, err = os.Open(f.Path)
			contentReader, err = f.open()
		}
	}
	contentReader = f.UpdateContentAttributesFromReader(contentReader)
	return
}

func (f *File) UpdateContentAttributesFromReader(reader io.ReadCloser) io.ReadCloser {
	var content strings.Builder
	hash := sha256.New()
	f.Size = 0
	f.Content = ""
	f.Sha256 = ""
	return iofilter.NewReader(reader, func(p []byte, readn int, readerr error) (n int, err error) {
		hash.Write(p[:readn])
		f.Sha256 = fmt.Sprintf("%x", hash.Sum(nil))
		f.Size += int64(readn)
		if len(f.ContentSourceRef) == 0 || f.SerializeContent {
			content.Write(p[:readn])
			f.Content = content.String()
		}
		return readn, readerr
	})
}

func (f *File) SetContent(r io.Reader) error {
	fileContent, ioErr := io.ReadAll(r)
	f.Content = string(fileContent)
	f.UpdateContentAttributes()
	return ioErr
}

func (f *File) GetContent(w io.Writer) (contentReader io.ReadCloser, err error) {
	slog.Info("File.GetContent()", "content", len(f.Content), "sourceref", f.ContentSourceRef)

	switch f.FileType {
	case RegularFile:
		contentReader, err = f.readThru()

		if w != nil {
			copyBuffer := make([]byte, 32 * 1024)
			_, writeErr := io.CopyBuffer(w, contentReader, copyBuffer)
			if writeErr != nil {
				return nil, fmt.Errorf("File.GetContent(): CopyBuffer failed %v %v: %w", w, contentReader, writeErr)
			}
			return nil, nil
		}
	}

	return
}

func (f *File) ContentReaderStream() (*transport.Reader, error) {
	if len(f.Content) == 0 && len(f.ContentSourceRef) != 0 {
		return f.ContentSourceRef.Lookup(nil).ContentReaderStream()
	}
	return nil, fmt.Errorf("Cannot provide transport reader for string content")
}

// ContentWriterStream() would not provide a mechanism to keep the in-memory state up-to-date
func (f *File) ContentWriterStream() (*transport.Writer, error) {
	if len(f.Content) == 0 && len(f.ContentSourceRef) != 0 {
		return f.ContentSourceRef.Lookup(nil).ContentWriterStream()
	}
	return nil, fmt.Errorf("Cannot provide transport writer for string content")
}

func (f *File) GetTarget() string { return f.Target }
func (f *File) Type() string { return "file" }

func (f *FileType) UnmarshalYAML(value *yaml.Node) error {
	var s string
	if err := value.Decode(&s); err != nil {
		return err
	}

	switch s {
	case string(RegularFile), string(DirectoryFile), string(BlockDeviceFile), string(CharacterDeviceFile), string(NamedPipeFile), string(SymbolicLinkFile), string(SocketFile):
		*f = FileType(s)
		return nil
	default:
		return errors.New("invalid FileType value")
	}
}

func (f *FileType) SetMode(mode os.FileMode) {
	switch true {
	case mode.IsRegular():
		*f = RegularFile
	case mode.IsDir():
		*f = DirectoryFile
	case mode&os.ModeSymlink != 0:
		*f = SymbolicLinkFile
	case mode&os.ModeNamedPipe != 0:
		*f = NamedPipeFile
	case mode&os.ModeSocket != 0:
		*f = SocketFile
	case mode&os.ModeCharDevice != 0:
		*f = CharacterDeviceFile
	case mode&os.ModeDevice != 0:
		*f = BlockDeviceFile
	}
}

func (f *FileType) GetMode() (mode os.FileMode) {
	switch *f {
	case RegularFile:
	case DirectoryFile:
		mode |= os.ModeDir
	case SymbolicLinkFile:
		mode |= os.ModeSymlink
	case NamedPipeFile:
		mode |= os.ModeNamedPipe
	case SocketFile:
		mode |= os.ModeSocket
	case CharacterDeviceFile:
		mode |= os.ModeCharDevice
	case BlockDeviceFile:
		mode |= os.ModeDevice
	}
	return
}

func (f *FileMode) GetMode() (os.FileMode, error) {
	if mode, modeErr := strconv.ParseInt(string(*f), 8, 64); modeErr != nil {
		return os.FileMode(mode), fmt.Errorf("%w: %s invalid mode %d - %w", ErrInvalidFileMode, *f, mode, modeErr)
	} else {
		return os.FileMode(mode), nil
	}
}