fix the handling of compressed files: ensure the gzip writer is closed before the file
Some checks failed
Lint / golangci-lint (push) Failing after 10m55s
Declarative Tests / test (push) Failing after 33s

This commit is contained in:
Matthew Rich 2024-10-03 23:34:27 +00:00
parent a34f83e684
commit d359d8bfab
4 changed files with 53 additions and 10 deletions

View File

@ -13,6 +13,7 @@ _ "errors"
"fmt"
"compress/gzip"
"log/slog"
"decl/internal/ext"
)
type File struct {
@ -98,6 +99,7 @@ func NewFileWriter(u *url.URL) (f *FileWriter, err error) {
}
f.extension()
f.DetectGzip()
slog.Info("transport.NewFileWriter()", "file", f.File)
exists := FileExists(u)
slog.Info("transport.NewFileWriter()", "file", f, "error", err, "exists", exists)
@ -110,7 +112,7 @@ func NewFileWriter(u *url.URL) (f *FileWriter, err error) {
}
}
if f.Gzip() {
f.gzipWriter = gzip.NewWriter(f.writeHandle)
f.gzipWriter = ext.WriteAddCloser(gzip.NewWriter(f.writeHandle), func() error { return f.writeHandle.Close() })
}
slog.Info("transport.NewFileWriter()", "file", f, "error", err)
return
@ -139,7 +141,7 @@ func NewFile(u *url.URL) (f *File, err error) {
}
if f.Gzip() {
f.gzipWriter = gzip.NewWriter(f.writeHandle)
f.gzipWriter = ext.WriteAddCloser(gzip.NewWriter(f.writeHandle), func() error { return f.writeHandle.Close() })
if exists {
if f.gzipReader, err = gzip.NewReader(f.readHandle); err != nil {
return
@ -164,7 +166,7 @@ func (f *File) extension() {
}
func (f *File) DetectGzip() {
f.gzip = (f.uri.Query().Get("gzip") == "true" || f.fileext == "gz")
f.gzip = (f.uri.Query().Get("gzip") == "true" || f.fileext == "gz" || f.exttype == "tgz" || f.exttype == "gz" || f.fileext == "tgz")
}
func (f *File) URI() *url.URL {

View File

@ -7,8 +7,11 @@ import (
"testing"
"fmt"
"os"
"io"
"net/url"
"path/filepath"
"compress/gzip"
"strings"
)
var TransportFileTestFile = fmt.Sprintf("%s/foo", TempDir)
@ -37,3 +40,35 @@ func TestNewTransportFileReaderExtension(t *testing.T) {
f.extension()
assert.Equal(t, f.exttype, "yaml")
}
// Use transport to write a compressed file then verify it.
func TestTransportFileGzipWriter(t *testing.T) {
var content strings.Builder
var testValue string = "write a compressed plaintext file"
path := TempDir.FilePath("foo.tgz")
u, e := url.Parse(fmt.Sprintf("file://%s", path))
assert.Nil(t, e)
fw, err := NewFileWriter(u)
assert.Nil(t, err)
assert.Equal(t, "tgz", fw.File.exttype)
assert.True(t, fw.Gzip())
writer := fw.Writer()
writer.Write([]byte(testValue))
assert.Nil(t, writer.Close())
assert.True(t, TempDir.FileExists("foo.tgz"))
r, openErr := TempDir.Open("foo.tgz")
assert.Nil(t, openErr)
gzipReader, gzipErr := gzip.NewReader(r)
assert.Nil(t, gzipErr)
_, readErr := io.Copy(&content, gzipReader)
assert.Nil(t, readErr)
assert.Equal(t, content.String(), testValue)
}

View File

@ -19,6 +19,7 @@ type Handler interface {
URI() *url.URL
ContentType() string
SetGzip(bool)
DetectGzip()
Gzip() bool
Signature() string
Stat() (fs.FileInfo, error)
@ -138,6 +139,8 @@ func (r *Reader) SetGzip(value bool) {
r.handle.SetGzip(value)
}
func (r *Reader) DetectGzip() { r.handle.DetectGzip() }
func (r *Reader) Gzip() bool {
return r.handle.Gzip()
}
@ -196,6 +199,8 @@ func (w *Writer) SetGzip(value bool) {
w.handle.SetGzip(value)
}
func (w *Writer) DetectGzip() { w.handle.DetectGzip() }
func (w *Writer) Gzip() bool {
return w.handle.Gzip()
}

View File

@ -9,9 +9,10 @@ import (
"fmt"
"os"
"log"
"decl/internal/tempdir"
)
var TempDir string
var TempDir tempdir.Path = "testtransportfile"
var testFileResourceDoc string = `
resources:
@ -23,19 +24,19 @@ resources:
func TestMain(m *testing.M) {
var err error
TempDir, err = os.MkdirTemp("", "testtransportfile")
err = TempDir.Create()
if err != nil || TempDir == "" {
log.Fatal(err)
}
rc := m.Run()
os.RemoveAll(TempDir)
TempDir.Remove()
os.Exit(rc)
}
func TestNewTransportReader(t *testing.T) {
path := fmt.Sprintf("%s/foo", TempDir)
path := TempDir.FilePath("foo")
u, e := url.Parse(fmt.Sprintf("file://%s", path))
assert.Nil(t, e)
@ -48,7 +49,7 @@ func TestNewTransportReader(t *testing.T) {
}
func TestTransportReaderContentType(t *testing.T) {
path := fmt.Sprintf("%s/foo.jx.yaml", TempDir)
path := TempDir.FilePath("foo.jx.yaml")
u, e := url.Parse(fmt.Sprintf("file://%s", path))
assert.Nil(t, e)
@ -66,7 +67,7 @@ func TestTransportReaderContentType(t *testing.T) {
}
func TestTransportReaderDir(t *testing.T) {
u, e := url.Parse(fmt.Sprintf("file://%s", TempDir))
u, e := url.Parse(fmt.Sprintf("file://%s", string(TempDir)))
assert.Nil(t, e)
reader, err := NewReader(u)
@ -76,7 +77,7 @@ func TestTransportReaderDir(t *testing.T) {
}
func TestTransportWriter(t *testing.T) {
path := fmt.Sprintf("%s/writefoo", TempDir)
path := TempDir.FilePath("writefoo")
u, e := url.Parse(fmt.Sprintf("file://%s", path))
assert.Nil(t, e)