From 10e854583feba488a66a9936d2e383d85a82c0aa Mon Sep 17 00:00:00 2001 From: Matthew Rich Date: Sat, 28 Sep 2024 05:04:15 +0000 Subject: [PATCH] fixes for http resource use of common --- internal/folio/declaration.go | 9 +- internal/folio/document.go | 1 + internal/resource/command.go | 6 +- internal/resource/common.go | 21 +++- internal/resource/common_test.go | 18 ++- internal/resource/container_image.go | 1 + internal/resource/exec.go | 25 +--- internal/resource/file.go | 14 +++ internal/resource/http.go | 182 +++++++++++---------------- internal/resource/pki.go | 4 +- internal/tempdir/tempdir.go | 8 ++ internal/transport/http.go | 3 +- 12 files changed, 145 insertions(+), 147 deletions(-) diff --git a/internal/folio/declaration.go b/internal/folio/declaration.go index 2882c63..528d021 100644 --- a/internal/folio/declaration.go +++ b/internal/folio/declaration.go @@ -17,6 +17,11 @@ _ "gitea.rosskeen.house/rosskeen.house/machine" "decl/internal/schema" "net/url" "runtime/debug" + "errors" +) + +var ( + ErrUnknownStateTransition error = errors.New("Unknown state transition") ) type ConfigName string @@ -177,6 +182,8 @@ func (d *Declaration) Apply(stateTransition string) (result error) { if doc, ok := DocumentRegistry.DeclarationMap[d]; ok { d.SetDocument(doc) } + case "stat": + result = stater.Trigger("stat") case "read": result = stater.Trigger("read") case "delete", "absent": @@ -189,7 +196,7 @@ func (d *Declaration) Apply(stateTransition string) (result error) { } result = stater.Trigger("read") default: - fallthrough + return fmt.Errorf("%w: %s on %s", ErrUnknownStateTransition, stateTransition, d.Attributes.URI()) case "create", "present": if stater.CurrentState() == "absent" || stater.CurrentState() == "unknown" { if result = stater.Trigger("create"); result != nil { diff --git a/internal/folio/document.go b/internal/folio/document.go index f0ca805..263a351 100644 --- a/internal/folio/document.go +++ b/internal/folio/document.go @@ -277,6 +277,7 @@ func (d *Document) Apply(state string) error { if d.ResourceDeclarations[idx].Requires.Check() { if e := d.ResourceDeclarations[idx].Apply(state); e != nil { d.ResourceDeclarations[idx].Error = e.Error() + slog.Info("Document.Apply() ERROR", "index", idx, "uri", d.ResourceDeclarations[idx].Resource().URI(), "error", e.Error()) switch d.ResourceDeclarations[idx].OnError.GetStrategy() { case OnErrorStop: return e diff --git a/internal/resource/command.go b/internal/resource/command.go index 945cf5a..deea835 100644 --- a/internal/resource/command.go +++ b/internal/resource/command.go @@ -3,14 +3,14 @@ package resource import ( - _ "context" +_ "context" "encoding/json" "fmt" "gopkg.in/yaml.v3" "io" "log/slog" - _ "net/url" - _ "os" +_ "net/url" +_ "os" "os/exec" "strings" "text/template" diff --git a/internal/resource/common.go b/internal/resource/common.go index 6d1e70e..150b8e3 100644 --- a/internal/resource/common.go +++ b/internal/resource/common.go @@ -9,9 +9,13 @@ import ( "path/filepath" "decl/internal/data" "decl/internal/folio" + "log/slog" ) +type UriSchemeValidator func(scheme string) bool + type Common struct { + SchemeCheck UriSchemeValidator `json:"-" yaml:"-"` includeQueryParamsInURI bool `json:"-" yaml:"-"` resourceType TypeName `json:"-" yaml:"-"` Uri folio.URI `json:"uri,omitempty" yaml:"uri,omitempty"` @@ -27,8 +31,14 @@ type Common struct { Resources data.ResourceMapper `json:"-" yaml:"-"` } -func NewCommon() *Common { - return &Common{ includeQueryParamsInURI: false } +func NewCommon(resourceType TypeName, includeQueryParams bool) *Common { + c := &Common{ includeQueryParamsInURI: includeQueryParams, resourceType: resourceType } + c.SchemeCheck = c.IsValidResourceScheme + return c +} + +func (c *Common) IsValidResourceScheme(scheme string) bool { + return c.Type() == scheme } func (c *Common) ContentType() string { @@ -65,7 +75,7 @@ func (c *Common) URIPath() string { } func (c *Common) URI() string { - return fmt.Sprintf("%s://%s", c.Type(), c.Path) + return string(c.Uri) } func (c *Common) SetURI(uri string) (err error) { @@ -81,11 +91,12 @@ func (c *Common) SetURIFromString(uri string) { func (c *Common) SetParsedURI(u *url.URL) (err error) { if u != nil { + slog.Info("Common.SetParsedURI()", "parsed", u, "uri", c.Uri) if c.Uri.IsEmpty() { c.SetURIFromString(u.String()) } c.parsedURI = u - if c.parsedURI.Scheme == c.Type() { + if c.SchemeCheck(c.parsedURI.Scheme) { if c.includeQueryParamsInURI { c.Path = filepath.Join(c.parsedURI.Hostname(), c.parsedURI.RequestURI()) } else { @@ -97,7 +108,7 @@ func (c *Common) SetParsedURI(u *url.URL) (err error) { return } } - err = fmt.Errorf("%w: %s is not a file", ErrInvalidResourceURI, c.Uri) + err = fmt.Errorf("%w: %s is not a %s resource, parsed: %t", ErrInvalidResourceURI, c.Uri, c.Type(), (u != nil)) return } diff --git a/internal/resource/common_test.go b/internal/resource/common_test.go index 022aaac..9fd186d 100644 --- a/internal/resource/common_test.go +++ b/internal/resource/common_test.go @@ -9,19 +9,27 @@ import ( ) func TestNewCommon(t *testing.T) { - c := NewCommon() + c := NewCommon(TypeName("foo"), false) assert.NotNil(t, c) } func TestCommon(t *testing.T) { - for _, v := range []struct{ uri string; expected Common }{ - { uri: "file:///tmp/foo", expected: Common{ resourceType: "file", Uri: "file:///tmp/foo", parsedURI: &url.URL{ Scheme: "file", Path: "/tmp/foo"}, Path: "/tmp/foo" } }, + expectedCommon := NewCommon(FileTypeName, false) + expectedCommon.resourceType = "file" + expectedCommon.Uri = "file:///tmp/foo" + expectedCommon.parsedURI = &url.URL{ Scheme: "file", Path: "/tmp/foo"} + expectedCommon.Path = "/tmp/foo" + for _, v := range []struct{ uri string; expected *Common }{ + { uri: "file:///tmp/foo", expected: expectedCommon }, }{ - c := NewCommon() + + c := NewCommon(FileTypeName, false) c.resourceType = "file" assert.Nil(t, c.SetURI(v.uri)) + assert.Equal(t, v.expected.resourceType , c.resourceType) assert.Equal(t, v.expected.Path, c.Path) - assert.Equal(t, &v.expected, c) + assert.Equal(t, v.expected.parsedURI.Scheme, c.parsedURI.Scheme) + assert.Equal(t, v.expected.parsedURI.Path, c.parsedURI.Path) } } diff --git a/internal/resource/container_image.go b/internal/resource/container_image.go index 4faab9b..ff943a3 100644 --- a/internal/resource/container_image.go +++ b/internal/resource/container_image.go @@ -587,6 +587,7 @@ func (c *ContainerImage) Create(ctx context.Context) (err error) { buildOptions := types.ImageBuildOptions{ Dockerfile: dockerfileURI.Path, Tags: []string{c.Name}, + NetworkMode: "host", } var reader io.ReadCloser diff --git a/internal/resource/exec.go b/internal/resource/exec.go index bb2142b..1b3eb05 100644 --- a/internal/resource/exec.go +++ b/internal/resource/exec.go @@ -46,7 +46,8 @@ func init() { } func NewExec() *Exec { - return &Exec{ Common: &Common{ includeQueryParamsInURI: true, resourceType: ExecTypeName } } + e := &Exec{ Common: NewCommon(ExecTypeName, true) } + return e } func (x *Exec) SetResourceMapper(resources data.ResourceMapper) { @@ -87,28 +88,6 @@ func (x *Exec) SetParsedURI(uri *url.URL) (err error) { return } -/* -func (x *Exec) SetURI(uri string) error { - resourceUri, e := url.Parse(uri) - if e == nil { - if resourceUri.Scheme == "exec" { - x.Id = filepath.Join(resourceUri.Hostname(), resourceUri.RequestURI()) - } else { - e = fmt.Errorf("%w: %s is not an exec resource ", ErrInvalidResourceURI, uri) - } - } - return e -} - -func (x *Exec) UseConfig(config data.ConfigurationValueGetter) { - x.config = config -} - -func (x *Exec) ResolveId(ctx context.Context) string { - return "" -} -*/ - func (x *Exec) Validate() (err error) { var execJson []byte if execJson, err = x.JSON(); err == nil { diff --git a/internal/resource/file.go b/internal/resource/file.go index c94f367..44a16bf 100644 --- a/internal/resource/file.go +++ b/internal/resource/file.go @@ -29,6 +29,10 @@ import ( "embed" ) +const ( + FileTypeName TypeName = "file" +) + // Describes the type of file the resource represents type FileType string @@ -176,6 +180,16 @@ func (f *File) Notify(m *machine.EventMessage) { switch m.On { case machine.ENTERSTATEEVENT: switch m.Dest { + case "start_stat": + if statErr := f.ReadStat(); statErr == nil { + if triggerErr := f.StateMachine().Trigger("exists"); triggerErr == nil { + return + } + } else { + if triggerErr := f.StateMachine().Trigger("notexists"); triggerErr == nil { + return + } + } case "start_read": if _,readErr := f.Read(ctx); readErr == nil { if triggerErr := f.StateMachine().Trigger("state_read"); triggerErr == nil { diff --git a/internal/resource/http.go b/internal/resource/http.go index b9a332e..16e4513 100644 --- a/internal/resource/http.go +++ b/internal/resource/http.go @@ -12,7 +12,7 @@ import ( "net/http" _ "os" "encoding/json" - "strings" + "strings" "log/slog" "gitea.rosskeen.house/rosskeen.house/machine" "decl/internal/codec" @@ -39,12 +39,13 @@ func init() { func HTTPFactory(u *url.URL) data.Resource { var err error h := NewHTTP() - (&h.Endpoint).SetURL(u) - h.parsedURI = u - if h.reader, err = transport.NewReader(u); err != nil { + + slog.Info("HTTP.Factory", "http", h, "url", u) + if err = h.SetParsedURI(u); err != nil { panic(err) } - if h.writer, err = transport.NewWriter(u); err != nil { + + if err = h.Open(); err != nil { panic(err) } return h @@ -58,7 +59,6 @@ type HTTPHeader struct { // Manage the state of an HTTP endpoint type HTTP struct { *Common `yaml:",inline" json:",inline"` - parsedURI *url.URL `yaml:"-" json:"-"` stater machine.Stater `yaml:"-" json:"-"` client *http.Client `yaml:"-" json:"-"` Endpoint folio.URI `yaml:"endpoint" json:"endpoint"` @@ -79,16 +79,37 @@ type HTTP struct { } func NewHTTP() *HTTP { - return &HTTP{ client: &http.Client{} } + return &HTTP{ client: &http.Client{}, Common: &Common{ includeQueryParamsInURI: true, resourceType: HTTPTypeName, SchemeCheck: func(scheme string) bool { + switch scheme { + case "http", "https": + return true + } + return false + } } } } func (h *HTTP) SetResourceMapper(resources data.ResourceMapper) { h.Resources = resources } +func (h *HTTP) Open() (err error) { + u := h.Common.parsedURI + if u != nil { + if h.reader, err = transport.NewReader(u); err != nil { + return + } + if h.writer, err = transport.NewWriter(u); err != nil { + return + } + } else { + err = fmt.Errorf("HTTP parsed URI is not set: %s", h.Endpoint) + } + return +} + func (h *HTTP) Clone() data.Resource { return &HTTP { - Common: &Common{ includeQueryParamsInURI: true, resourceType: HTTPTypeName }, + Common: h.Common.Clone(), client: h.client, Endpoint: h.Endpoint, Headers: h.Headers, @@ -106,11 +127,21 @@ func (h *HTTP) StateMachine() machine.Stater { } func (h *HTTP) Notify(m *machine.EventMessage) { - ctx := context.Background() + ctx := context.Background() slog.Info("Notify()", "http", h, "m", m) - switch m.On { - case machine.ENTERSTATEEVENT: - switch m.Dest { + switch m.On { + case machine.ENTERSTATEEVENT: + switch m.Dest { + case "start_stat": + if statErr := h.ReadStat(); statErr == nil { + if triggerErr := h.StateMachine().Trigger("exists"); triggerErr == nil { + return + } + } else { + if triggerErr := h.StateMachine().Trigger("notexists"); triggerErr == nil { + return + } + } case "start_read": if _,readErr := h.Read(ctx); readErr == nil { if triggerErr := h.StateMachine().Trigger("state_read"); triggerErr == nil { @@ -123,12 +154,12 @@ func (h *HTTP) Notify(m *machine.EventMessage) { h.Common.State = "absent" panic(readErr) } - case "start_create": - if e := h.Create(ctx); e == nil { - if triggerErr := h.stater.Trigger("created"); triggerErr == nil { + case "start_create": + if e := h.Create(ctx); e == nil { + if triggerErr := h.stater.Trigger("created"); triggerErr == nil { return } - } + } h.Common.State = "absent" case "start_delete": if deleteErr := h.Delete(ctx); deleteErr == nil { @@ -144,36 +175,34 @@ func (h *HTTP) Notify(m *machine.EventMessage) { } case "absent": h.Common.State = "absent" - case "present", "created", "read": - h.Common.State = "present" - } - case machine.EXITSTATEEVENT: - } + case "present", "created", "read": + h.Common.State = "present" + } + case machine.EXITSTATEEVENT: + } } func (h *HTTP) URI() string { - return string(h.Endpoint) -} - -func (h *HTTP) setParsedURI(uri folio.URI) error { - if parsed := uri.Parse(); parsed == nil { - return folio.ErrInvalidURI - } else { - h.parsedURI = parsed - } - return nil + return h.Endpoint.String() } func (h *HTTP) SetURI(uri string) (err error) { - v := folio.URI(uri) - if err = h.setParsedURI(v); err == nil { - h.Endpoint = v + if err = h.Common.SetURI(uri); err == nil { + h.Endpoint = h.Common.Uri } return } -func (h *HTTP) UseConfig(config data.ConfigurationValueGetter) { - h.config = config +func (h *HTTP) SetParsedURI(u *url.URL) (err error) { + if err = h.Common.SetParsedURI(u); err == nil { + h.Endpoint = h.Common.Uri + } + return +} + +func (h *HTTP) ReadStat() (err error) { + err = h.Open() + return } func (h *HTTP) JSON() ([]byte, error) { @@ -203,21 +232,23 @@ func (h *HTTP) Apply() error { func (h *HTTP) Load(docData []byte, f codec.Format) (err error) { if err = f.StringDecoder(string(docData)).Decode(h); err == nil { - err = h.setParsedURI(h.Endpoint) + err = h.Common.SetURI(string(h.Endpoint)) } return } func (h *HTTP) LoadReader(r io.ReadCloser, f codec.Format) (err error) { if err = f.Decoder(r).Decode(h); err == nil { - err = h.setParsedURI(h.Endpoint) + err = h.Common.SetURI(string(h.Endpoint)) + //err = h.setParsedURI(h.Endpoint) } return } func (h *HTTP) LoadString(docData string, f codec.Format) (err error) { if err = f.StringDecoder(docData).Decode(h); err == nil { - err = h.setParsedURI(h.Endpoint) + err = h.Common.SetURI(string(h.Endpoint)) + //err = h.setParsedURI(h.Endpoint) } return } @@ -227,6 +258,8 @@ func (h *HTTP) LoadDecl(yamlResourceDeclaration string) error { } func (h *HTTP) ResolveId(ctx context.Context) string { + _ = h.Common.SetURI(h.Endpoint.String()) + slog.Info("HTTP.ResolveId()", "uri", h.Endpoint.String()) return h.Endpoint.String() } @@ -251,7 +284,7 @@ func (h *HTTP) Create(ctx context.Context) (err error) { slog.Error("HTTP.Create()", "http", h) var contentReader io.ReadCloser - h.writer, err = transport.NewWriterWithContext(h.parsedURI, ctx) + h.writer, err = transport.NewWriterWithContext(h.Common.parsedURI, ctx) if err != nil { slog.Error("HTTP.Create()", "http", h, "error", err) //panic(err) @@ -297,30 +330,6 @@ func (h *HTTP) Create(ctx context.Context) (err error) { h.Status = h.writer.Status() h.StatusCode = h.writer.StatusCode() return -/* - body := strings.NewReader(h.Content) - req, reqErr := http.NewRequest("POST", h.Endpoint, body) - if reqErr != nil { - return reqErr - } - - if tokenErr := h.ReadAuthorizationTokenFromConfig(req); tokenErr != nil { - slog.Error("ReadAuthorizationTokenFromConfig()", "error", tokenErr) - } - - for _,header := range h.Headers { - req.Header.Add(header.Name, header.Value) - } - - resp, err := h.client.Do(req) - h.Status = resp.Status - h.StatusCode = resp.StatusCode - if err != nil { - return err - } - defer resp.Body.Close() - return err -*/ } func (h *HTTP) Update(ctx context.Context) error { @@ -447,7 +456,7 @@ func (h *HTTP) readThru(ctx context.Context) (contentReader io.ReadCloser, err e contentReader, err = h.contentSourceReader() } else { if h.reader == nil { - h.reader, err = transport.NewReaderWithContext(h.parsedURI, ctx) + h.reader, err = transport.NewReaderWithContext(h.Common.parsedURI, ctx) h.reader.SetGzip(false) } contentReader = h.reader @@ -475,14 +484,7 @@ func (h *HTTP) Read(ctx context.Context) (yamlData []byte, err error) { } } - slog.Info("HTTP.Read()", "reader", h.reader) -/* - copyBuffer := make([]byte, 32 * 1024) - _, writeErr := io.CopyBuffer(w, h.reader, copyBuffer) - if writeErr != nil { - return nil, fmt.Errorf("Http.GetContent(): CopyBuffer failed %v %v: %w", w, contentReader, writeErr) - } -*/ + slog.Info("HTTP.Read()", "reader", h.reader, "http", h) if err = h.SetContent(contentReader); err != nil { return } @@ -490,40 +492,6 @@ func (h *HTTP) Read(ctx context.Context) (yamlData []byte, err error) { h.Status = h.reader.Status() h.StatusCode = h.reader.StatusCode() return yaml.Marshal(h) - -/* - req, reqErr := http.NewRequestWithContext(ctx, "GET", h.Endpoint, nil) - if reqErr != nil { - return nil, reqErr - } - slog.Info("HTTP.Read() ", "request", req, "err", reqErr) - - tokenErr := h.ReadAuthorizationTokenFromConfig(req) - if tokenErr != nil { - slog.Error("ReadAuthorizationTokenFromConfig()", "error", tokenErr) - } - - if len(h.Headers) > 0 { - for _,header := range h.Headers { - req.Header.Add(header.Name, header.Value) - } - } - - resp, err := h.client.Do(req) - slog.Info("Http.Read()", "response", resp, "error", err) - h.Status = resp.Status - h.StatusCode = resp.StatusCode - if err != nil { - return nil, err - } - defer resp.Body.Close() - body, errReadBody := io.ReadAll(resp.Body) - if errReadBody != nil { - return nil, errReadBody - } - h.Body = string(body) - return yaml.Marshal(h) -*/ } func (h *HTTP) Delete(ctx context.Context) error { diff --git a/internal/resource/pki.go b/internal/resource/pki.go index 637428d..ea1b259 100644 --- a/internal/resource/pki.go +++ b/internal/resource/pki.go @@ -89,7 +89,7 @@ type PKI struct { } func NewPKI() *PKI { - p := &PKI{ EncodingType: EncodingTypePem, Bits: 2048, Common: &Common{ resourceType: PKITypeName } } + p := &PKI{ EncodingType: EncodingTypePem, Bits: 2048, Common: NewCommon(PKITypeName, false) } return p } @@ -100,7 +100,7 @@ func (k *PKI) SetResourceMapper(resources data.ResourceMapper) { func (k *PKI) Clone() data.Resource { return &PKI { - Common: k.Common, + Common: k.Common.Clone(), EncodingType: k.EncodingType, //State: k.State, } diff --git a/internal/tempdir/tempdir.go b/internal/tempdir/tempdir.go index 72c6615..f79c843 100644 --- a/internal/tempdir/tempdir.go +++ b/internal/tempdir/tempdir.go @@ -22,6 +22,9 @@ func (t *Path) ValidPath() bool { } func (t *Path) Create() (err error) { + if t.ValidPath() && t.Exists() { + return + } slog.Info("tempdir.Create()", "path", string(*t)) var TempDir string TempDir, err = os.MkdirTemp("", string(*t)) @@ -68,6 +71,11 @@ func (t *Path) CreateFile(name string, content string) (err error) { return } +func (t *Path) Exists() (bool) { + _, statErr := os.Stat(string(*t)) + return ! os.IsNotExist(statErr) +} + func (t *Path) FilePath(name string) string { return filepath.Join(string(*t), name) } diff --git a/internal/transport/http.go b/internal/transport/http.go index 2fe34ea..c195b6b 100644 --- a/internal/transport/http.go +++ b/internal/transport/http.go @@ -84,7 +84,7 @@ func (h *HTTPConnection) Reader() io.ReadCloser { } func (h *HTTPConnection) Do() (err error) { - slog.Info("transport.HTTPConnection.Do()", "connection", h) + slog.Info("transport.HTTPConnection.Do()", "connection", h, "request", h.request) h.response, err = h.Client.Do(h.request) return } @@ -205,6 +205,7 @@ func (h *HTTP) Gzip() bool { func (h *HTTP) Reader() io.ReadCloser { if h.get == nil { h.get = NewHTTPConnection(h.Client) + if err := h.get.NewGetRequest(h.ctx, h.uri.String()); err != nil { panic(err) }