// Copyright 2024 Matthew Rich . All rights reserved. package fan import ( _ "context" _ "encoding/json" "fmt" _ "gopkg.in/yaml.v3" "net/url" _ "net/http" _ "path/filepath" _ "decl/internal/resource" "decl/internal/codec" "decl/internal/data" "decl/internal/folio" _ "os" "io" "errors" "log/slog" ) type HTTP struct { Endpoint folio.URI `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` url *url.URL `yaml:"-" json:"-"` Format codec.Format `yaml:"format,omitempty" json:"format,omitempty"` reader io.ReadCloser `yaml:"-" json:"-"` writer io.WriteCloser `yaml:"-" json:"-"` decoder codec.Decoder `yaml:"-" json:"-"` encoder codec.Encoder `yaml:"-" json:"-"` closer func() error `yaml:"-" json:"-"` index int `yaml:"-" json:"-"` signature data.Signature `yaml:"-" json:"-"` } func NewHTTP() *HTTP { return &HTTP{ Format: codec.FormatYaml, index: 0, closer: func() error { return nil } } } func init() { folio.DocumentRegistry.ConverterTypes.Register([]string{"http","https"}, func(u *url.URL) data.Converter { t := NewHTTP() t.Endpoint = folio.URI(u.String()) t.url = u return t }) } func (h *HTTP) Type() data.TypeName { return "http" } /* func (h *HTTP) setencoder(target data.ContentIdentifier) { if formatErr := h.Format.Set(target.ContentType()); formatErr != nil { h.Format = codec.FormatYaml if format,ok := h.url.Query()["format"]; ok { if queryFormatErr := h.Format.Set(format[0]); queryFormatErr != nil { h.Format = codec.FormatYaml } } } if h.encoder == nil { h.encoder = codec.NewEncoder(h.writer, h.Format) } } */ func (h *HTTP) setdecoder(source data.ContentIdentifier) { if h.decoder == nil { _ = h.Format.Set(source.ContentType()) h.decoder = codec.NewDecoder(h.reader, h.Format) } } func (h *HTTP) Extract(sourceResource data.Resource, filter data.ElementSelector) (document data.Document, err error) { if h.index == 0 { if sourceResource == nil { if len(h.Endpoint) > 0 { sourceResource, err = h.Endpoint.NewResource(nil) } else { return nil, ErrInvalidSource } } slog.Info("HTTP.Extract()", "source", sourceResource, "error", err) var jxSourceFile data.FileResource = sourceResource.(data.FileResource) h.reader, err = jxSourceFile.(data.ContentGetter).GetContent(nil) slog.Info("HTTP.Extract()", "file", h, "error", err) if err != nil { return } h.signature = sourceResource.(data.Signed).Signature() h.setdecoder(jxSourceFile.(data.ContentIdentifier)) slog.Info("HTTP.Extract()", "jx", h) } u := fmt.Sprintf("%s?index=%d", sourceResource.URI(), h.index) document = folio.DocumentRegistry.NewDocument(folio.URI(u)) err = h.decoder.Decode(document) slog.Info("HTTP.Extract()", "doc", document, "http", h, "error", err) h.index++ if err != nil { return } if err = document.Validate(); err != nil { return } if h.signature.String() != "" { if v, ok := sourceResource.(data.ContentHasher); ok { err = h.signature.Verify(v) } } return /* defer h.Close() documentSignature := h.transport.Signature() hash := sha256.New() sumReadData := iofilter.NewReader(h.transport, func(p []byte, readn int, readerr error) (n int, err error) { hash.Write(p) return }) decoder := codec.NewYAMLDecoder(sumReadData) index := 0 for { doc = folio.DocumentRegistry.NewDocument(folio.URI(u)) doc := resource.NewDocument() e := decoder.Decode(doc) if errors.Is(e, io.EOF) { break } if e != nil { return documents, e } if validationErr := doc.Validate(); validationErr != nil { return documents, validationErr } documents = append(documents, doc) index++ } if documentSignature != "" { sig := &signature.Ident{} sigErr := sig.VerifySum(hash.Sum(nil), []byte(documentSignature)) if sigErr != nil { return documents, sigErr } } */ } func (h *HTTP) ExtractMany(resourceSource data.Resource, filter data.ElementSelector) (documents []data.Document, err error) { documents = make([]data.Document, 0, 100) defer h.Close() h.index = 0 for { var doc data.Document if doc, err = h.Extract(resourceSource, filter); err == nil { documents = append(documents, doc) } else { if errors.Is(err, io.EOF) { err = nil //documents = append(documents, doc) } break } } slog.Info("HTTP.ExtractMany()", "file", h, "error", err) return } func (h *HTTP) Emit(document data.Document, filter data.ElementSelector) (resource data.Resource, err error) { return nil, nil } func (h *HTTP) Close() (err error) { /* if h.decoder != nil { h.decoder.Close() } */ if h.encoder != nil { h.encoder.Close() } if h.reader != nil { h.reader.Close() } if h.writer != nil { h.writer.Close() } return }