From 94f3998fcda6fcc48962604506bb1a9001516fe2 Mon Sep 17 00:00:00 2001 From: Matthew Rich Date: Sun, 10 Nov 2024 10:16:44 -0800 Subject: [PATCH] add support for remote command execution --- internal/command/command.go | 93 ++++++++---------- internal/command/commandarg.go | 31 ++++++ internal/command/commandtargetref.go | 30 ++++++ internal/command/commandtype.go | 44 +++++++++ internal/command/commandtype_test.go | 18 ++++ internal/command/container_test.go | 104 ++++++++++++++++++++ internal/command/containercommand.go | 142 +++++++++++++++++++++++++++ internal/command/execcommand.go | 34 +++++++ internal/command/sshcommand.go | 54 ++++++++++ 9 files changed, 499 insertions(+), 51 deletions(-) create mode 100644 internal/command/commandarg.go create mode 100644 internal/command/commandtargetref.go create mode 100644 internal/command/commandtype.go create mode 100644 internal/command/commandtype_test.go create mode 100644 internal/command/container_test.go create mode 100644 internal/command/containercommand.go create mode 100644 internal/command/execcommand.go create mode 100644 internal/command/sshcommand.go diff --git a/internal/command/command.go b/internal/command/command.go index 6b69f71..f61c879 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -3,15 +3,12 @@ package command import ( - _ "context" - "encoding/json" +_ "context" "fmt" "errors" - "gopkg.in/yaml.v3" "io" "log/slog" - _ "net/url" - "os" +_ "net/url" "os/exec" "strings" "text/template" @@ -19,31 +16,43 @@ import ( "syscall" ) +// A resource that implements the ExecProvider interface can be used as an exec target. +type CommandProvider interface { + Start() error + Wait() error + SetCmdEnv([]string) + SetStdin(io.Reader) + StdinPipe() (io.WriteCloser, error) + StdoutPipe() (io.ReadCloser, error) + StderrPipe() (io.ReadCloser, error) +} + var ErrUnknownCommand error = errors.New("Unable to find command in path") type CommandExecutor func(value any) ([]byte, error) type CommandExtractAttributes func(output []byte, target any) error type CommandExists func() error -type CommandArg string - type CommandInput string type Command struct { - Path string `json:"path" yaml:"path"` - Args []CommandArg `json:"args" yaml:"args"` - Env []string `json:"env" yaml:"env"` - Split bool `json:"split" yaml:"split"` - FailOnError bool `json:"failonerror" yaml:"failonerror"` + Path string `json:"path" yaml:"path"` + Args []CommandArg `json:"args" yaml:"args"` + Env []string `json:"env" yaml:"env"` + Split bool `json:"split" yaml:"split"` + FailOnError bool `json:"failonerror" yaml:"failonerror"` StdinAvailable bool `json:"stdinavailable,omitempty" yaml:"stdinavailable,omitempty"` - ExitCode int `json:"exitcode,omitempty" yaml:"exitcode,omitempty"` - Stdout string `json:"stdout,omitempty" yaml:"stdout,omitempty"` - Stderr string `json:"stderr,omitempty" yaml:"stderr,omitempty"` - Executor CommandExecutor `json:"-" yaml:"-"` - Extractor CommandExtractAttributes `json:"-" yaml:"-"` + ExitCode int `json:"exitcode,omitempty" yaml:"exitcode,omitempty"` + Stdout string `json:"stdout,omitempty" yaml:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty" yaml:"stderr,omitempty"` + Executor CommandExecutor `json:"-" yaml:"-"` + Extractor CommandExtractAttributes `json:"-" yaml:"-"` CommandExists CommandExists `json:"-" yaml:"-"` - Input CommandInput `json:"-" yaml:"-"` - stdin io.Reader `json:"-" yaml:"-"` + Input CommandInput `json:"-" yaml:"-"` + stdin io.Reader `json:"-" yaml:"-"` + + TargetRef CommandTargetRef `json:"targetref,omitempty" yaml:"targetref,omitempty"` + execHandle CommandProvider `json:"-" yaml:"-"` } func NewCommand() *Command { @@ -70,21 +79,20 @@ func (c *Command) Defaults() { } c.Executor = func(value any) ([]byte, error) { c.ClearOutput() - args, err := c.Template(value) - if err != nil { - return nil, err - } + + c.execHandle = c.TargetRef.Provider(c, value) + if inputErr := c.SetInput(value); inputErr != nil { return nil, inputErr } - cmd := exec.Command(c.Path, args...) - c.SetCmdEnv(cmd) - + + c.SetCmdEnv() + + cmd := c.execHandle if c.stdin != nil { - cmd.Stdin = c.stdin + cmd.SetStdin(c.stdin) } - slog.Info("execute() - cmd", "path", c.Path, "args", args) output, stdoutPipeErr := cmd.StdoutPipe() if stdoutPipeErr != nil { return nil, stdoutPipeErr @@ -113,11 +121,13 @@ func (c *Command) Defaults() { c.Stderr = string(stdErrOutput) c.ExitCode = c.GetExitCodeFromError(waitErr) +/* if len(stdOutOutput) > 100 { slog.Info("execute()", "path", c.Path, "args", args, "output", string(stdOutOutput[:100]), "error", string(stdErrOutput)) } else { slog.Info("execute()", "path", c.Path, "args", args, "output", string(stdOutOutput), "error", string(stdErrOutput)) } +*/ if len(stdErrOutput) > 0 && c.FailOnError { return stdOutOutput, fmt.Errorf("%w %s", waitErr, string(stdErrOutput)) @@ -134,8 +144,8 @@ func (c *Command) LoadDecl(yamlResourceDeclaration string) error { return codec.NewYAMLStringDecoder(yamlResourceDeclaration).Decode(c) } -func (c *Command) SetCmdEnv(cmd *exec.Cmd) { - cmd.Env = append(os.Environ(), c.Env...) +func (c *Command) SetCmdEnv() { + c.execHandle.SetCmdEnv(c.Env) } func (c *Command) SetStdinReader(r io.Reader) { @@ -188,8 +198,10 @@ func (c *Command) Execute(value any) ([]byte, error) { func (c *Command) SetInput(value any) error { if len(c.Input) > 0 { if r, err := c.Input.Template(value); err != nil { + slog.Info("Command.SetInput", "input", r.String(), "error", err) return err } else { + slog.Info("Command.SetInput", "input", r.String()) c.SetStdinReader(strings.NewReader(r.String())) } } @@ -200,24 +212,3 @@ func (c *CommandInput) Template(value any) (result strings.Builder, err error) { err = template.Must(template.New("commandInput").Parse(string(*c))).Execute(&result, value) return } - -func (c *CommandArg) UnmarshalValue(value string) error { - *c = CommandArg(value) - return nil -} - -func (c *CommandArg) UnmarshalJSON(data []byte) error { - var s string - if unmarshalRouteTypeErr := json.Unmarshal(data, &s); unmarshalRouteTypeErr != nil { - return unmarshalRouteTypeErr - } - return c.UnmarshalValue(s) -} - -func (c *CommandArg) UnmarshalYAML(value *yaml.Node) error { - var s string - if err := value.Decode(&s); err != nil { - return err - } - return c.UnmarshalValue(s) -} diff --git a/internal/command/commandarg.go b/internal/command/commandarg.go new file mode 100644 index 0000000..fc49158 --- /dev/null +++ b/internal/command/commandarg.go @@ -0,0 +1,31 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package command + +import ( + "encoding/json" + "gopkg.in/yaml.v3" +) + +type CommandArg string + +func (c *CommandArg) UnmarshalValue(value string) error { + *c = CommandArg(value) + return nil +} + +func (c *CommandArg) UnmarshalJSON(data []byte) error { + var s string + if unmarshalRouteTypeErr := json.Unmarshal(data, &s); unmarshalRouteTypeErr != nil { + return unmarshalRouteTypeErr + } + return c.UnmarshalValue(s) +} + +func (c *CommandArg) UnmarshalYAML(value *yaml.Node) error { + var s string + if err := value.Decode(&s); err != nil { + return err + } + return c.UnmarshalValue(s) +} diff --git a/internal/command/commandtargetref.go b/internal/command/commandtargetref.go new file mode 100644 index 0000000..dce6c3c --- /dev/null +++ b/internal/command/commandtargetref.go @@ -0,0 +1,30 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package command + +import ( + "path/filepath" + "decl/internal/identifier" +) + +type CommandTargetRef identifier.ID + +func (r CommandTargetRef) GetType() (CommandType) { + uri := identifier.ID(r).Parse() + if uri != nil { + var ct CommandType = CommandType(uri.Scheme) + if ct.Validate() == nil { + return ct + } + } + return "" +} + +func (r CommandTargetRef) Provider(cmd *Command, value any) CommandProvider { + return r.GetType().Provider(cmd, value) +} + +func (r CommandTargetRef) Name() string { + u := identifier.ID(r).Parse() + return filepath.Join(u.Hostname(), u.Path) +} diff --git a/internal/command/commandtype.go b/internal/command/commandtype.go new file mode 100644 index 0000000..1b6b060 --- /dev/null +++ b/internal/command/commandtype.go @@ -0,0 +1,44 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package command + +import ( + "errors" + "fmt" +) + +var ( + ErrInvalidCommandType error = errors.New("Invalid command type") +) + +type CommandType string + +const ( + ExecCommand CommandType = "exec" + ContainerCommand CommandType = "container" + SSHCommand CommandType = "ssh" +) + + +func (t CommandType) Validate() error { + switch t { + case ExecCommand, ContainerCommand, SSHCommand: + return nil + } + return fmt.Errorf("%w: %s", ErrInvalidCommandType, t) +} + +func (t CommandType) Provider(cmd *Command, value any) CommandProvider { + switch t { + case ContainerCommand: + return NewContainerProvider(cmd, value) + case SSHCommand: + return NewSSHProvider(cmd, value) + default: + fallthrough + case ExecCommand: + return NewExecProvider(cmd, value) + } + return nil +} + diff --git a/internal/command/commandtype_test.go b/internal/command/commandtype_test.go new file mode 100644 index 0000000..b0160fc --- /dev/null +++ b/internal/command/commandtype_test.go @@ -0,0 +1,18 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + + +package command + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCommandType(t *testing.T) { + + var cmdType CommandType = "container" + + assert.Nil(t, cmdType.Validate()) +} + + diff --git a/internal/command/container_test.go b/internal/command/container_test.go new file mode 100644 index 0000000..3a73554 --- /dev/null +++ b/internal/command/container_test.go @@ -0,0 +1,104 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + + +package command + +import ( +_ "fmt" + "github.com/stretchr/testify/assert" +_ "os" +_ "strings" + "testing" + "bytes" +) + +/* +func TestNewContainerProvider(t *testing.T) { + c := NewContainerProvider(nil, nil) + assert.NotNil(t, c) +} +*/ + +func TestContainerCommandLoad(t *testing.T) { + c := NewCommand() + assert.NotNil(t, c) + + decl := ` +path: find +args: +- "{{ .Path }}" +` + + assert.Nil(t, c.LoadDecl(decl)) + assert.Equal(t, "find", c.Path) +} + +func TestContainerCommandTemplate(t *testing.T) { + c := NewCommand() + assert.NotNil(t, c) + + decl := ` +path: find +args: +- "{{ .Path }}" +` + + assert.Nil(t, c.LoadDecl(decl)) + assert.Equal(t, "find", c.Path) + assert.Equal(t, 1, len(c.Args)) + + f := struct { Path string } { + Path: "./", + } + + args, templateErr := c.Template(f) + assert.Nil(t, templateErr) + assert.Equal(t, 1, len(args)) + + assert.Equal(t, "./", string(args[0])) + + out, err := c.Execute(f) + assert.Nil(t, err) + assert.Greater(t, len(out), 0) +} + +func TestContainerCommandStdin(t *testing.T) { + var expected string = "stdin test data" + var stdinBuffer bytes.Buffer + stdinBuffer.WriteString(expected) + + c := NewCommand() + assert.NotNil(t, c) + + decl := ` +path: cat +stdinavailable: true +` + + assert.Nil(t, c.LoadDecl(decl)) + assert.Equal(t, "cat", c.Path) + + c.SetStdinReader(&stdinBuffer) + out, err := c.Execute(nil) + assert.Nil(t, err) + assert.Equal(t, expected, string(out)) +} + +func TestContainerCommandExitCode(t *testing.T) { + c := NewCommand() + assert.NotNil(t, c) + decl := ` +path: ls +args: +- "amissingfile" +` + + assert.Nil(t, c.LoadDecl(decl)) + assert.Equal(t, "ls", c.Path) + + out, err := c.Execute(nil) + assert.NotNil(t, err) + assert.Greater(t, c.ExitCode, 0) + assert.Equal(t, string(out), c.Stdout) + assert.Equal(t, string("ls: amissingfile: No such file or directory\n"), c.Stderr) +} diff --git a/internal/command/containercommand.go b/internal/command/containercommand.go new file mode 100644 index 0000000..165873d --- /dev/null +++ b/internal/command/containercommand.go @@ -0,0 +1,142 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package command + +import ( + "context" + "fmt" + "io" + "os" + "decl/internal/containerlog" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/strslice" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "time" + "log/slog" +) + +type ContainerExecClient interface { + ContainerExecAttach(ctx context.Context, execID string, config container.ExecAttachOptions) (types.HijackedResponse, error) + ContainerExecCreate(ctx context.Context, containerID string, options container.ExecOptions) (types.IDResponse, error) + ContainerExecInspect(ctx context.Context, execID string) (container.ExecInspect, error) + ContainerExecStart(ctx context.Context, execID string, config container.ExecStartOptions) (error) + ContainerList(context.Context, container.ListOptions) ([]types.Container, error) + Close() error +} + +type ContainerCommandProvider struct { + containerID string + execID string + ExitCode int + container.ExecOptions + response types.HijackedResponse + pipes *containerlog.StreamReader + Stdin io.Reader + apiClient ContainerExecClient `json:"-" yaml:"-"` +} + +// ref could be a resource, but it just needs to implement the ExecResource interface +func NewContainerProvider(cmd *Command, value any) (p *ContainerCommandProvider) { + if args, err := cmd.Template(value); err == nil { + p = &ContainerCommandProvider { + ExecOptions: container.ExecOptions { + Cmd: strslice.StrSlice(append([]string{cmd.Path}, args...)), + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + }, + } + p.containerID = p.ResolveId(context.Background(), cmd.TargetRef) + slog.Info("command.NewContainerProvider", "command", cmd.Path, "args", args, "target", cmd.TargetRef, "container", p.containerID) + } + return +} + +func (c *ContainerCommandProvider) ResolveId(ctx context.Context, ref CommandTargetRef) (containerID string) { + name := ref.Name() + filterArgs := filters.NewArgs() + filterArgs.Add("name", "/"+name) + containers, listErr := c.apiClient.ContainerList(ctx, container.ListOptions{ + All: true, + Filters: filterArgs, + }) + + if listErr != nil { + panic(listErr) + } + + for _, container := range containers { + for _, containerName := range container.Names { + if containerName == "/"+name { + containerID = container.ID + } + } + } + return +} + +func (c *ContainerCommandProvider) Start() (err error) { + var execIDResponse types.IDResponse + ctx := context.Background() + if execIDResponse, err = c.apiClient.ContainerExecCreate(ctx, c.containerID, c.ExecOptions); err == nil { + c.execID = execIDResponse.ID + if c.execID == "" { + return fmt.Errorf("Failed creating a container exec ID") + } + + execStartCheck := types.ExecStartCheck{ + Tty: false, + } + + if c.response, err = c.apiClient.ContainerExecAttach(ctx, c.execID, execStartCheck); err == nil { + c.pipes = containerlog.NewStreamReader(c.response.Conn) + } + } + return +} + +func (c *ContainerCommandProvider) Wait() (err error) { + var containerDetails container.ExecInspect + ctx := context.Background() + for { + // copy Stdin to the connection + if c.Stdin != nil { + io.Copy(c.response.Conn, c.Stdin) + } + + if containerDetails, err = c.apiClient.ContainerExecInspect(ctx, c.execID); err != nil || ! containerDetails.Running { + c.ExitCode = containerDetails.ExitCode + break + } else { + time.Sleep(500 * time.Millisecond) + } + } + return +} + +func (c *ContainerCommandProvider) SetCmdEnv(env []string) { + c.Env = append(os.Environ(), env...) +} + +func (c *ContainerCommandProvider) SetStdin(r io.Reader) { + c.Stdin = r +} + +func (c *ContainerCommandProvider) StdinPipe() (io.WriteCloser, error) { + return c.response.Conn, nil +} + +func (c *ContainerCommandProvider) StdoutPipe() (io.ReadCloser, error) { + return c.pipes.StdoutPipe(), nil +} + +func (c *ContainerCommandProvider) StderrPipe() (io.ReadCloser, error) { + return c.pipes.StderrPipe(), nil +} + +func (c *ContainerCommandProvider) Close() (err error) { + err = c.response.CloseWrite() + c.response.Close() + return +} diff --git a/internal/command/execcommand.go b/internal/command/execcommand.go new file mode 100644 index 0000000..41e01c6 --- /dev/null +++ b/internal/command/execcommand.go @@ -0,0 +1,34 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package command + +import ( + "os" + "os/exec" + "io" + "log/slog" +) + +var ( +) + +type ExecCommandProvider struct { + *exec.Cmd +} + +// Consturct a new exec +func NewExecProvider(c *Command, value any) *ExecCommandProvider { + if args, err := c.Template(value); err == nil { + slog.Info("command.NewExecProvider", "command", c.Path, "args", args, "target", c.TargetRef) + return &ExecCommandProvider{exec.Command(c.Path, args...)} + } + return nil +} + +func (e *ExecCommandProvider) SetCmdEnv(env []string) { + e.Cmd.Env = append(os.Environ(), env...) +} + +func (e *ExecCommandProvider) SetStdin(r io.Reader) { + e.Cmd.Stdin = r +} diff --git a/internal/command/sshcommand.go b/internal/command/sshcommand.go new file mode 100644 index 0000000..96cc8b7 --- /dev/null +++ b/internal/command/sshcommand.go @@ -0,0 +1,54 @@ +// Copyright 2024 Matthew Rich . All rights reserved. + +package command + +import ( + "io" + "os" + "log/slog" +) + +var ( +) + +type SSHCommandProvider struct { + Env []string + Stdin io.Reader +} + +// Consturct a new ssh exec +func NewSSHProvider(c *Command, value any) (p *SSHCommandProvider) { + if args, err := c.Template(value); err == nil { + slog.Info("command.NewSSHProvider", "command", c.Path, "args", args, "target", c.TargetRef) + } + return nil +} + +func (s *SSHCommandProvider) Start() error { + return nil +} + +func (s *SSHCommandProvider) Wait() error { + return nil +} + +func (s *SSHCommandProvider) StdinPipe() (w io.WriteCloser, err error) { + return +} + +func (s *SSHCommandProvider) StdoutPipe() (r io.ReadCloser, err error) { + return +} + +func (s *SSHCommandProvider) StderrPipe() (r io.ReadCloser, err error) { + return +} + +func (s *SSHCommandProvider) SetCmdEnv(env []string) { + s.Env = append(os.Environ(), env...) +} + +func (s *SSHCommandProvider) SetStdin(r io.Reader) { + s.Stdin = r +} +