diff --git a/go.mod b/go.mod index 03231ab..767d1f8 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jkaninda/go-storage v0.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/internal/backup.go b/internal/backup.go index b6a6bb8..494e3da 100644 --- a/internal/backup.go +++ b/internal/backup.go @@ -3,15 +3,14 @@ package internal import ( "fmt" "github.com/jkaninda/encryptor" - "github.com/jkaninda/go-storage/pkg/ftp" - "github.com/jkaninda/go-storage/pkg/local" - "github.com/jkaninda/go-storage/pkg/s3" - "github.com/jkaninda/go-storage/pkg/ssh" "github.com/jkaninda/pg-bkup/pkg/logger" + "github.com/jkaninda/pg-bkup/pkg/storage/ftp" + "github.com/jkaninda/pg-bkup/pkg/storage/local" + "github.com/jkaninda/pg-bkup/pkg/storage/s3" + "github.com/jkaninda/pg-bkup/pkg/storage/ssh" "github.com/jkaninda/pg-bkup/utils" "github.com/robfig/cron/v3" "github.com/spf13/cobra" - "log" "os" "os/exec" diff --git a/internal/restore.go b/internal/restore.go index 64e1c35..d29bf82 100644 --- a/internal/restore.go +++ b/internal/restore.go @@ -9,15 +9,15 @@ package internal import ( "github.com/jkaninda/pg-bkup/pkg/logger" + "github.com/jkaninda/pg-bkup/pkg/storage/ftp" + "github.com/jkaninda/pg-bkup/pkg/storage/local" + "github.com/jkaninda/pg-bkup/pkg/storage/s3" + "github.com/jkaninda/pg-bkup/pkg/storage/ssh" "os" "os/exec" "path/filepath" "github.com/jkaninda/encryptor" - "github.com/jkaninda/go-storage/pkg/ftp" - "github.com/jkaninda/go-storage/pkg/local" - "github.com/jkaninda/go-storage/pkg/s3" - "github.com/jkaninda/go-storage/pkg/ssh" "github.com/jkaninda/pg-bkup/utils" "github.com/spf13/cobra" ) diff --git a/pkg/storage/ftp/ftp.go b/pkg/storage/ftp/ftp.go new file mode 100644 index 0000000..86d76a2 --- /dev/null +++ b/pkg/storage/ftp/ftp.go @@ -0,0 +1,142 @@ +package ftp + +import ( + "fmt" + pkg "github.com/jkaninda/pg-bkup/pkg/storage" + "github.com/jlaffaye/ftp" + "io" + "os" + "path/filepath" + "time" +) + +type ftpStorage struct { + *pkg.Backend + client *ftp.ServerConn +} + +// Config holds the SSH connection details +type Config struct { + Host string + User string + Password string + Port string + LocalPath string + RemotePath string +} + +// createClient creates FTP Client +func createClient(conf Config) (*ftp.ServerConn, error) { + ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second)) + if err != nil { + return nil, fmt.Errorf("failed to connect to FTP: %w", err) + } + + err = ftpClient.Login(conf.User, conf.Password) + if err != nil { + return nil, fmt.Errorf("failed to log in to FTP: %w", err) + } + + return ftpClient, nil +} + +// NewStorage creates new Storage +func NewStorage(conf Config) (pkg.Storage, error) { + client, err := createClient(conf) + if err != nil { + return nil, err + } + return &ftpStorage{ + client: client, + Backend: &pkg.Backend{ + RemotePath: conf.RemotePath, + LocalPath: conf.LocalPath, + }, + }, nil +} + +// Copy copies file to the remote server +func (s ftpStorage) Copy(fileName string) error { + ftpClient := s.client + defer func(ftpClient *ftp.ServerConn) { + err := ftpClient.Quit() + if err != nil { + return + } + }(ftpClient) + + filePath := filepath.Join(s.LocalPath, fileName) + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", fileName, err) + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + return + } + }(file) + + remoteFilePath := filepath.Join(s.RemotePath, fileName) + err = ftpClient.Stor(remoteFilePath, file) + if err != nil { + return fmt.Errorf("failed to upload file %s: %w", filepath.Join(s.LocalPath, fileName), err) + } + + return nil +} + +// CopyFrom copies a file from the remote server to local storage +func (s ftpStorage) CopyFrom(fileName string) error { + ftpClient := s.client + + defer func(ftpClient *ftp.ServerConn) { + err := ftpClient.Quit() + if err != nil { + return + } + }(ftpClient) + + remoteFilePath := filepath.Join(s.RemotePath, fileName) + r, err := ftpClient.Retr(remoteFilePath) + if err != nil { + return fmt.Errorf("failed to retrieve file %s: %w", fileName, err) + } + defer func(r *ftp.Response) { + err := r.Close() + if err != nil { + return + } + }(r) + + localFilePath := filepath.Join(s.LocalPath, fileName) + outFile, err := os.Create(localFilePath) + if err != nil { + return fmt.Errorf("failed to create local file %s: %w", fileName, err) + } + defer func(outFile *os.File) { + err := outFile.Close() + if err != nil { + return + } + }(outFile) + + _, err = io.Copy(outFile, r) + if err != nil { + return fmt.Errorf("failed to copy data to local file %s: %w", fileName, err) + } + + return nil +} + +// Prune deletes old backup created more than specified days +func (s ftpStorage) Prune(retentionDays int) error { + fmt.Println("Deleting old backup from a remote server is not implemented yet") + return nil + +} + +// Name returns the storage name +func (s ftpStorage) Name() string { + return "ftp" +} diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go new file mode 100644 index 0000000..eecb32c --- /dev/null +++ b/pkg/storage/local/local.go @@ -0,0 +1,116 @@ +package local + +import ( + pkg "github.com/jkaninda/pg-bkup/pkg/storage" + "io" + "os" + "path/filepath" + "time" +) + +type localStorage struct { + *pkg.Backend +} +type Config struct { + LocalPath string + RemotePath string +} + +// NewStorage creates new Storage +func NewStorage(conf Config) pkg.Storage { + return &localStorage{ + Backend: &pkg.Backend{ + LocalPath: conf.LocalPath, + RemotePath: conf.RemotePath, + }, + } +} + +// Copy copies file to the local destination path +func (l localStorage) Copy(file string) error { + if _, err := os.Stat(filepath.Join(l.LocalPath, file)); os.IsNotExist(err) { + return err + } + err := copyFile(filepath.Join(l.LocalPath, file), filepath.Join(l.RemotePath, file)) + if err != nil { + return err + } + return nil +} + +// CopyFrom copies file from a Path to local path +func (l localStorage) CopyFrom(file string) error { + if _, err := os.Stat(filepath.Join(l.RemotePath, file)); os.IsNotExist(err) { + return err + } + err := copyFile(filepath.Join(l.RemotePath, file), filepath.Join(l.LocalPath, file)) + if err != nil { + return err + } + return nil +} + +// Prune deletes old backup created more than specified days +func (l localStorage) Prune(retentionDays int) error { + currentTime := time.Now() + // Delete file + deleteFile := func(filePath string) error { + err := os.Remove(filePath) + return err + } + // Walk through the directory and delete files modified more than specified days ago + err := filepath.Walk(l.RemotePath, func(filePath string, fileInfo os.FileInfo, err error) error { + if err != nil { + return err + } + // Check if it's a regular file and if it was modified more than specified days ago + if fileInfo.Mode().IsRegular() { + timeDiff := currentTime.Sub(fileInfo.ModTime()) + if timeDiff.Hours() > 24*float64(retentionDays) { + err := deleteFile(filePath) + if err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } + return nil +} + +// Name returns the storage name +func (l localStorage) Name() string { + return "local" +} + +// copyFile copies file +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func(in *os.File) { + err := in.Close() + if err != nil { + return + } + }(in) + + out, err := os.Create(dst) + if err != nil { + return err + } + + _, err = io.Copy(out, in) + if err != nil { + err := out.Close() + if err != nil { + return err + } + return err + } + return out.Close() +} diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go new file mode 100644 index 0000000..37ae31d --- /dev/null +++ b/pkg/storage/local/local_test.go @@ -0,0 +1,66 @@ +package local + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +const content = "Lorem ipsum dolor sit amet. Eum eius voluptas sit vitae vitae aut sequi molestias hic accusamus consequatur" +const inputFile = "file.txt" +const localPath = "./tests/local" +const RemotePath = "./tests/remote" + +func TestCopy(t *testing.T) { + + err := os.MkdirAll(localPath, 0777) + if err != nil { + t.Error(err) + } + err = os.MkdirAll(RemotePath, 0777) + if err != nil { + t.Error(err) + } + + _, err = createFile(filepath.Join(localPath, inputFile), content) + if err != nil { + t.Error(err) + } + + l := NewStorage(Config{ + LocalPath: "./tests/local", + RemotePath: "./tests/remote", + }) + err = l.Copy(inputFile) + if err != nil { + t.Error(err) + } + fmt.Printf("File copied to %s\n", filepath.Join(RemotePath, inputFile)) +} +func createFile(fileName, content string) ([]byte, error) { + // Create a file named hello.txt + file, err := os.Create(fileName) + if err != nil { + fmt.Println("Error creating file:", err) + return nil, err + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + fmt.Println("Error closing file:", err) + return + } + }(file) + + // Write the message to the file + _, err = file.WriteString(content) + if err != nil { + fmt.Println("Error writing to file:", err) + return nil, err + } + + fmt.Printf("Successfully wrote to %s\n", fileName) + fileBytes, err := os.ReadFile(fileName) + return fileBytes, err +} diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go new file mode 100644 index 0000000..e974329 --- /dev/null +++ b/pkg/storage/s3/s3.go @@ -0,0 +1,176 @@ +package s3 + +import ( + "bytes" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + pkg "github.com/jkaninda/pg-bkup/pkg/storage" + "net/http" + "os" + "path/filepath" + "time" +) + +type s3Storage struct { + *pkg.Backend + client *session.Session + bucket string +} + +// Config holds the AWS S3 config +type Config struct { + Endpoint string + Bucket string + AccessKey string + SecretKey string + Region string + DisableSsl bool + ForcePathStyle bool + LocalPath string + RemotePath string +} + +// CreateSession creates a new AWS session +func createSession(conf Config) (*session.Session, error) { + s3Config := &aws.Config{ + Credentials: credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""), + Endpoint: aws.String(conf.Endpoint), + Region: aws.String(conf.Region), + DisableSSL: aws.Bool(conf.DisableSsl), + S3ForcePathStyle: aws.Bool(conf.ForcePathStyle), + } + + return session.NewSession(s3Config) +} + +// NewStorage creates new Storage +func NewStorage(conf Config) (pkg.Storage, error) { + sess, err := createSession(conf) + if err != nil { + return nil, err + } + return &s3Storage{ + client: sess, + bucket: conf.Bucket, + Backend: &pkg.Backend{ + RemotePath: conf.RemotePath, + LocalPath: conf.LocalPath, + }, + }, nil +} + +// Copy copies file to S3 storage +func (s s3Storage) Copy(fileName string) error { + svc := s3.New(s.client) + file, err := os.Open(filepath.Join(s.LocalPath, fileName)) + if err != nil { + return err + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + return + } + }(file) + + fileInfo, err := file.Stat() + if err != nil { + return err + } + objectKey := filepath.Join(s.RemotePath, fileName) + buffer := make([]byte, fileInfo.Size()) + _, err = file.Read(buffer) + if err != nil { + return err + } + fileBytes := bytes.NewReader(buffer) + fileType := http.DetectContentType(buffer) + + _, err = svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(objectKey), + Body: fileBytes, + ContentLength: aws.Int64(fileInfo.Size()), + ContentType: aws.String(fileType), + }) + if err != nil { + return err + } + + return nil +} + +// CopyFrom copies a file from S3 to local storage +func (s s3Storage) CopyFrom(fileName string) error { + file, err := os.Create(filepath.Join(s.LocalPath, fileName)) + if err != nil { + return err + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + fmt.Printf("Error closing file: %v\n", err) + return + } + }(file) + + objectKey := filepath.Join(s.RemotePath, fileName) + + downloader := s3manager.NewDownloader(s.client) + _, err = downloader.Download(file, + &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(objectKey), + }) + if err != nil { + return err + } + return nil +} + +// Prune deletes old backup created more than specified days +func (s s3Storage) Prune(retentionDays int) error { + svc := s3.New(s.client) + + // Get the current time + now := time.Now() + backupRetentionDays := now.AddDate(0, 0, -retentionDays) + + // List objects in the bucket + listObjectsInput := &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), + Prefix: aws.String(s.RemotePath), + } + err := svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, object := range page.Contents { + if object.LastModified.Before(backupRetentionDays) { + // Object is older than retention days, delete it + _, err := svc.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: object.Key, + }) + if err != nil { + fmt.Printf("failed to delete object %s: %v", *object.Key, err) + } else { + fmt.Printf("Deleted object %s", *object.Key) + } + } + } + return !lastPage + }) + if err != nil { + return fmt.Errorf("failed to list objects: %v", err) + } + + return nil + +} + +// Name returns the storage name +func (s s3Storage) Name() string { + return "s3" +} diff --git a/pkg/storage/ssh/ssh.go b/pkg/storage/ssh/ssh.go new file mode 100644 index 0000000..b4c5d3c --- /dev/null +++ b/pkg/storage/ssh/ssh.go @@ -0,0 +1,124 @@ +package ssh + +import ( + "context" + "errors" + "fmt" + "github.com/bramvdbogaerde/go-scp" + "github.com/bramvdbogaerde/go-scp/auth" + pkg "github.com/jkaninda/pg-bkup/pkg/storage" + "golang.org/x/crypto/ssh" + "os" + "path/filepath" +) + +type sshStorage struct { + *pkg.Backend + client scp.Client +} + +// Config holds the SSH connection details +type Config struct { + Host string + User string + Password string + Port string + IdentifyFile string + LocalPath string + RemotePath string +} + +// createClient creates SSH Client +func createClient(conf Config) (scp.Client, error) { + if _, err := os.Stat(conf.IdentifyFile); os.IsNotExist(err) { + clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey()) + return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err + } else { + if conf.Password == "" { + return scp.Client{}, errors.New("ssh password required") + } + clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey()) + return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err + + } +} + +// NewStorage creates new Storage +func NewStorage(conf Config) (pkg.Storage, error) { + client, err := createClient(conf) + if err != nil { + return nil, err + } + return &sshStorage{ + client: client, + Backend: &pkg.Backend{ + RemotePath: conf.RemotePath, + LocalPath: conf.LocalPath, + }, + }, nil +} + +// Copy copies file to the remote server +func (s sshStorage) Copy(fileName string) error { + client := s.client + // Connect to the remote server + err := client.Connect() + if err != nil { + return errors.New("couldn't establish a connection to the remote server") + } + // Open the local file + filePath := filepath.Join(s.LocalPath, fileName) + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", filePath, err) + } + defer client.Close() + // Copy file to the remote server + err = client.CopyFromFile(context.Background(), *file, filepath.Join(s.RemotePath, fileName), "0655") + if err != nil { + return fmt.Errorf("failed to copy file to remote server: %w", err) + } + + return nil +} + +// CopyFrom copies a file from the remote server to local storage +func (s sshStorage) CopyFrom(fileName string) error { + // Create a new SCP client + client := s.client + // Connect to the remote server + err := client.Connect() + if err != nil { + return errors.New("couldn't establish a connection to the remote server") + } + // Close client connection after the file has been copied + defer client.Close() + file, err := os.OpenFile(filepath.Join(s.LocalPath, fileName), os.O_RDWR|os.O_CREATE, 0777) + if err != nil { + return errors.New("couldn't open the output file") + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + return + } + }(file) + + err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName)) + + if err != nil { + return err + } + return nil +} + +// Prune deletes old backup created more than specified days +func (s sshStorage) Prune(retentionDays int) error { + fmt.Println("Deleting old backup from a remote server is not implemented yet") + return nil +} + +// Name returns the storage name +func (s sshStorage) Name() string { + return "ssh" +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 0000000..140fed9 --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,14 @@ +package pkg + +type Storage interface { + Copy(fileName string) error + CopyFrom(fileName string) error + Prune(retentionDays int) error + Name() string +} +type Backend struct { + // Local Path + LocalPath string + // Remote path or Destination path + RemotePath string +}