chore: integrate external storage module

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221 // indirect
+	github.com/jkaninda/go-storage v0.1.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	golang.org/x/sys v0.22.0 // indirect

go.sum

@@ -30,6 +30,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221 h1:AwkCf7el1kzeCJ89A+gUAK0ero5JYnvLOKsYMzq+rs4=
 github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221/go.mod h1:9F8ZJ+ZXE8DZBo77+aneGj8LMjrYXX6eFUCC/uqZOUo=
+github.com/jkaninda/go-storage v0.1.0 h1:YJNmkZAl4MOcdbxklh69Ss9sworhPbwC9X7HjoblOLo=
+github.com/jkaninda/go-storage v0.1.0/go.mod h1:7VK5gQISQaLxtLfBtc+een8spcgLVSBAKTRuyF1N81I=
+github.com/jkaninda/go-storage v0.1.1 h1:vjpdD/fh39S5HGyfHvLE5HGYOEPIukINlOX3OnM3GW4=
+github.com/jkaninda/go-storage v0.1.1/go.mod h1:7VK5gQISQaLxtLfBtc+een8spcgLVSBAKTRuyF1N81I=
 github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
 github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=

pkg/backup.go

@@ -9,10 +9,10 @@ package pkg
 import (
 	"fmt"
 	"github.com/jkaninda/encryptor"
-	"github.com/jkaninda/mysql-bkup/pkg/storage/ftp"
+	"github.com/jkaninda/go-storage/pkg/ftp"
-	"github.com/jkaninda/mysql-bkup/pkg/storage/local"
+	"github.com/jkaninda/go-storage/pkg/local"
-	"github.com/jkaninda/mysql-bkup/pkg/storage/s3"
+	"github.com/jkaninda/go-storage/pkg/s3"
-	"github.com/jkaninda/mysql-bkup/pkg/storage/ssh"
+	"github.com/jkaninda/go-storage/pkg/ssh"
 	"github.com/jkaninda/mysql-bkup/utils"
 	"github.com/robfig/cron/v3"
 	"github.com/spf13/cobra"

pkg/restore.go

@@ -8,10 +8,10 @@ package pkg
 
 import (
 	"github.com/jkaninda/encryptor"
-	"github.com/jkaninda/mysql-bkup/pkg/storage/ftp"
+	"github.com/jkaninda/go-storage/pkg/ftp"
-	"github.com/jkaninda/mysql-bkup/pkg/storage/local"
+	"github.com/jkaninda/go-storage/pkg/local"
-	"github.com/jkaninda/mysql-bkup/pkg/storage/s3"
+	"github.com/jkaninda/go-storage/pkg/s3"
-	"github.com/jkaninda/mysql-bkup/pkg/storage/ssh"
+	"github.com/jkaninda/go-storage/pkg/ssh"
 	"github.com/jkaninda/mysql-bkup/utils"
 	"github.com/spf13/cobra"
 	"os"

pkg/storage/ftp/ftp.go

@@ -1,118 +0,0 @@
-package ftp
-
-import (
-	"fmt"
-	"github.com/jkaninda/mysql-bkup/pkg/storage"
-	"github.com/jkaninda/mysql-bkup/utils"
-	"github.com/jlaffaye/ftp"
-	"io"
-	"os"
-	"path/filepath"
-	"time"
-)
-
-type ftpStorage struct {
-	*storage.Backend
-	client *ftp.ServerConn
-}
-
-// Config holds the SSH connection details
-type Config struct {
-	Host       string
-	User       string
-	Password   string
-	Port       string
-	LocalPath  string
-	RemotePath string
-}
-
-// createClient creates FTP Client
-func createClient(conf Config) (*ftp.ServerConn, error) {
-	ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
-	if err != nil {
-		return nil, fmt.Errorf("failed to connect to FTP: %w", err)
-	}
-
-	err = ftpClient.Login(conf.User, conf.Password)
-	if err != nil {
-		return nil, fmt.Errorf("failed to log in to FTP: %w", err)
-	}
-
-	return ftpClient, nil
-}
-
-// NewStorage creates new Storage
-func NewStorage(conf Config) (storage.Storage, error) {
-	client, err := createClient(conf)
-	if err != nil {
-		return nil, err
-	}
-	return &ftpStorage{
-		client: client,
-		Backend: &storage.Backend{
-			RemotePath: conf.RemotePath,
-			LocalPath:  conf.LocalPath,
-		},
-	}, nil
-}
-
-// Copy copies file to the remote server
-func (s ftpStorage) Copy(fileName string) error {
-	ftpClient := s.client
-	defer ftpClient.Quit()
-
-	filePath := filepath.Join(s.LocalPath, fileName)
-	file, err := os.Open(filePath)
-	if err != nil {
-		return fmt.Errorf("failed to open file %s: %w", fileName, err)
-	}
-	defer file.Close()
-
-	remoteFilePath := filepath.Join(s.RemotePath, fileName)
-	err = ftpClient.Stor(remoteFilePath, file)
-	if err != nil {
-		return fmt.Errorf("failed to upload file %s: %w", filepath.Join(s.LocalPath, fileName), err)
-	}
-
-	return nil
-}
-
-// CopyFrom copies a file from the remote server to local storage
-func (s ftpStorage) CopyFrom(fileName string) error {
-	ftpClient := s.client
-
-	defer ftpClient.Quit()
-
-	remoteFilePath := filepath.Join(s.RemotePath, fileName)
-	r, err := ftpClient.Retr(remoteFilePath)
-	if err != nil {
-		return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
-	}
-	defer r.Close()
-
-	localFilePath := filepath.Join(s.LocalPath, fileName)
-	outFile, err := os.Create(localFilePath)
-	if err != nil {
-		return fmt.Errorf("failed to create local file %s: %w", fileName, err)
-	}
-	defer outFile.Close()
-
-	_, err = io.Copy(outFile, r)
-	if err != nil {
-		return fmt.Errorf("failed to copy data to local file %s: %w", fileName, err)
-	}
-
-	return nil
-}
-
-// Prune deletes old backup created more than specified days
-func (s ftpStorage) Prune(retentionDays int) error {
-	utils.Info("Deleting old backup from a remote server is not implemented yet")
-	return nil
-
-}
-
-// Name returns the storage name
-func (s ftpStorage) Name() string {
-	return "ftp"
-}

pkg/storage/local/local.go

@@ -1,108 +0,0 @@
-package local
-
-import (
-	"github.com/jkaninda/mysql-bkup/pkg/storage"
-	"github.com/jkaninda/mysql-bkup/utils"
-	"io"
-	"os"
-	"path/filepath"
-	"time"
-)
-
-type localStorage struct {
-	*storage.Backend
-}
-type Config struct {
-	LocalPath  string
-	RemotePath string
-}
-
-func NewStorage(conf Config) storage.Storage {
-	return &localStorage{
-		Backend: &storage.Backend{
-			LocalPath:  conf.LocalPath,
-			RemotePath: conf.RemotePath,
-		},
-	}
-}
-func (l localStorage) Copy(file string) error {
-	if _, err := os.Stat(filepath.Join(l.LocalPath, file)); os.IsNotExist(err) {
-		return err
-	}
-	err := copyFile(filepath.Join(l.LocalPath, file), filepath.Join(l.RemotePath, file))
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (l localStorage) CopyFrom(file string) error {
-	if _, err := os.Stat(filepath.Join(l.RemotePath, file)); os.IsNotExist(err) {
-		return err
-	}
-	err := copyFile(filepath.Join(l.RemotePath, file), filepath.Join(l.LocalPath, file))
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// Prune deletes old backup created more than specified days
-func (l localStorage) Prune(retentionDays int) error {
-	currentTime := time.Now()
-	// Delete file
-	deleteFile := func(filePath string) error {
-		err := os.Remove(filePath)
-		if err != nil {
-			utils.Fatal("Error:", err)
-		} else {
-			utils.Info("File %s deleted successfully", filePath)
-		}
-		return err
-	}
-	// Walk through the directory and delete files modified more than specified days ago
-	err := filepath.Walk(l.RemotePath, func(filePath string, fileInfo os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		// Check if it's a regular file and if it was modified more than specified days ago
-		if fileInfo.Mode().IsRegular() {
-			timeDiff := currentTime.Sub(fileInfo.ModTime())
-			if timeDiff.Hours() > 24*float64(retentionDays) {
-				err := deleteFile(filePath)
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (l localStorage) Name() string {
-	return "local"
-}
-
-func copyFile(src, dst string) error {
-	in, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer in.Close()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-
-	_, err = io.Copy(out, in)
-	if err != nil {
-		out.Close()
-		return err
-	}
-	return out.Close()
-}

pkg/storage/s3/s3.go

@@ -1,162 +0,0 @@
-package s3
-
-import (
-	"bytes"
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-	"github.com/jkaninda/mysql-bkup/pkg/storage"
-	"github.com/jkaninda/mysql-bkup/utils"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-)
-
-type s3Storage struct {
-	*storage.Backend
-	client *session.Session
-	bucket string
-}
-type Config struct {
-	Endpoint       string
-	Bucket         string
-	AccessKey      string
-	SecretKey      string
-	Region         string
-	DisableSsl     bool
-	ForcePathStyle bool
-	LocalPath      string
-	RemotePath     string
-}
-
-// CreateSession creates a new AWS session
-func createSession(conf Config) (*session.Session, error) {
-	s3Config := &aws.Config{
-		Credentials:      credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""),
-		Endpoint:         aws.String(conf.Endpoint),
-		Region:           aws.String(conf.Region),
-		DisableSSL:       aws.Bool(conf.DisableSsl),
-		S3ForcePathStyle: aws.Bool(conf.ForcePathStyle),
-	}
-
-	return session.NewSession(s3Config)
-}
-
-func NewStorage(conf Config) (storage.Storage, error) {
-	sess, err := createSession(conf)
-	if err != nil {
-		return nil, err
-	}
-	return &s3Storage{
-		client: sess,
-		bucket: conf.Bucket,
-		Backend: &storage.Backend{
-			RemotePath: conf.RemotePath,
-			LocalPath:  conf.LocalPath,
-		},
-	}, nil
-}
-func (s s3Storage) Copy(fileName string) error {
-	svc := s3.New(s.client)
-	file, err := os.Open(filepath.Join(s.LocalPath, fileName))
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	fileInfo, err := file.Stat()
-	if err != nil {
-		return err
-	}
-	objectKey := filepath.Join(s.RemotePath, fileName)
-	buffer := make([]byte, fileInfo.Size())
-	file.Read(buffer)
-	fileBytes := bytes.NewReader(buffer)
-	fileType := http.DetectContentType(buffer)
-
-	_, err = svc.PutObject(&s3.PutObjectInput{
-		Bucket:        aws.String(s.bucket),
-		Key:           aws.String(objectKey),
-		Body:          fileBytes,
-		ContentLength: aws.Int64(fileInfo.Size()),
-		ContentType:   aws.String(fileType),
-	})
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// CopyFrom copies a file from the remote server to local storage
-func (s s3Storage) CopyFrom(fileName string) error {
-	file, err := os.Create(filepath.Join(s.LocalPath, fileName))
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	objectKey := filepath.Join(s.RemotePath, fileName)
-
-	downloader := s3manager.NewDownloader(s.client)
-	numBytes, err := downloader.Download(file,
-		&s3.GetObjectInput{
-			Bucket: aws.String(s.bucket),
-			Key:    aws.String(objectKey),
-		})
-	if err != nil {
-		utils.Error("Failed to download file %s", fileName)
-		return err
-	}
-	utils.Info("Backup downloaded:  %s , bytes size: %d ", file.Name(), uint64(numBytes))
-
-	return nil
-}
-
-// Prune deletes old backup created more than specified days
-func (s s3Storage) Prune(retentionDays int) error {
-	svc := s3.New(s.client)
-
-	// Get the current time
-	now := time.Now()
-	backupRetentionDays := now.AddDate(0, 0, -retentionDays)
-
-	// List objects in the bucket
-	listObjectsInput := &s3.ListObjectsV2Input{
-		Bucket: aws.String(s.bucket),
-		Prefix: aws.String(s.RemotePath),
-	}
-	err := svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
-		for _, object := range page.Contents {
-			if object.LastModified.Before(backupRetentionDays) {
-				utils.Info("Deleting old backup: %s", *object.Key)
-				// Object is older than retention days, delete it
-				_, err := svc.DeleteObject(&s3.DeleteObjectInput{
-					Bucket: aws.String(s.bucket),
-					Key:    object.Key,
-				})
-				if err != nil {
-					utils.Info("Failed to delete object %s: %v", *object.Key, err)
-				} else {
-					utils.Info("Deleted object %s", *object.Key)
-				}
-			}
-		}
-		return !lastPage
-	})
-	if err != nil {
-		utils.Error("Failed to list objects: %v", err)
-	}
-
-	utils.Info("Deleting old backups...done")
-	return nil
-
-}
-
-// Name returns the storage name
-func (s s3Storage) Name() string {
-	return "s3"
-}

pkg/storage/ssh/ssh.go

@@ -1,116 +0,0 @@
-package ssh
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"github.com/bramvdbogaerde/go-scp"
-	"github.com/bramvdbogaerde/go-scp/auth"
-	"github.com/jkaninda/mysql-bkup/pkg/storage"
-	"github.com/jkaninda/mysql-bkup/utils"
-	"golang.org/x/crypto/ssh"
-	"os"
-	"path/filepath"
-)
-
-type sshStorage struct {
-	*storage.Backend
-	client scp.Client
-}
-
-// Config holds the SSH connection details
-type Config struct {
-	Host         string
-	User         string
-	Password     string
-	Port         string
-	IdentifyFile string
-	LocalPath    string
-	RemotePath   string
-}
-
-func createClient(conf Config) (scp.Client, error) {
-	if conf.IdentifyFile != "" && utils.FileExists(conf.IdentifyFile) {
-		clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
-		return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
-	} else {
-		if conf.Password == "" {
-			return scp.Client{}, errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty")
-		}
-		utils.Warn("Accessing the remote server using password, which is not recommended.")
-		clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
-		return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
-
-	}
-}
-
-func NewStorage(conf Config) (storage.Storage, error) {
-	client, err := createClient(conf)
-	if err != nil {
-		return nil, err
-	}
-	return &sshStorage{
-		client: client,
-		Backend: &storage.Backend{
-			RemotePath: conf.RemotePath,
-			LocalPath:  conf.LocalPath,
-		},
-	}, nil
-}
-func (s sshStorage) Copy(fileName string) error {
-	client := s.client
-	// Connect to the remote server
-	err := client.Connect()
-	if err != nil {
-		return errors.New("couldn't establish a connection to the remote server")
-	}
-	// Open the local file
-	filePath := filepath.Join(s.LocalPath, fileName)
-	file, err := os.Open(filePath)
-	if err != nil {
-		return fmt.Errorf("failed to open file %s: %w", filePath, err)
-	}
-	defer client.Close()
-	// Copy file to the remote server
-	err = client.CopyFromFile(context.Background(), *file, filepath.Join(s.RemotePath, fileName), "0655")
-	if err != nil {
-		return fmt.Errorf("failed to copy file to remote server: %w", err)
-	}
-
-	return nil
-}
-
-// CopyFrom copies a file from the remote server to local storage
-func (s sshStorage) CopyFrom(fileName string) error {
-	// Create a new SCP client
-	client := s.client
-	// Connect to the remote server
-	err := client.Connect()
-	if err != nil {
-		return errors.New("couldn't establish a connection to the remote server")
-	}
-	// Close client connection after the file has been copied
-	defer client.Close()
-	file, err := os.OpenFile(filepath.Join(s.LocalPath, fileName), os.O_RDWR|os.O_CREATE, 0777)
-	if err != nil {
-		return errors.New("couldn't open the output file")
-	}
-	defer file.Close()
-
-	err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))
-
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// Prune deletes old backup created more than specified days
-func (s sshStorage) Prune(retentionDays int) error {
-	utils.Info("Deleting old backup from a remote server is not implemented yet")
-	return nil
-}
-
-func (s sshStorage) Name() string {
-	return "ssh"
-}

pkg/storage/storage.go

@@ -1,14 +0,0 @@
-package storage
-
-type Storage interface {
-	Copy(fileName string) error
-	CopyFrom(fileName string) error
-	Prune(retentionDays int) error
-	Name() string
-}
-type Backend struct {
-	//Local Path
-	LocalPath string
-	//Remote path or Destination path
-	RemotePath string
-}