jkaninda/mysql-bkup
1
1
Fork 0
mirror of https://github.com/jkaninda/mysql-bkup.git synced 2025-12-06 05:29:41 +01:00
Code Issues Actions Projects Releases Wiki Activity

refactor: Restructure project files for better organization, readability, and maintainability

Browse Source
This commit is contained in:
Jonas Kaninda
2024-11-04 09:02:41 +01:00
parent 8e20e9595f
commit eb414d818c
9 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

514
pkg/backup.go
View File

@@ -1,514 +0,0 @@
// Package pkg /
/*****
@author Jonas Kaninda
@license MIT License <https://opensource.org/licenses/MIT>
@Copyright © 2024 Jonas Kaninda
**/
package pkg
import (
"fmt"
"github.com/jkaninda/encryptor"
"github.com/jkaninda/go-storage/pkg/ftp"
"github.com/jkaninda/go-storage/pkg/local"
"github.com/jkaninda/go-storage/pkg/s3"
"github.com/jkaninda/go-storage/pkg/ssh"
"github.com/jkaninda/mysql-bkup/utils"
"github.com/robfig/cron/v3"
"github.com/spf13/cobra"
"log"
"os"
"os/exec"
"path/filepath"
"time"
)
func StartBackup(cmd *cobra.Command) {
intro()
//Initialize backup configs
config := initBackupConfig(cmd)
//Load backup configuration file
configFile, err := loadConfigFile()
if err != nil {
dbConf = initDbConfig(cmd)
if config.cronExpression == "" {
BackupTask(dbConf, config)
} else {
if utils.IsValidCronExpression(config.cronExpression) {
scheduledMode(dbConf, config)
} else {
utils.Fatal("Cron expression is not valid: %s", config.cronExpression)
}
}
} else {
startMultiBackup(config, configFile)
}
}
// scheduledMode Runs backup in scheduled mode
func scheduledMode(db *dbConfig, config *BackupConfig) {
utils.Info("Running in Scheduled mode")
utils.Info("Backup cron expression: %s", config.cronExpression)
utils.Info("The next scheduled time is: %v", utils.CronNextTime(config.cronExpression).Format(timeFormat))
utils.Info("Storage type %s ", config.storage)
//Test backup
utils.Info("Testing backup configurations...")
BackupTask(db, config)
utils.Info("Testing backup configurations...done")
utils.Info("Creating backup job...")
// Create a new cron instance
c := cron.New()
_, err := c.AddFunc(config.cronExpression, func() {
BackupTask(db, config)
utils.Info("Next backup time is: %v", utils.CronNextTime(config.cronExpression).Format(timeFormat))
})
if err != nil {
return
}
// Start the cron scheduler
c.Start()
utils.Info("Creating backup job...done")
utils.Info("Backup job started")
defer c.Stop()
select {}
}
// multiBackupTask backup multi database
func multiBackupTask(databases []Database, bkConfig *BackupConfig) {
for _, db := range databases {
//Check if path is defined in config file
if db.Path != "" {
bkConfig.remotePath = db.Path
}
BackupTask(getDatabase(db), bkConfig)
}
}
// BackupTask backups database
func BackupTask(db *dbConfig, config *BackupConfig) {
utils.Info("Starting backup task...")
//Generate file name
backupFileName := fmt.Sprintf("%s_%s.sql.gz", db.dbName, time.Now().Format("20060102_150405"))
if config.disableCompression {
backupFileName = fmt.Sprintf("%s_%s.sql", db.dbName, time.Now().Format("20060102_150405"))
}
config.backupFileName = backupFileName
switch config.storage {
case "local":
localBackup(db, config)
case "s3", "S3":
s3Backup(db, config)
case "ssh", "SSH", "remote":
sshBackup(db, config)
case "ftp", "FTP":
ftpBackup(db, config)
default:
localBackup(db, config)
}
}
func startMultiBackup(bkConfig *BackupConfig, configFile string) {
utils.Info("Starting backup task...")
conf, err := readConf(configFile)
if err != nil {
utils.Fatal("Error reading config file: %s", err)
}
//Check if cronExpression is defined in config file
if conf.CronExpression != "" {
bkConfig.cronExpression = conf.CronExpression
}
// Check if cronExpression is defined
if bkConfig.cronExpression == "" {
multiBackupTask(conf.Databases, bkConfig)
} else {
// Check if cronExpression is valid
if utils.IsValidCronExpression(bkConfig.cronExpression) {
utils.Info("Running backup in Scheduled mode")
utils.Info("Backup cron expression: %s", bkConfig.cronExpression)
utils.Info("The next scheduled time is: %v", utils.CronNextTime(bkConfig.cronExpression).Format(timeFormat))
utils.Info("Storage type %s ", bkConfig.storage)
//Test backup
utils.Info("Testing backup configurations...")
multiBackupTask(conf.Databases, bkConfig)
utils.Info("Testing backup configurations...done")
utils.Info("Creating backup job...")
// Create a new cron instance
c := cron.New()
_, err := c.AddFunc(bkConfig.cronExpression, func() {
multiBackupTask(conf.Databases, bkConfig)
utils.Info("Next backup time is: %v", utils.CronNextTime(bkConfig.cronExpression).Format(timeFormat))
})
if err != nil {
return
}
// Start the cron scheduler
c.Start()
utils.Info("Creating backup job...done")
utils.Info("Backup job started")
defer c.Stop()
select {}
} else {
utils.Fatal("Cron expression is not valid: %s", bkConfig.cronExpression)
}
}
}
// BackupDatabase backup database
func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool) {
storagePath = os.Getenv("STORAGE_PATH")
utils.Info("Starting database backup...")
err := os.Setenv("MYSQL_PWD", db.dbPassword)
if err != nil {
return
}
testDatabaseConnection(db)
// Backup Database database
utils.Info("Backing up database...")
// Verify is compression is disabled
if disableCompression {
// Execute mysqldump
cmd := exec.Command("mysqldump",
"-h", db.dbHost,
"-P", db.dbPort,
"-u", db.dbUserName,
db.dbName,
)
output, err := cmd.Output()
if err != nil {
log.Fatal(err)
}
// save output
file, err := os.Create(filepath.Join(tmpPath, backupFileName))
if err != nil {
log.Fatal(err)
}
defer file.Close()
_, err = file.Write(output)
if err != nil {
log.Fatal(err)
}
utils.Info("Database has been backed up")
} else {
// Execute mysqldump
cmd := exec.Command("mysqldump", "-h", db.dbHost, "-P", db.dbPort, "-u", db.dbUserName, db.dbName)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
gzipCmd := exec.Command("gzip")
gzipCmd.Stdin = stdout
gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName))
gzipCmd.Start()
if err != nil {
log.Fatal(err)
}
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
if err := gzipCmd.Wait(); err != nil {
log.Fatal(err)
}
utils.Info("Database has been backed up")
}
}
func localBackup(db *dbConfig, config *BackupConfig) {
utils.Info("Backup database to local storage")
startTime = time.Now().Format(utils.TimeFormat())
BackupDatabase(db, config.backupFileName, disableCompression)
finalFileName := config.backupFileName
if config.encryption {
encryptBackup(config)
finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, gpgExtension)
}
fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName))
if err != nil {
utils.Error("Error: %s", err)
}
backupSize = fileInfo.Size()
utils.Info("Backup name is %s", finalFileName)
localStorage := local.NewStorage(local.Config{
LocalPath: tmpPath,
RemotePath: storagePath,
})
err = localStorage.Copy(finalFileName)
if err != nil {
utils.Fatal("Error copying backup file: %s", err)
}
utils.Info("Backup saved in %s", filepath.Join(storagePath, finalFileName))
//Send notification
utils.NotifySuccess(&utils.NotificationData{
File: finalFileName,
BackupSize: backupSize,
Database: db.dbName,
Storage: config.storage,
BackupLocation: filepath.Join(storagePath, finalFileName),
StartTime: startTime,
EndTime: time.Now().Format(utils.TimeFormat()),
})
//Delete old backup
if config.prune {
err = localStorage.Prune(config.backupRetention)
if err != nil {
utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err)
}
}
//Delete temp
deleteTemp()
utils.Info("Backup completed successfully")
}
func s3Backup(db *dbConfig, config *BackupConfig) {
utils.Info("Backup database to s3 storage")
startTime = time.Now().Format(utils.TimeFormat())
//Backup database
BackupDatabase(db, config.backupFileName, disableCompression)
finalFileName := config.backupFileName
if config.encryption {
encryptBackup(config)
finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg")
}
utils.Info("Uploading backup archive to remote storage S3 ... ")
awsConfig := initAWSConfig()
if config.remotePath == "" {
config.remotePath = awsConfig.remotePath
}
utils.Info("Backup name is %s", finalFileName)
s3Storage, err := s3.NewStorage(s3.Config{
Endpoint: awsConfig.endpoint,
Bucket: awsConfig.bucket,
AccessKey: awsConfig.accessKey,
SecretKey: awsConfig.secretKey,
Region: awsConfig.region,
DisableSsl: awsConfig.disableSsl,
ForcePathStyle: awsConfig.forcePathStyle,
RemotePath: awsConfig.remotePath,
LocalPath: tmpPath,
})
if err != nil {
utils.Fatal("Error creating s3 storage: %s", err)
}
err = s3Storage.Copy(finalFileName)
if err != nil {
utils.Fatal("Error copying backup file: %s", err)
}
//Get backup info
fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName))
if err != nil {
utils.Error("Error: %s", err)
}
backupSize = fileInfo.Size()
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, config.backupFileName))
if err != nil {
fmt.Println("Error deleting file: ", err)
}
// Delete old backup
if config.prune {
err := s3Storage.Prune(config.backupRetention)
if err != nil {
utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err)
}
}
utils.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName))
utils.Info("Uploading backup archive to remote storage S3 ... done ")
//Send notification
utils.NotifySuccess(&utils.NotificationData{
File: finalFileName,
BackupSize: backupSize,
Database: db.dbName,
Storage: config.storage,
BackupLocation: filepath.Join(config.remotePath, finalFileName),
StartTime: startTime,
EndTime: time.Now().Format(utils.TimeFormat()),
})
//Delete temp
deleteTemp()
utils.Info("Backup completed successfully")
}
func sshBackup(db *dbConfig, config *BackupConfig) {
utils.Info("Backup database to Remote server")
startTime = time.Now().Format(utils.TimeFormat())
//Backup database
BackupDatabase(db, config.backupFileName, disableCompression)
finalFileName := config.backupFileName
if config.encryption {
encryptBackup(config)
finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg")
}
utils.Info("Uploading backup archive to remote storage ... ")
utils.Info("Backup name is %s", finalFileName)
sshConfig, err := loadSSHConfig()
if err != nil {
utils.Fatal("Error loading ssh config: %s", err)
}
sshStorage, err := ssh.NewStorage(ssh.Config{
Host: sshConfig.hostName,
Port: sshConfig.port,
User: sshConfig.user,
Password: sshConfig.password,
IdentifyFile: sshConfig.identifyFile,
RemotePath: config.remotePath,
LocalPath: tmpPath,
})
if err != nil {
utils.Fatal("Error creating SSH storage: %s", err)
}
err = sshStorage.Copy(finalFileName)
if err != nil {
utils.Fatal("Error copying backup file: %s", err)
}
//Get backup info
fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName))
if err != nil {
utils.Error("Error: %s", err)
}
backupSize = fileInfo.Size()
utils.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName))
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName))
if err != nil {
utils.Error("Error deleting file: %v", err)
}
if config.prune {
err := sshStorage.Prune(config.backupRetention)
if err != nil {
utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err)
}
}
utils.Info("Uploading backup archive to remote storage ... done ")
//Send notification
utils.NotifySuccess(&utils.NotificationData{
File: finalFileName,
BackupSize: backupSize,
Database: db.dbName,
Storage: config.storage,
BackupLocation: filepath.Join(config.remotePath, finalFileName),
StartTime: startTime,
EndTime: time.Now().Format(utils.TimeFormat()),
})
//Delete temp
deleteTemp()
utils.Info("Backup completed successfully")
}
func ftpBackup(db *dbConfig, config *BackupConfig) {
utils.Info("Backup database to the remote FTP server")
startTime = time.Now().Format(utils.TimeFormat())
//Backup database
BackupDatabase(db, config.backupFileName, disableCompression)
finalFileName := config.backupFileName
if config.encryption {
encryptBackup(config)
finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg")
}
utils.Info("Uploading backup archive to the remote FTP server ... ")
utils.Info("Backup name is %s", finalFileName)
ftpConfig := loadFtpConfig()
ftpStorage, err := ftp.NewStorage(ftp.Config{
Host: ftpConfig.host,
Port: ftpConfig.port,
User: ftpConfig.user,
Password: ftpConfig.password,
RemotePath: config.remotePath,
LocalPath: tmpPath,
})
if err != nil {
utils.Fatal("Error creating SSH storage: %s", err)
}
err = ftpStorage.Copy(finalFileName)
if err != nil {
utils.Fatal("Error copying backup file: %s", err)
}
utils.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName))
//Get backup info
fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName))
if err != nil {
utils.Error("Error: %s", err)
}
backupSize = fileInfo.Size()
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName))
if err != nil {
utils.Error("Error deleting file: %v", err)
}
if config.prune {
err := ftpStorage.Prune(config.backupRetention)
if err != nil {
utils.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err)
}
}
utils.Info("Uploading backup archive to the remote FTP server ... done ")
//Send notification
utils.NotifySuccess(&utils.NotificationData{
File: finalFileName,
BackupSize: backupSize,
Database: db.dbName,
Storage: config.storage,
BackupLocation: filepath.Join(config.remotePath, finalFileName),
StartTime: startTime,
EndTime: time.Now().Format(utils.TimeFormat()),
})
//Delete temp
deleteTemp()
utils.Info("Backup completed successfully")
}
func encryptBackup(config *BackupConfig) {
backupFile, err := os.ReadFile(filepath.Join(tmpPath, config.backupFileName))
outputFile := fmt.Sprintf("%s.%s", filepath.Join(tmpPath, config.backupFileName), gpgExtension)
if err != nil {
utils.Fatal("Error reading backup file: %s ", err)
}
if config.usingKey {
utils.Info("Encrypting backup using public key...")
pubKey, err := os.ReadFile(config.publicKey)
if err != nil {
utils.Fatal("Error reading public key: %s ", err)
}
err = encryptor.EncryptWithPublicKey(backupFile, fmt.Sprintf("%s.%s", filepath.Join(tmpPath, config.backupFileName), gpgExtension), pubKey)
if err != nil {
utils.Fatal("Error encrypting backup file: %v ", err)
}
utils.Info("Encrypting backup using public key...done")
} else if config.passphrase != "" {
utils.Info("Encrypting backup using passphrase...")
err := encryptor.Encrypt(backupFile, outputFile, config.passphrase)
if err != nil {
utils.Fatal("error during encrypting backup %v", err)
}
utils.Info("Encrypting backup using passphrase...done")
}
}

279
pkg/config.go
View File

@@ -1,279 +0,0 @@
// Package pkg /
/*****
@author Jonas Kaninda
@license MIT License <https://opensource.org/licenses/MIT>
@Copyright © 2024 Jonas Kaninda
**/
package pkg
import (
"fmt"
"github.com/jkaninda/mysql-bkup/utils"
"github.com/spf13/cobra"
"os"
"strconv"
)
type Database struct {
Host string `yaml:"host"`
Port string `yaml:"port"`
Name string `yaml:"name"`
User string `yaml:"user"`
Password string `yaml:"password"`
Path string `yaml:"path"`
}
type Config struct {
Databases []Database `yaml:"databases"`
CronExpression string `yaml:"cronExpression"`
}
type dbConfig struct {
dbHost string
dbPort string
dbName string
dbUserName string
dbPassword string
}
type targetDbConfig struct {
targetDbHost string
targetDbPort string
targetDbUserName string
targetDbPassword string
targetDbName string
}
type TgConfig struct {
Token string
ChatId string
}
type BackupConfig struct {
backupFileName string
backupRetention int
disableCompression bool
prune bool
remotePath string
encryption bool
usingKey bool
passphrase string
publicKey string
storage string
cronExpression string
}
type FTPConfig struct {
host string
user string
password string
port string
remotePath string
}
// SSHConfig holds the SSH connection details
type SSHConfig struct {
user string
password string
hostName string
port string
identifyFile string
}
type AWSConfig struct {
endpoint string
bucket string
accessKey string
secretKey string
region string
remotePath string
disableSsl bool
forcePathStyle bool
}
func initDbConfig(cmd *cobra.Command) *dbConfig {
//Set env
utils.GetEnv(cmd, "dbname", "DB_NAME")
dConf := dbConfig{}
dConf.dbHost = os.Getenv("DB_HOST")
dConf.dbPort = os.Getenv("DB_PORT")
dConf.dbName = os.Getenv("DB_NAME")
dConf.dbUserName = os.Getenv("DB_USERNAME")
dConf.dbPassword = os.Getenv("DB_PASSWORD")
err := utils.CheckEnvVars(dbHVars)
if err != nil {
utils.Error("Please make sure all required environment variables for database are set")
utils.Fatal("Error checking environment variables: %s", err)
}
return &dConf
}
func getDatabase(database Database) *dbConfig {
return &dbConfig{
dbHost: database.Host,
dbPort: database.Port,
dbName: database.Name,
dbUserName: database.User,
dbPassword: database.Password,
}
}
// loadSSHConfig loads the SSH configuration from environment variables
func loadSSHConfig() (*SSHConfig, error) {
utils.GetEnvVariable("SSH_HOST", "SSH_HOST_NAME")
sshVars := []string{"SSH_USER", "SSH_HOST", "SSH_PORT", "REMOTE_PATH"}
err := utils.CheckEnvVars(sshVars)
if err != nil {
return nil, fmt.Errorf("error missing environment variables: %w", err)
}
return &SSHConfig{
user: os.Getenv("SSH_USER"),
password: os.Getenv("SSH_PASSWORD"),
hostName: os.Getenv("SSH_HOST"),
port: os.Getenv("SSH_PORT"),
identifyFile: os.Getenv("SSH_IDENTIFY_FILE"),
}, nil
}
func loadFtpConfig() *FTPConfig {
//Initialize data configs
fConfig := FTPConfig{}
fConfig.host = utils.GetEnvVariable("FTP_HOST", "FTP_HOST_NAME")
fConfig.user = os.Getenv("FTP_USER")
fConfig.password = os.Getenv("FTP_PASSWORD")
fConfig.port = os.Getenv("FTP_PORT")
fConfig.remotePath = os.Getenv("REMOTE_PATH")
err := utils.CheckEnvVars(ftpVars)
if err != nil {
utils.Error("Please make sure all required environment variables for FTP are set")
utils.Fatal("Error missing environment variables: %s", err)
}
return &fConfig
}
func initAWSConfig() *AWSConfig {
//Initialize AWS configs
aConfig := AWSConfig{}
aConfig.endpoint = utils.GetEnvVariable("AWS_S3_ENDPOINT", "S3_ENDPOINT")
aConfig.accessKey = utils.GetEnvVariable("AWS_ACCESS_KEY", "ACCESS_KEY")
aConfig.secretKey = utils.GetEnvVariable("AWS_SECRET_KEY", "SECRET_KEY")
aConfig.bucket = utils.GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME")
aConfig.remotePath = utils.GetEnvVariable("AWS_S3_PATH", "S3_PATH")
aConfig.region = os.Getenv("AWS_REGION")
disableSsl, err := strconv.ParseBool(os.Getenv("AWS_DISABLE_SSL"))
if err != nil {
disableSsl = false
}
forcePathStyle, err := strconv.ParseBool(os.Getenv("AWS_FORCE_PATH_STYLE"))
if err != nil {
forcePathStyle = false
}
aConfig.disableSsl = disableSsl
aConfig.forcePathStyle = forcePathStyle
err = utils.CheckEnvVars(awsVars)
if err != nil {
utils.Error("Please make sure all required environment variables for AWS S3 are set")
utils.Fatal("Error checking environment variables: %s", err)
}
return &aConfig
}
func initBackupConfig(cmd *cobra.Command) *BackupConfig {
utils.SetEnv("STORAGE_PATH", storagePath)
utils.GetEnv(cmd, "cron-expression", "BACKUP_CRON_EXPRESSION")
utils.GetEnv(cmd, "path", "REMOTE_PATH")
//Get flag value and set env
remotePath := utils.GetEnvVariable("REMOTE_PATH", "SSH_REMOTE_PATH")
storage = utils.GetEnv(cmd, "storage", "STORAGE")
prune := false
backupRetention := utils.GetIntEnv("BACKUP_RETENTION_DAYS")
if backupRetention > 0 {
prune = true
}
disableCompression, _ = cmd.Flags().GetBool("disable-compression")
_, _ = cmd.Flags().GetString("mode")
passphrase := os.Getenv("GPG_PASSPHRASE")
_ = utils.GetEnv(cmd, "path", "AWS_S3_PATH")
cronExpression := os.Getenv("BACKUP_CRON_EXPRESSION")
publicKeyFile, err := checkPubKeyFile(os.Getenv("GPG_PUBLIC_KEY"))
if err == nil {
encryption = true
usingKey = true
} else if passphrase != "" {
encryption = true
usingKey = false
}
//Initialize backup configs
config := BackupConfig{}
config.backupRetention = backupRetention
config.disableCompression = disableCompression
config.prune = prune
config.storage = storage
config.encryption = encryption
config.remotePath = remotePath
config.passphrase = passphrase
config.publicKey = publicKeyFile
config.usingKey = usingKey
config.cronExpression = cronExpression
return &config
}
type RestoreConfig struct {
s3Path string
remotePath string
storage string
file string
bucket string
usingKey bool
passphrase string
privateKey string
}
func initRestoreConfig(cmd *cobra.Command) *RestoreConfig {
utils.SetEnv("STORAGE_PATH", storagePath)
utils.GetEnv(cmd, "path", "REMOTE_PATH")
//Get flag value and set env
s3Path := utils.GetEnv(cmd, "path", "AWS_S3_PATH")
remotePath := utils.GetEnvVariable("REMOTE_PATH", "SSH_REMOTE_PATH")
storage = utils.GetEnv(cmd, "storage", "STORAGE")
file = utils.GetEnv(cmd, "file", "FILE_NAME")
bucket := utils.GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME")
passphrase := os.Getenv("GPG_PASSPHRASE")
privateKeyFile, err := checkPrKeyFile(os.Getenv("GPG_PRIVATE_KEY"))
if err == nil {
usingKey = true
} else if passphrase != "" {
usingKey = false
}
//Initialize restore configs
rConfig := RestoreConfig{}
rConfig.s3Path = s3Path
rConfig.remotePath = remotePath
rConfig.storage = storage
rConfig.bucket = bucket
rConfig.file = file
rConfig.storage = storage
rConfig.passphrase = passphrase
rConfig.usingKey = usingKey
rConfig.privateKey = privateKeyFile
return &rConfig
}
func initTargetDbConfig() *targetDbConfig {
tdbConfig := targetDbConfig{}
tdbConfig.targetDbHost = os.Getenv("TARGET_DB_HOST")
tdbConfig.targetDbPort = utils.EnvWithDefault("TARGET_DB_PORT", "5432")
tdbConfig.targetDbName = os.Getenv("TARGET_DB_NAME")
tdbConfig.targetDbUserName = os.Getenv("TARGET_DB_USERNAME")
tdbConfig.targetDbPassword = os.Getenv("TARGET_DB_PASSWORD")
err := utils.CheckEnvVars(tdbRVars)
if err != nil {
utils.Error("Please make sure all required environment variables for the target database are set")
utils.Fatal("Error checking target database environment variables: %s", err)
}
return &tdbConfig
}
func loadConfigFile() (string, error) {
backupConfigFile, err := checkConfigFile(os.Getenv("BACKUP_CONFIG_FILE"))
if err == nil {
return backupConfigFile, nil
}
return "", fmt.Errorf("backup config file not found")
}

162
pkg/helper.go
View File

@@ -1,162 +0,0 @@
// Package pkg /
/*****
@author Jonas Kaninda
@license MIT License <https://opensource.org/licenses/MIT>
@Copyright © 2024 Jonas Kaninda
**/
package pkg
import (
"bytes"
"fmt"
"github.com/jkaninda/mysql-bkup/utils"
"gopkg.in/yaml.v3"
"os"
"os/exec"
"path/filepath"
"strings"
)
func intro() {
utils.Info("Starting MySQL Backup...")
utils.Info("Copyright (c) 2024 Jonas Kaninda ")
}
// copyToTmp copy file to temporary directory
func deleteTemp() {
utils.Info("Deleting %s ...", tmpPath)
err := filepath.Walk(tmpPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Check if the current item is a file
if !info.IsDir() {
// Delete the file
err = os.Remove(path)
if err != nil {
return err
}
}
return nil
})
if err != nil {
utils.Error("Error deleting files: %v", err)
} else {
utils.Info("Deleting %s ... done", tmpPath)
}
}
// TestDatabaseConnection tests the database connection
func testDatabaseConnection(db *dbConfig) {
err := os.Setenv("MYSQL_PWD", db.dbPassword)
if err != nil {
return
}
utils.Info("Connecting to %s database ...", db.dbName)
cmd := exec.Command("mysql", "-h", db.dbHost, "-P", db.dbPort, "-u", db.dbUserName, db.dbName, "-e", "quit")
// Capture the output
var out bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &out
err = cmd.Run()
if err != nil {
utils.Fatal("Error testing database connection: %v\nOutput: %s", err, out.String())
}
utils.Info("Successfully connected to %s database", db.dbName)
}
// checkPubKeyFile checks gpg public key
func checkPubKeyFile(pubKey string) (string, error) {
// Define possible key file names
keyFiles := []string{filepath.Join(gpgHome, "public_key.asc"), filepath.Join(gpgHome, "public_key.gpg"), pubKey}
// Loop through key file names and check if they exist
for _, keyFile := range keyFiles {
if _, err := os.Stat(keyFile); err == nil {
// File exists
return keyFile, nil
} else if os.IsNotExist(err) {
// File does not exist, continue to the next one
continue
} else {
// An unexpected error occurred
return "", err
}
}
// Return an error if neither file exists
return "", fmt.Errorf("no public key file found")
}
// checkPrKeyFile checks private key
func checkPrKeyFile(prKey string) (string, error) {
// Define possible key file names
keyFiles := []string{filepath.Join(gpgHome, "private_key.asc"), filepath.Join(gpgHome, "private_key.gpg"), prKey}
// Loop through key file names and check if they exist
for _, keyFile := range keyFiles {
if _, err := os.Stat(keyFile); err == nil {
// File exists
return keyFile, nil
} else if os.IsNotExist(err) {
// File does not exist, continue to the next one
continue
} else {
// An unexpected error occurred
return "", err
}
}
// Return an error if neither file exists
return "", fmt.Errorf("no public key file found")
}
// readConf reads config file and returns Config
func readConf(configFile string) (*Config, error) {
if utils.FileExists(configFile) {
buf, err := os.ReadFile(configFile)
if err != nil {
return nil, err
}
c := &Config{}
err = yaml.Unmarshal(buf, c)
if err != nil {
return nil, fmt.Errorf("in file %q: %w", configFile, err)
}
return c, err
}
return nil, fmt.Errorf("config file %q not found", configFile)
}
// checkConfigFile checks config files and returns one config file
func checkConfigFile(filePath string) (string, error) {
// Define possible config file names
configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath}
// Loop through config file names and check if they exist
for _, configFile := range configFiles {
if _, err := os.Stat(configFile); err == nil {
// File exists
return configFile, nil
} else if os.IsNotExist(err) {
// File does not exist, continue to the next one
continue
} else {
// An unexpected error occurred
return "", err
}
}
// Return an error if neither file exists
return "", fmt.Errorf("no config file found")
}
func RemoveLastExtension(filename string) string {
if idx := strings.LastIndex(filename, "."); idx != -1 {
return filename[:idx]
}
return filename
}

42
pkg/migrate.go
View File

@@ -1,42 +0,0 @@
// Package pkg /
/*****
@author Jonas Kaninda
@license MIT License <https://opensource.org/licenses/MIT>
@Copyright © 2024 Jonas Kaninda
**/
package pkg
import (
"fmt"
"github.com/jkaninda/mysql-bkup/utils"
"github.com/spf13/cobra"
"time"
)
func StartMigration(cmd *cobra.Command) {
intro()
utils.Info("Starting database migration...")
//Get DB config
dbConf = initDbConfig(cmd)
targetDbConf = initTargetDbConfig()
//Defining the target database variables
newDbConfig := dbConfig{}
newDbConfig.dbHost = targetDbConf.targetDbHost
newDbConfig.dbPort = targetDbConf.targetDbPort
newDbConfig.dbName = targetDbConf.targetDbName
newDbConfig.dbUserName = targetDbConf.targetDbUserName
newDbConfig.dbPassword = targetDbConf.targetDbPassword
//Generate file name
backupFileName := fmt.Sprintf("%s_%s.sql", dbConf.dbName, time.Now().Format("20060102_150405"))
conf := &RestoreConfig{}
conf.file = backupFileName
//Backup source Database
BackupDatabase(dbConf, backupFileName, true)
//Restore source database into target database
utils.Info("Restoring [%s] database into [%s] database...", dbConf.dbName, targetDbConf.targetDbName)
RestoreDatabase(&newDbConfig, conf)
utils.Info("[%s] database has been restored into [%s] database", dbConf.dbName, targetDbConf.targetDbName)
utils.Info("Database migration completed.")
}

208
pkg/restore.go
View File

@@ -1,208 +0,0 @@
// Package pkg /
/*****
@author Jonas Kaninda
@license MIT License <https://opensource.org/licenses/MIT>
@Copyright © 2024 Jonas Kaninda
**/
package pkg
import (
"github.com/jkaninda/encryptor"
"github.com/jkaninda/go-storage/pkg/ftp"
"github.com/jkaninda/go-storage/pkg/local"
"github.com/jkaninda/go-storage/pkg/s3"
"github.com/jkaninda/go-storage/pkg/ssh"
"github.com/jkaninda/mysql-bkup/utils"
"github.com/spf13/cobra"
"os"
"os/exec"
"path/filepath"
)
func StartRestore(cmd *cobra.Command) {
intro()
dbConf = initDbConfig(cmd)
restoreConf := initRestoreConfig(cmd)
switch restoreConf.storage {
case "local":
localRestore(dbConf, restoreConf)
case "s3", "S3":
restoreFromS3(dbConf, restoreConf)
case "ssh", "SSH", "remote":
restoreFromRemote(dbConf, restoreConf)
case "ftp", "FTP":
restoreFromFTP(dbConf, restoreConf)
default:
localRestore(dbConf, restoreConf)
}
}
func localRestore(dbConf *dbConfig, restoreConf *RestoreConfig) {
utils.Info("Restore database from local")
localStorage := local.NewStorage(local.Config{
RemotePath: storagePath,
LocalPath: tmpPath,
})
err := localStorage.CopyFrom(restoreConf.file)
if err != nil {
utils.Fatal("Error copying backup file: %s", err)
}
RestoreDatabase(dbConf, restoreConf)
}
func restoreFromS3(db *dbConfig, conf *RestoreConfig) {
utils.Info("Restore database from s3")
awsConfig := initAWSConfig()
if conf.remotePath == "" {
conf.remotePath = awsConfig.remotePath
}
s3Storage, err := s3.NewStorage(s3.Config{
Endpoint: awsConfig.endpoint,
Bucket: awsConfig.bucket,
AccessKey: awsConfig.accessKey,
SecretKey: awsConfig.secretKey,
Region: awsConfig.region,
DisableSsl: awsConfig.disableSsl,
ForcePathStyle: awsConfig.forcePathStyle,
RemotePath: awsConfig.remotePath,
LocalPath: tmpPath,
})
if err != nil {
utils.Fatal("Error creating s3 storage: %s", err)
}
err = s3Storage.CopyFrom(conf.file)
if err != nil {
utils.Fatal("Error download file from S3 storage: %s", err)
}
RestoreDatabase(db, conf)
}
func restoreFromRemote(db *dbConfig, conf *RestoreConfig) {
utils.Info("Restore database from remote server")
sshConfig, err := loadSSHConfig()
if err != nil {
utils.Fatal("Error loading ssh config: %s", err)
}
sshStorage, err := ssh.NewStorage(ssh.Config{
Host: sshConfig.hostName,
Port: sshConfig.port,
User: sshConfig.user,
Password: sshConfig.password,
RemotePath: conf.remotePath,
LocalPath: tmpPath,
})
if err != nil {
utils.Fatal("Error creating SSH storage: %s", err)
}
err = sshStorage.CopyFrom(conf.file)
if err != nil {
utils.Fatal("Error copying backup file: %s", err)
}
RestoreDatabase(db, conf)
}
func restoreFromFTP(db *dbConfig, conf *RestoreConfig) {
utils.Info("Restore database from FTP server")
ftpConfig := loadFtpConfig()
ftpStorage, err := ftp.NewStorage(ftp.Config{
Host: ftpConfig.host,
Port: ftpConfig.port,
User: ftpConfig.user,
Password: ftpConfig.password,
RemotePath: conf.remotePath,
LocalPath: tmpPath,
})
if err != nil {
utils.Fatal("Error creating SSH storage: %s", err)
}
err = ftpStorage.CopyFrom(conf.file)
if err != nil {
utils.Fatal("Error copying backup file: %s", err)
}
RestoreDatabase(db, conf)
}
// RestoreDatabase restore database
func RestoreDatabase(db *dbConfig, conf *RestoreConfig) {
if conf.file == "" {
utils.Fatal("Error, file required")
}
extension := filepath.Ext(filepath.Join(tmpPath, conf.file))
rFile, err := os.ReadFile(filepath.Join(tmpPath, conf.file))
outputFile := RemoveLastExtension(filepath.Join(tmpPath, conf.file))
if err != nil {
utils.Fatal("Error reading backup file: %s ", err)
}
if extension == ".gpg" {
if conf.usingKey {
utils.Info("Decrypting backup using private key...")
utils.Warn("Backup decryption using a private key is not fully supported")
prKey, err := os.ReadFile(conf.privateKey)
if err != nil {
utils.Fatal("Error reading public key: %s ", err)
}
err = encryptor.DecryptWithPrivateKey(rFile, outputFile, prKey, conf.passphrase)
if err != nil {
utils.Fatal("error during decrypting backup %v", err)
}
utils.Info("Decrypting backup using private key...done")
} else {
if conf.passphrase == "" {
utils.Error("Error, passphrase or private key required")
utils.Fatal("Your file seems to be a GPG file.\nYou need to provide GPG keys. GPG_PASSPHRASE or GPG_PRIVATE_KEY environment variable is required.")
} else {
utils.Info("Decrypting backup using passphrase...")
//decryptWithGPG file
err := encryptor.Decrypt(rFile, outputFile, conf.passphrase)
if err != nil {
utils.Fatal("Error decrypting file %s %v", file, err)
}
utils.Info("Decrypting backup using passphrase...done")
//Update file name
conf.file = RemoveLastExtension(file)
}
}
}
if utils.FileExists(filepath.Join(tmpPath, conf.file)) {
err := os.Setenv("MYSQL_PWD", db.dbPassword)
if err != nil {
return
}
testDatabaseConnection(db)
utils.Info("Restoring database...")
extension := filepath.Ext(filepath.Join(tmpPath, conf.file))
// Restore from compressed file / .sql.gz
if extension == ".gz" {
str := "zcat " + filepath.Join(tmpPath, conf.file) + " | mysql -h " + db.dbHost + " -P " + db.dbPort + " -u " + db.dbUserName + " " + db.dbName
_, err := exec.Command("sh", "-c", str).Output()
if err != nil {
utils.Fatal("Error, in restoring the database %v", err)
}
utils.Info("Restoring database... done")
utils.Info("Database has been restored")
//Delete temp
deleteTemp()
} else if extension == ".sql" {
//Restore from sql file
str := "cat " + filepath.Join(tmpPath, conf.file) + " | mysql -h " + db.dbHost + " -P " + db.dbPort + " -u " + db.dbUserName + " " + db.dbName
_, err := exec.Command("sh", "-c", str).Output()
if err != nil {
utils.Fatal("Error in restoring the database %v", err)
}
utils.Info("Restoring database... done")
utils.Info("Database has been restored")
//Delete temp
deleteTemp()
} else {
utils.Fatal("Unknown file extension %s", extension)
}
} else {
utils.Fatal("File not found in %s", filepath.Join(tmpPath, conf.file))
}
}

67
pkg/var.go
View File

@@ -1,67 +0,0 @@
// Package pkg /
/*****
@author Jonas Kaninda
@license MIT License <https://opensource.org/licenses/MIT>
@Copyright © 2024 Jonas Kaninda
**/
package pkg
const tmpPath = "/tmp/backup"
const gpgHome = "/config/gnupg"
const gpgExtension = "gpg"
const timeFormat = "2006-01-02 at 15:04:05"
var (
storage = "local"
file = ""
storagePath = "/backup"
workingDir = "/config"
disableCompression = false
encryption = false
usingKey = false
backupSize int64 = 0
startTime string
)
// dbHVars Required environment variables for database
var dbHVars = []string{
"DB_HOST",
"DB_PORT",
"DB_PASSWORD",
"DB_USERNAME",
"DB_NAME",
}
var tdbRVars = []string{
"TARGET_DB_HOST",
"TARGET_DB_PORT",
"TARGET_DB_NAME",
"TARGET_DB_USERNAME",
"TARGET_DB_PASSWORD",
}
var dbConf *dbConfig
var targetDbConf *targetDbConfig
// sshVars Required environment variables for SSH remote server storage
var sshVars = []string{
"SSH_USER",
"SSH_HOST_NAME",
"SSH_PORT",
"REMOTE_PATH",
}
var ftpVars = []string{
"FTP_HOST_NAME",
"FTP_USER",
"FTP_PASSWORD",
"FTP_PORT",
}
// AwsVars Required environment variables for AWS S3 storage
var awsVars = []string{
"AWS_S3_ENDPOINT",
"AWS_S3_BUCKET_NAME",
"AWS_ACCESS_KEY",
"AWS_SECRET_KEY",
"AWS_REGION",
}