From fc028a2c553a33144de6572ead284357e0fe661b Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Sun, 26 Jan 2025 13:43:39 +0100 Subject: [PATCH 1/3] feat: add multiple backup rescued mode for scheduled mode --- pkg/azure.go | 6 +++- pkg/backup.go | 59 +++++++++++++++++++++++++++++--------- pkg/config.go | 5 ++-- pkg/helper.go | 8 +++--- pkg/remote.go | 12 ++++++-- pkg/s3.go | 6 +++- pkg/var.go | 1 + templates/email-error.tmpl | 4 +-- 8 files changed, 75 insertions(+), 26 deletions(-) diff --git a/pkg/azure.go b/pkg/azure.go index 64f7048..7a0c602 100644 --- a/pkg/azure.go +++ b/pkg/azure.go @@ -39,7 +39,11 @@ func azureBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Azure Blob Storage") // Backup database - BackupDatabase(db, config.backupFileName, disableCompression) + err := BackupDatabase(db, config.backupFileName, disableCompression) + if err != nil { + recoverMode(err, "Error backing up database") + return + } finalFileName := config.backupFileName if config.encryption { encryptBackup(config) diff --git a/pkg/backup.go b/pkg/backup.go index 0836021..b52c566 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -72,13 +72,17 @@ func scheduledMode(db *dbConfig, config *BackupConfig) { // Test backup utils.Info("Testing backup configurations...") - testDatabaseConnection(db) + err := testDatabaseConnection(db) + if err != nil { + utils.Error("Error connecting to database: %s", db.dbName) + utils.Fatal("Error: %s", err) + } utils.Info("Testing backup configurations...done") utils.Info("Creating backup job...") // Create a new cron instance c := cron.New() - _, err := c.AddFunc(config.cronExpression, func() { + _, err = c.AddFunc(config.cronExpression, func() { BackupTask(db, config) utils.Info("Next backup time is: %v", utils.CronNextTime(config.cronExpression).Format(timeFormat)) @@ -147,6 +151,7 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) { if bkConfig.cronExpression == "" { multiBackupTask(conf.Databases, bkConfig) } else { + backupRescueMode = conf.BackupRescueMode // Check if cronExpression is valid if utils.IsValidCronExpression(bkConfig.cronExpression) { utils.Info("Running backup in Scheduled mode") @@ -157,7 +162,11 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) { // Test backup utils.Info("Testing backup configurations...") for _, db := range conf.Databases { - testDatabaseConnection(getDatabase(db)) + err = testDatabaseConnection(getDatabase(db)) + if err != nil { + recoverMode(err, fmt.Sprintf("Error connecting to database: %s", db.Name)) + continue + } } utils.Info("Testing backup configurations...done") utils.Info("Creating backup job...") @@ -187,16 +196,19 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) { } // BackupDatabase backup database -func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool) { +func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool) error { storagePath = os.Getenv("STORAGE_PATH") utils.Info("Starting database backup...") err := os.Setenv("MYSQL_PWD", db.dbPassword) if err != nil { - return + return fmt.Errorf("failed to set MYSQL_PWD environment variable: %v", err) + } + err = testDatabaseConnection(db) + if err != nil { + return fmt.Errorf(err.Error()) } - testDatabaseConnection(db) // Backup Database database utils.Info("Backing up database...") @@ -211,24 +223,24 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool ) output, err := cmd.Output() if err != nil { - utils.Fatal(err.Error()) + return fmt.Errorf("failed to backup database: %v", err) } // save output file, err := os.Create(filepath.Join(tmpPath, backupFileName)) if err != nil { - utils.Fatal(err.Error()) + return fmt.Errorf("failed to create backup file: %v", err) } defer func(file *os.File) { err := file.Close() if err != nil { - utils.Fatal(err.Error()) + return } }(file) _, err = file.Write(output) if err != nil { - utils.Fatal(err.Error()) + return err } utils.Info("Database has been backed up") @@ -237,14 +249,14 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool cmd := exec.Command("mysqldump", "-h", db.dbHost, "-P", db.dbPort, "-u", db.dbUserName, db.dbName) stdout, err := cmd.StdoutPipe() if err != nil { - log.Fatal(err) + return fmt.Errorf("failed to backup database: %v", err) } gzipCmd := exec.Command("gzip") gzipCmd.Stdin = stdout gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName)) err = gzipCmd.Start() if err != nil { - return + return fmt.Errorf("failed to backup database: %v", err) } if err := cmd.Run(); err != nil { log.Fatal(err) @@ -252,13 +264,18 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool if err := gzipCmd.Wait(); err != nil { log.Fatal(err) } - utils.Info("Database has been backed up") } + utils.Info("Database has been backed up") + return nil } func localBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to local storage") - BackupDatabase(db, config.backupFileName, disableCompression) + err := BackupDatabase(db, config.backupFileName, disableCompression) + if err != nil { + recoverMode(err, "Error backing up database") + return + } finalFileName := config.backupFileName if config.encryption { encryptBackup(config) @@ -333,3 +350,17 @@ func encryptBackup(config *BackupConfig) { } } +func recoverMode(err error, msg string) { + if err != nil { + if backupRescueMode { + utils.NotifyError(fmt.Sprintf("%s : %v", msg, err)) + utils.Error(msg) + utils.Error("Backup rescue mode is enabled") + utils.Error("Backup will continue") + } else { + utils.Error(msg) + utils.Fatal("Error: %v", err) + } + } + +} diff --git a/pkg/config.go b/pkg/config.go index 726a9b6..349d6c6 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -42,8 +42,9 @@ type Database struct { Path string `yaml:"path"` } type Config struct { - Databases []Database `yaml:"databases"` - CronExpression string `yaml:"cronExpression"` + CronExpression string `yaml:"cronExpression"` + BackupRescueMode bool `yaml:"backupRescueMode"` + Databases []Database `yaml:"databases"` } type dbConfig struct { diff --git a/pkg/helper.go b/pkg/helper.go index 0fc86e4..4ca5030 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -66,10 +66,10 @@ func deleteTemp() { } // TestDatabaseConnection tests the database connection -func testDatabaseConnection(db *dbConfig) { +func testDatabaseConnection(db *dbConfig) error { err := os.Setenv("MYSQL_PWD", db.dbPassword) if err != nil { - return + return fmt.Errorf("failed to set MYSQL_PWD environment variable: %v", err) } utils.Info("Connecting to %s database ...", db.dbName) // Set database name for notification error @@ -81,11 +81,11 @@ func testDatabaseConnection(db *dbConfig) { cmd.Stderr = &out err = cmd.Run() if err != nil { - utils.Fatal("Error testing database connection: %v\nOutput: %s", err, out.String()) + return fmt.Errorf("failed to connect to %s database: %v", db.dbName, err) } utils.Info("Successfully connected to %s database", db.dbName) - + return nil } // checkPubKeyFile checks gpg public key diff --git a/pkg/remote.go b/pkg/remote.go index 0e126a5..d01069d 100644 --- a/pkg/remote.go +++ b/pkg/remote.go @@ -39,7 +39,11 @@ import ( func sshBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Remote server") // Backup database - BackupDatabase(db, config.backupFileName, disableCompression) + err := BackupDatabase(db, config.backupFileName, disableCompression) + if err != nil { + recoverMode(err, "Error backing up database") + return + } finalFileName := config.backupFileName if config.encryption { encryptBackup(config) @@ -156,7 +160,11 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to the remote FTP server") // Backup database - BackupDatabase(db, config.backupFileName, disableCompression) + err := BackupDatabase(db, config.backupFileName, disableCompression) + if err != nil { + recoverMode(err, "Error backing up database") + return + } finalFileName := config.backupFileName if config.encryption { encryptBackup(config) diff --git a/pkg/s3.go b/pkg/s3.go index 4f903a5..90822b9 100644 --- a/pkg/s3.go +++ b/pkg/s3.go @@ -39,7 +39,11 @@ func s3Backup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to s3 storage") // Backup database - BackupDatabase(db, config.backupFileName, disableCompression) + err := BackupDatabase(db, config.backupFileName, disableCompression) + if err != nil { + recoverMode(err, "Error backing up database") + return + } finalFileName := config.backupFileName if config.encryption { encryptBackup(config) diff --git a/pkg/var.go b/pkg/var.go index 4b8327d..4da2c8b 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -42,6 +42,7 @@ var ( usingKey = false backupSize int64 = 0 startTime = time.Now() + backupRescueMode = false ) // dbHVars Required environment variables for database diff --git a/templates/email-error.tmpl b/templates/email-error.tmpl index 8919241..7dc2fbb 100644 --- a/templates/email-error.tmpl +++ b/templates/email-error.tmpl @@ -60,10 +60,10 @@

We recommend investigating the issue as soon as possible to prevent potential data loss or service disruptions.

-

For more information, visit the pg-bkup documentation.

+

For more information, visit the mysql-bkup documentation.

From 75b809511ea61bea0db4464314f22b26088289e1 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Sun, 26 Jan 2025 13:54:41 +0100 Subject: [PATCH 2/3] fix go lint --- .golangci.yml | 1 + cmd/backup.go | 2 +- cmd/restore.go | 2 +- cmd/root.go | 1 - pkg/backup.go | 6 +++--- pkg/migrate.go | 5 ++++- pkg/restore.go | 5 ++++- pkg/var.go | 7 ------- 8 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index ddaa1aa..579881a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -27,6 +27,7 @@ linters: - gosimple - govet - ineffassign + # - lll - misspell - nakedret - prealloc diff --git a/cmd/backup.go b/cmd/backup.go index f1545a4..a6b2f13 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -44,7 +44,7 @@ var BackupCmd = &cobra.Command{ } func init() { - //Backup + // Backup BackupCmd.PersistentFlags().StringP("storage", "s", "local", "Define storage: local, s3, ssh, ftp, azure") BackupCmd.PersistentFlags().StringP("path", "P", "", "Storage path without file name. e.g: /custom_path or ssh remote path `/home/foo/backup`") BackupCmd.PersistentFlags().StringP("cron-expression", "e", "", "Backup cron expression (e.g., `0 0 * * *` or `@daily`)") diff --git a/cmd/restore.go b/cmd/restore.go index c7fca9f..e0fc68b 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -46,7 +46,7 @@ var RestoreCmd = &cobra.Command{ } func init() { - //Restore + // Restore RestoreCmd.PersistentFlags().StringP("file", "f", "", "File name of database") RestoreCmd.PersistentFlags().StringP("storage", "s", "local", "Define storage: local, s3, ssh, ftp") RestoreCmd.PersistentFlags().StringP("path", "P", "", "AWS S3 path without file name. eg: /custom_path or ssh remote path `/home/foo/backup`") diff --git a/cmd/root.go b/cmd/root.go index c8400bd..cd32128 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -38,7 +38,6 @@ var rootCmd = &cobra.Command{ Example: utils.MainExample, Version: appVersion, } -var operation = "" // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. diff --git a/pkg/backup.go b/pkg/backup.go index b52c566..f0a4144 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -207,7 +207,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool } err = testDatabaseConnection(db) if err != nil { - return fmt.Errorf(err.Error()) + return fmt.Errorf("failed to connect to the database: %v", err) } // Backup Database database utils.Info("Backing up database...") @@ -354,11 +354,11 @@ func recoverMode(err error, msg string) { if err != nil { if backupRescueMode { utils.NotifyError(fmt.Sprintf("%s : %v", msg, err)) - utils.Error(msg) + utils.Error("Error: %s", msg) utils.Error("Backup rescue mode is enabled") utils.Error("Backup will continue") } else { - utils.Error(msg) + utils.Error("Error: %s", msg) utils.Fatal("Error: %v", err) } } diff --git a/pkg/migrate.go b/pkg/migrate.go index ac77a95..1edfd3e 100644 --- a/pkg/migrate.go +++ b/pkg/migrate.go @@ -51,7 +51,10 @@ func StartMigration(cmd *cobra.Command) { conf := &RestoreConfig{} conf.file = backupFileName // Backup source Database - BackupDatabase(dbConf, backupFileName, true) + err := BackupDatabase(dbConf, backupFileName, true) + if err != nil { + utils.Fatal("Error backing up database: %s", err) + } // Restore source database into target database utils.Info("Restoring [%s] database into [%s] database...", dbConf.dbName, targetDbConf.targetDbName) RestoreDatabase(&newDbConfig, conf) diff --git a/pkg/restore.go b/pkg/restore.go index 611e6ae..1dfba20 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -118,7 +118,10 @@ func RestoreDatabase(db *dbConfig, conf *RestoreConfig) { if err != nil { return } - testDatabaseConnection(db) + err = testDatabaseConnection(db) + if err != nil { + utils.Fatal("Error connecting to the database %v", err) + } utils.Info("Restoring database...") extension := filepath.Ext(filepath.Join(tmpPath, conf.file)) diff --git a/pkg/var.go b/pkg/var.go index 4da2c8b..24a44df 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -62,13 +62,6 @@ var tdbRVars = []string{ var dbConf *dbConfig var targetDbConf *targetDbConfig -// sshVars Required environment variables for SSH remote server storage -var sshVars = []string{ - "SSH_USER", - "SSH_HOST_NAME", - "SSH_PORT", - "REMOTE_PATH", -} var ftpVars = []string{ "FTP_HOST_NAME", "FTP_USER", From bd65db24182ede90a8e1849409e635543e5e3116 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Wed, 5 Feb 2025 07:39:52 +0100 Subject: [PATCH 3/3] chore: update helper func to check env with prefix or suffix for multi backups --- pkg/config.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/config.go b/pkg/config.go index 349d6c6..7573c60 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -145,15 +145,26 @@ func getDatabase(database Database) *dbConfig { // Helper function to get environment variable or use a default value func getEnvOrDefault(currentValue, envKey, suffix, defaultValue string) string { + // Return the current value if it's already set if currentValue != "" { return currentValue } + + // Check for suffixed or prefixed environment variables if a suffix is provided if suffix != "" { - envSuffix := os.Getenv(fmt.Sprintf("%s_%s", envKey, strings.ToUpper(suffix))) + suffixUpper := strings.ToUpper(suffix) + envSuffix := os.Getenv(fmt.Sprintf("%s_%s", envKey, suffixUpper)) if envSuffix != "" { return envSuffix } + + envPrefix := os.Getenv(fmt.Sprintf("%s_%s", suffixUpper, envKey)) + if envPrefix != "" { + return envPrefix + } } + + // Fall back to the default value using a helper function return utils.EnvWithDefault(envKey, defaultValue) }