feat: add backup all databases

cmd/backup.go

@@ -50,7 +50,7 @@ func init() {
 	BackupCmd.PersistentFlags().StringP("cron-expression", "e", "", "Backup cron expression (e.g., `0 0 * * *` or `@daily`)")
 	BackupCmd.PersistentFlags().StringP("config", "c", "", "Configuration file for multi database backup. (e.g: `/backup/config.yaml`)")
 	BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression")
-	BackupCmd.PersistentFlags().BoolP("all", "a", false, "Backup all databases")
+	BackupCmd.PersistentFlags().BoolP("all-databases", "a", false, "Backup all databases")
-	BackupCmd.PersistentFlags().BoolP("single-file", "", false, "Backup all databases in a single file")
+	BackupCmd.PersistentFlags().BoolP("all-in-one", "A", false, "Backup all databases in a single file")
 
 }

docs/how-tos/backup-all.md

@@ -0,0 +1,61 @@
+---
+title: Backup all databases in the server
+layout: default
+parent: How Tos
+nav_order: 12
+---
+
+# Backup All Databases
+
+MySQL-Bkup supports backing up all databases on the server using the `--all-databases` (`-a`) flag. By default, this creates separate backup files for each database. If you prefer a single backup file, you can use the `--all-in-on`e (`-A`) flag.
+
+Backing up all databases is useful for creating a snapshot of the entire database server, whether for disaster recovery or migration purposes.
+## Backup Modes
+
+### Separate Backup Files (Default)
+
+Using --all-databases without --all-in-one creates individual backup files for each database.
+
+- Creates separate backup files for each database.
+- Provides more flexibility in restoring individual databases or tables.
+- Can be more manageable in cases where different databases have different retention policies.
+- Might take slightly longer due to multiple file operations.
+- It is the default behavior when using the `--all-databases` flag.
+- It does not backup system databases (`information_schema`, `performance_schema`, `mysql`, `sys`, `innodb`).
+
+**Command:**
+
+```bash
+docker run --rm --network your_network_name \
+  -v $PWD/backup:/backup/ \
+  -e "DB_HOST=dbhost" \
+  -e "DB_PORT=3306" \
+  -e "DB_USERNAME=username" \
+  -e "DB_PASSWORD=password" \
+  jkaninda/mysql-bkup backup --all-databases
+```
+### Single Backup File
+
+Using --all-in-one (-A) creates a single backup file containing all databases.
+
+- Creates a single backup file containing all databases.
+- Easier to manage if you need to restore everything at once.
+- Faster to back up and restore in bulk.
+- Can be problematic if you only need to restore a specific database or table.
+- It is recommended to use this option for disaster recovery purposes.
+- It backups system databases as well.
+
+```bash
+docker run --rm --network your_network_name \
+  -v $PWD/backup:/backup/ \
+  -e "DB_HOST=dbhost" \
+  -e "DB_PORT=3306" \
+  -e "DB_USERNAME=username" \
+  -e "DB_PASSWORD=password" \
+  jkaninda/mysql-bkup backup --all-in-one
+```
+
+### When to Use Which?
+
+- Use `--all-in-one` if you want a quick, simple backup for disaster recovery where you'll restore everything at once.
+- Use `--all-databases` if you need granularity in restoring specific databases or tables without affecting others.

docs/how-tos/mutli-backup.md

@@ -34,8 +34,8 @@ Below is an example configuration file (`config.yaml`) that defines multiple dat
 ```yaml
 # Optional: Define a global cron expression for scheduled backups.
 # Example: "@every 20m" (runs every 20 minutes). If omitted, backups run immediately.
-cronExpression: ""
+cronExpression: "" # Optional: Define a global cron expression for scheduled backups.
-
+backupRescueMode: false # Optional: Set to true to enable rescue mode for backups.
 databases:
   - host: mysql1       # Optional: Overrides DB_HOST or uses DB_HOST_DATABASE1.
     port: 3306            # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_DATABASE1.

docs/how-tos/receive-notification.md

@@ -2,7 +2,7 @@
 title: Receive notifications
 layout: default
 parent: How Tos
-nav_order: 12
+nav_order: 13
 ---
 
 # Receive Notifications

docs/reference/index.md

@@ -6,28 +6,30 @@ nav_order: 3
 
 # Configuration Reference
 
-Backup, restore, and migration targets, schedules, and retention policies are configured using **environment variables** or **CLI flags**.
+MySQL backup, restore, and migration processes can be configured using **environment variables** or **CLI flags**.
-
----
 
 ## CLI Utility Usage
 
-| Option                  | Short Flag | Description                                                                   |
+The `mysql-bkup` CLI provides commands and options to manage MySQL backups efficiently.
-|-------------------------|------------|-------------------------------------------------------------------------------|
+
-| `pg-bkup`               | `bkup`     | CLI utility for managing PostgreSQL backups.                                  |
+| Option                  | Short Flag | Description                                                                             |
-| `backup`                |            | Perform a backup operation.                                                   |
+|-------------------------|------------|-----------------------------------------------------------------------------------------|
-| `restore`               |            | Perform a restore operation.                                                  |
+| `mysql-bkup`            | `bkup`     | CLI tool for managing MySQL backups, restoration, and migration.                        |
-| `migrate`               |            | Migrate a database from one instance to another.                              |
+| `backup`                |            | Executes a backup operation.                                                            |
-| `--storage`             | `-s`       | Storage type (`local`, `s3`, `ssh`, etc.). Default: `local`.                  |
+| `restore`               |            | Restores a database from a backup file.                                                 |
-| `--file`                | `-f`       | File name for restoration.                                                    |
+| `migrate`               |            | Migrates a database from one instance to another.                                       |
-| `--path`                |            | Path for storage (e.g., `/custom_path` for S3 or `/home/foo/backup` for SSH). |
+| `--storage`             | `-s`       | Specifies the storage type (`local`, `s3`, `ssh`, etc.). Default: `local`.              |
-| `--config`              | `-c`       | Configuration file for multi database backup. (e.g: `/backup/config.yaml`).   |
+| `--file`                | `-f`       | Defines the backup file name for restoration.                                           |
-| `--dbname`              | `-d`       | Database name.                                                                |
+| `--path`                |            | Sets the storage path (e.g., `/custom_path` for S3 or `/home/foo/backup` for SSH).      |
-| `--port`                | `-p`       | Database port. Default: `3306`.                                               |
+| `--config`              | `-c`       | Provides a configuration file for multi-database backups (e.g., `/backup/config.yaml`). |
-| `--disable-compression` |            | Disable compression for database backups.                                     |
+| `--dbname`              | `-d`       | Specifies the database name to back up or restore.                                      |
-| `--cron-expression`     | `-e`       | Cron expression for scheduled backups (e.g., `0 0 * * *` or `@daily`).        |
+| `--port`                | `-p`       | Defines the database port. Default: `3306`.                                             |
-| `--help`                | `-h`       | Display help message and exit.                                                |
+| `--disable-compression` |            | Disables compression for database backups.                                              |
-| `--version`             | `-V`       | Display version information and exit.                                         |
+| `--cron-expression`     | `-e`       | Schedules backups using a cron expression (e.g., `0 0 * * *` or `@daily`).              |
+| `--all-databases`       | `-a`       | Backs up all databases separately (e.g., `backup --all-databases`).                     |
+| `--all-in-one`          | `-A`       | Backs up all databases in a single file (e.g., `backup --all-databases --single-file`). |
+| `--help`                | `-h`       | Displays the help message and exits.                                                    |
+| `--version`             | `-V`       | Shows version information and exits.                                                    |
 
 ---
 
@@ -40,6 +42,8 @@ Backup, restore, and migration targets, schedules, and retention policies are co
 | `DB_NAME`                      | Optional (if provided via `-d` flag) | Database name.                                                             |
 | `DB_USERNAME`                  | Required                             | Database username.                                                         |
 | `DB_PASSWORD`                  | Required                             | Database password.                                                         |
+| `DB_SSL_CA`                    | Optional                             | Database client CA certificate file                                        |
+| `DB_SSL_MODE`                  | Optional(`0 or 1`) default: `0`      | Database client Enable CA validation                                       |
 | `AWS_ACCESS_KEY`               | Required for S3 storage              | AWS S3 Access Key.                                                         |
 | `AWS_SECRET_KEY`               | Required for S3 storage              | AWS S3 Secret Key.                                                         |
 | `AWS_BUCKET_NAME`              | Required for S3 storage              | AWS S3 Bucket Name.                                                        |

pkg/azure.go

@@ -39,7 +39,7 @@ func azureBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Azure Blob Storage")
 
 	// Backup database
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return

pkg/backup.go

@@ -112,7 +112,7 @@ func multiBackupTask(databases []Database, bkConfig *BackupConfig) {
 
 // createBackupTask backup task
 func createBackupTask(db *dbConfig, config *BackupConfig) {
-	if config.all && !config.singleFile {
+	if config.all && !config.allInOne {
 		backupAll(db, config)
 	} else {
 		backupTask(db, config)
@@ -141,7 +141,7 @@ func backupTask(db *dbConfig, config *BackupConfig) {
 	utils.Info("Starting backup task...")
 	startTime = time.Now()
 	prefix := db.dbName
-	if config.all && config.singleFile {
+	if config.all && config.allInOne {
 		prefix = "all_databases"
 	}
 	// Generate file name
@@ -302,7 +302,7 @@ func runCommandWithCompression(command string, args []string, outputPath string)
 // localBackup backup database to local storage
 func localBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to local storage")
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return

pkg/config.go

@@ -78,7 +78,7 @@ type BackupConfig struct {
 	storage            string
 	cronExpression     string
 	all                bool
-	singleFile         bool
+	allInOne           bool
 }
 type FTPConfig struct {
 	host       string
@@ -253,13 +253,17 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig {
 	remotePath := utils.GetEnvVariable("REMOTE_PATH", "SSH_REMOTE_PATH")
 	storage = utils.GetEnv(cmd, "storage", "STORAGE")
 	prune := false
+	configFile := os.Getenv("BACKUP_CONFIG_FILE")
 	backupRetention := utils.GetIntEnv("BACKUP_RETENTION_DAYS")
 	if backupRetention > 0 {
 		prune = true
 	}
 	disableCompression, _ = cmd.Flags().GetBool("disable-compression")
-	all, _ := cmd.Flags().GetBool("all")
+	all, _ := cmd.Flags().GetBool("all-databases")
-	singleFile, _ := cmd.Flags().GetBool("single-file")
+	allInOne, _ := cmd.Flags().GetBool("all-in-one")
+	if allInOne {
+		all = true
+	}
 	_, _ = cmd.Flags().GetString("mode")
 	passphrase := os.Getenv("GPG_PASSPHRASE")
 	_ = utils.GetEnv(cmd, "path", "AWS_S3_PATH")
@@ -273,6 +277,10 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig {
 		encryption = true
 		usingKey = false
 	}
+	dbName := os.Getenv("DB_NAME")
+	if dbName == "" && !all && configFile == "" {
+		utils.Fatal("Database name is required, use DB_NAME environment variable or -d flag")
+	}
 	// Initialize backup configs
 	config := BackupConfig{}
 	config.backupRetention = backupRetention
@@ -286,7 +294,7 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig {
 	config.usingKey = usingKey
 	config.cronExpression = cronExpression
 	config.all = all
-	config.singleFile = singleFile
+	config.allInOne = allInOne
 	return &config
 }
 

pkg/helper.go

@@ -37,7 +37,7 @@ import (
 )
 
 func intro() {
-	fmt.Println("Starting MySQL Backup...")
+	fmt.Println("Starting MYSQL-BKUP...")
 	fmt.Printf("Version: %s\n", utils.Version)
 	fmt.Println("Copyright (c) 2024 Jonas Kaninda")
 }

pkg/remote.go

@@ -39,7 +39,7 @@ import (
 func sshBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Remote server")
 	// Backup database
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return
@@ -160,7 +160,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to the remote FTP server")
 
 	// Backup database
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return

pkg/restore.go

@@ -57,11 +57,17 @@ func StartRestore(cmd *cobra.Command) {
 }
 func localRestore(dbConf *dbConfig, restoreConf *RestoreConfig) {
 	utils.Info("Restore database from local")
+	basePath := filepath.Dir(restoreConf.file)
+	fileName := filepath.Base(restoreConf.file)
+	restoreConf.file = fileName
+	if basePath == "" || basePath == "." {
+		basePath = storagePath
+	}
 	localStorage := local.NewStorage(local.Config{
-		RemotePath: storagePath,
+		RemotePath: basePath,
 		LocalPath:  tmpPath,
 	})
-	err := localStorage.CopyFrom(restoreConf.file)
+	err := localStorage.CopyFrom(fileName)
 	if err != nil {
 		utils.Fatal("Error copying backup file: %s", err)
 	}
@@ -69,87 +75,79 @@ func localRestore(dbConf *dbConfig, restoreConf *RestoreConfig) {
 
 }
 
-// RestoreDatabase restore database
+// RestoreDatabase restores the database from a backup file
 func RestoreDatabase(db *dbConfig, conf *RestoreConfig) {
 	if conf.file == "" {
 		utils.Fatal("Error, file required")
 	}
-	extension := filepath.Ext(filepath.Join(tmpPath, conf.file))
+
-	rFile, err := os.ReadFile(filepath.Join(tmpPath, conf.file))
+	filePath := filepath.Join(tmpPath, conf.file)
-	outputFile := RemoveLastExtension(filepath.Join(tmpPath, conf.file))
+	rFile, err := os.ReadFile(filePath)
 	if err != nil {
-		utils.Fatal("Error reading backup file: %s ", err)
+		utils.Fatal("Error reading backup file: %v", err)
 	}
 
+	extension := filepath.Ext(filePath)
+	outputFile := RemoveLastExtension(filePath)
+
 	if extension == ".gpg" {
-
+		decryptBackup(conf, rFile, outputFile)
-		if conf.usingKey {
-			utils.Info("Decrypting backup using private key...")
-			utils.Warn("Backup decryption using a private key is not fully supported")
-			prKey, err := os.ReadFile(conf.privateKey)
-			if err != nil {
-				utils.Fatal("Error reading public key: %s ", err)
-			}
-			err = encryptor.DecryptWithPrivateKey(rFile, outputFile, prKey, conf.passphrase)
-			if err != nil {
-				utils.Fatal("error during decrypting backup %v", err)
-			}
-			utils.Info("Decrypting backup using private key...done")
-		} else {
-			if conf.passphrase == "" {
-				utils.Error("Error, passphrase or private key required")
-				utils.Fatal("Your file seems to be a GPG file.\nYou need to provide GPG keys. GPG_PASSPHRASE or GPG_PRIVATE_KEY environment variable is required.")
-			} else {
-				utils.Info("Decrypting backup using passphrase...")
-				// decryptWithGPG file
-				err := encryptor.Decrypt(rFile, outputFile, conf.passphrase)
-				if err != nil {
-					utils.Fatal("Error decrypting file %s %v", file, err)
-				}
-				utils.Info("Decrypting backup using passphrase...done")
-				// Update file name
-				conf.file = RemoveLastExtension(file)
-			}
-		}
-
 	}
 
-	if utils.FileExists(filepath.Join(tmpPath, conf.file)) {
+	restorationFile := filepath.Join(tmpPath, conf.file)
-		err = testDatabaseConnection(db)
+	if !utils.FileExists(restorationFile) {
+		utils.Fatal("File not found: %s", restorationFile)
+	}
+
+	if err := testDatabaseConnection(db); err != nil {
+		utils.Fatal("Error connecting to the database: %v", err)
+	}
+
+	utils.Info("Restoring database...")
+	restoreDatabaseFile(db, restorationFile)
+}
+
+func decryptBackup(conf *RestoreConfig, rFile []byte, outputFile string) {
+	if conf.usingKey {
+		utils.Info("Decrypting backup using private key...")
+		prKey, err := os.ReadFile(conf.privateKey)
 		if err != nil {
-			utils.Fatal("Error connecting to the database %v", err)
+			utils.Fatal("Error reading private key: %v", err)
 		}
-		utils.Info("Restoring database...")
+		if err := encryptor.DecryptWithPrivateKey(rFile, outputFile, prKey, conf.passphrase); err != nil {
-
+			utils.Fatal("Error decrypting backup: %v", err)
-		extension := filepath.Ext(filepath.Join(tmpPath, conf.file))
-		// Restore from a compressed file / .sql.gz
-		if extension == ".gz" {
-			str := "zcat " + filepath.Join(tmpPath, conf.file) + " | mariadb " + fmt.Sprintf("--defaults-file=%s", mysqlClientConfig) + " " + db.dbName
-			output, err := exec.Command("sh", "-c", str).Output()
-			if err != nil {
-				utils.Fatal("Error, in restoring the database  %v output: %v", err, output)
-			}
-			utils.Info("Restoring database... done")
-			utils.Info("Database has been restored")
-			// Delete temp
-			deleteTemp()
-
-		} else if extension == ".sql" {
-			// Restore from SQL file
-			str := "cat " + filepath.Join(tmpPath, conf.file) + " | mariadb " + fmt.Sprintf("--defaults-file=%s", mysqlClientConfig) + " " + db.dbName
-			output, err := exec.Command("sh", "-c", str).Output()
-			if err != nil {
-				utils.Fatal("Error, in restoring the database  %v output: %v", err, output)
-			}
-			utils.Info("Restoring database... done")
-			utils.Info("Database has been restored")
-			// Delete temp
-			deleteTemp()
-		} else {
-			utils.Fatal("Unknown file extension %s", extension)
 		}
-
 	} else {
-		utils.Fatal("File not found in %s", filepath.Join(tmpPath, conf.file))
+		if conf.passphrase == "" {
+			utils.Fatal("Passphrase or private key required for GPG file.")
+		}
+		utils.Info("Decrypting backup using passphrase...")
+		if err := encryptor.Decrypt(rFile, outputFile, conf.passphrase); err != nil {
+			utils.Fatal("Error decrypting file: %v", err)
+		}
+		conf.file = RemoveLastExtension(conf.file)
 	}
 }
+
+func restoreDatabaseFile(db *dbConfig, restorationFile string) {
+	extension := filepath.Ext(restorationFile)
+	var cmdStr string
+
+	switch extension {
+	case ".gz":
+		cmdStr = fmt.Sprintf("zcat %s | mariadb --defaults-file=%s %s", restorationFile, mysqlClientConfig, db.dbName)
+	case ".sql":
+		cmdStr = fmt.Sprintf("cat %s | mariadb --defaults-file=%s %s", restorationFile, mysqlClientConfig, db.dbName)
+	default:
+		utils.Fatal("Unknown file extension: %s", extension)
+	}
+
+	cmd := exec.Command("sh", "-c", cmdStr)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		utils.Fatal("Error restoring database: %v\nOutput: %s", err, string(output))
+	}
+
+	utils.Info("Database has been restored successfully.")
+	deleteTemp()
+}

pkg/s3.go

@@ -39,7 +39,7 @@ func s3Backup(db *dbConfig, config *BackupConfig) {
 
 	utils.Info("Backup database to s3 storage")
 	// Backup database
-	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile)
+	err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne)
 	if err != nil {
 		recoverMode(err, "Error backing up database")
 		return

pkg/var.go

@@ -54,7 +54,6 @@ var dbHVars = []string{
 	"DB_HOST",
 	"DB_PASSWORD",
 	"DB_USERNAME",
-	"DB_NAME",
 }
 var tdbRVars = []string{
 	"TARGET_DB_HOST",