diff --git a/cmd/backup.go b/cmd/backup.go index d792772..4873ceb 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -50,7 +50,7 @@ func init() { BackupCmd.PersistentFlags().StringP("cron-expression", "e", "", "Backup cron expression (e.g., `0 0 * * *` or `@daily`)") BackupCmd.PersistentFlags().StringP("config", "c", "", "Configuration file for multi database backup. (e.g: `/backup/config.yaml`)") BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression") - BackupCmd.PersistentFlags().BoolP("all", "a", false, "Backup all databases") - BackupCmd.PersistentFlags().BoolP("single-file", "", false, "Backup all databases in a single file") + BackupCmd.PersistentFlags().BoolP("all-databases", "a", false, "Backup all databases") + BackupCmd.PersistentFlags().BoolP("all-in-one", "A", false, "Backup all databases in a single file") } diff --git a/docs/how-tos/backup-all.md b/docs/how-tos/backup-all.md new file mode 100644 index 0000000..a1f7355 --- /dev/null +++ b/docs/how-tos/backup-all.md @@ -0,0 +1,61 @@ +--- +title: Backup all databases in the server +layout: default +parent: How Tos +nav_order: 12 +--- + +# Backup All Databases + +MySQL-Bkup supports backing up all databases on the server using the `--all-databases` (`-a`) flag. By default, this creates separate backup files for each database. If you prefer a single backup file, you can use the `--all-in-on`e (`-A`) flag. + +Backing up all databases is useful for creating a snapshot of the entire database server, whether for disaster recovery or migration purposes. +## Backup Modes + +### Separate Backup Files (Default) + +Using --all-databases without --all-in-one creates individual backup files for each database. + +- Creates separate backup files for each database. +- Provides more flexibility in restoring individual databases or tables. +- Can be more manageable in cases where different databases have different retention policies. +- Might take slightly longer due to multiple file operations. +- It is the default behavior when using the `--all-databases` flag. +- It does not backup system databases (`information_schema`, `performance_schema`, `mysql`, `sys`, `innodb`). + +**Command:** + +```bash +docker run --rm --network your_network_name \ + -v $PWD/backup:/backup/ \ + -e "DB_HOST=dbhost" \ + -e "DB_PORT=3306" \ + -e "DB_USERNAME=username" \ + -e "DB_PASSWORD=password" \ + jkaninda/mysql-bkup backup --all-databases +``` +### Single Backup File + +Using --all-in-one (-A) creates a single backup file containing all databases. + +- Creates a single backup file containing all databases. +- Easier to manage if you need to restore everything at once. +- Faster to back up and restore in bulk. +- Can be problematic if you only need to restore a specific database or table. +- It is recommended to use this option for disaster recovery purposes. +- It backups system databases as well. + +```bash +docker run --rm --network your_network_name \ + -v $PWD/backup:/backup/ \ + -e "DB_HOST=dbhost" \ + -e "DB_PORT=3306" \ + -e "DB_USERNAME=username" \ + -e "DB_PASSWORD=password" \ + jkaninda/mysql-bkup backup --all-in-one +``` + +### When to Use Which? + +- Use `--all-in-one` if you want a quick, simple backup for disaster recovery where you'll restore everything at once. +- Use `--all-databases` if you need granularity in restoring specific databases or tables without affecting others. diff --git a/docs/how-tos/mutli-backup.md b/docs/how-tos/mutli-backup.md index b9617ef..49d367c 100644 --- a/docs/how-tos/mutli-backup.md +++ b/docs/how-tos/mutli-backup.md @@ -34,8 +34,8 @@ Below is an example configuration file (`config.yaml`) that defines multiple dat ```yaml # Optional: Define a global cron expression for scheduled backups. # Example: "@every 20m" (runs every 20 minutes). If omitted, backups run immediately. -cronExpression: "" - +cronExpression: "" # Optional: Define a global cron expression for scheduled backups. +backupRescueMode: false # Optional: Set to true to enable rescue mode for backups. databases: - host: mysql1 # Optional: Overrides DB_HOST or uses DB_HOST_DATABASE1. port: 3306 # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_DATABASE1. diff --git a/docs/how-tos/receive-notification.md b/docs/how-tos/receive-notification.md index 3067c92..43fa17f 100644 --- a/docs/how-tos/receive-notification.md +++ b/docs/how-tos/receive-notification.md @@ -2,7 +2,7 @@ title: Receive notifications layout: default parent: How Tos -nav_order: 12 +nav_order: 13 --- # Receive Notifications diff --git a/docs/reference/index.md b/docs/reference/index.md index aa3c81d..55c7134 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -6,28 +6,30 @@ nav_order: 3 # Configuration Reference -Backup, restore, and migration targets, schedules, and retention policies are configured using **environment variables** or **CLI flags**. - ---- +MySQL backup, restore, and migration processes can be configured using **environment variables** or **CLI flags**. ## CLI Utility Usage -| Option | Short Flag | Description | -|-------------------------|------------|-------------------------------------------------------------------------------| -| `pg-bkup` | `bkup` | CLI utility for managing PostgreSQL backups. | -| `backup` | | Perform a backup operation. | -| `restore` | | Perform a restore operation. | -| `migrate` | | Migrate a database from one instance to another. | -| `--storage` | `-s` | Storage type (`local`, `s3`, `ssh`, etc.). Default: `local`. | -| `--file` | `-f` | File name for restoration. | -| `--path` | | Path for storage (e.g., `/custom_path` for S3 or `/home/foo/backup` for SSH). | -| `--config` | `-c` | Configuration file for multi database backup. (e.g: `/backup/config.yaml`). | -| `--dbname` | `-d` | Database name. | -| `--port` | `-p` | Database port. Default: `3306`. | -| `--disable-compression` | | Disable compression for database backups. | -| `--cron-expression` | `-e` | Cron expression for scheduled backups (e.g., `0 0 * * *` or `@daily`). | -| `--help` | `-h` | Display help message and exit. | -| `--version` | `-V` | Display version information and exit. | +The `mysql-bkup` CLI provides commands and options to manage MySQL backups efficiently. + +| Option | Short Flag | Description | +|-------------------------|------------|-----------------------------------------------------------------------------------------| +| `mysql-bkup` | `bkup` | CLI tool for managing MySQL backups, restoration, and migration. | +| `backup` | | Executes a backup operation. | +| `restore` | | Restores a database from a backup file. | +| `migrate` | | Migrates a database from one instance to another. | +| `--storage` | `-s` | Specifies the storage type (`local`, `s3`, `ssh`, etc.). Default: `local`. | +| `--file` | `-f` | Defines the backup file name for restoration. | +| `--path` | | Sets the storage path (e.g., `/custom_path` for S3 or `/home/foo/backup` for SSH). | +| `--config` | `-c` | Provides a configuration file for multi-database backups (e.g., `/backup/config.yaml`). | +| `--dbname` | `-d` | Specifies the database name to back up or restore. | +| `--port` | `-p` | Defines the database port. Default: `3306`. | +| `--disable-compression` | | Disables compression for database backups. | +| `--cron-expression` | `-e` | Schedules backups using a cron expression (e.g., `0 0 * * *` or `@daily`). | +| `--all-databases` | `-a` | Backs up all databases separately (e.g., `backup --all-databases`). | +| `--all-in-one` | `-A` | Backs up all databases in a single file (e.g., `backup --all-databases --single-file`). | +| `--help` | `-h` | Displays the help message and exits. | +| `--version` | `-V` | Shows version information and exits. | --- @@ -40,6 +42,8 @@ Backup, restore, and migration targets, schedules, and retention policies are co | `DB_NAME` | Optional (if provided via `-d` flag) | Database name. | | `DB_USERNAME` | Required | Database username. | | `DB_PASSWORD` | Required | Database password. | +| `DB_SSL_CA` | Optional | Database client CA certificate file | +| `DB_SSL_MODE` | Optional(`0 or 1`) default: `0` | Database client Enable CA validation | | `AWS_ACCESS_KEY` | Required for S3 storage | AWS S3 Access Key. | | `AWS_SECRET_KEY` | Required for S3 storage | AWS S3 Secret Key. | | `AWS_BUCKET_NAME` | Required for S3 storage | AWS S3 Bucket Name. | diff --git a/pkg/azure.go b/pkg/azure.go index 0e61a47..b16edcc 100644 --- a/pkg/azure.go +++ b/pkg/azure.go @@ -39,7 +39,7 @@ func azureBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Azure Blob Storage") // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) if err != nil { recoverMode(err, "Error backing up database") return diff --git a/pkg/backup.go b/pkg/backup.go index 46ab8ee..f0758b7 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -112,7 +112,7 @@ func multiBackupTask(databases []Database, bkConfig *BackupConfig) { // createBackupTask backup task func createBackupTask(db *dbConfig, config *BackupConfig) { - if config.all && !config.singleFile { + if config.all && !config.allInOne { backupAll(db, config) } else { backupTask(db, config) @@ -141,7 +141,7 @@ func backupTask(db *dbConfig, config *BackupConfig) { utils.Info("Starting backup task...") startTime = time.Now() prefix := db.dbName - if config.all && config.singleFile { + if config.all && config.allInOne { prefix = "all_databases" } // Generate file name @@ -302,7 +302,7 @@ func runCommandWithCompression(command string, args []string, outputPath string) // localBackup backup database to local storage func localBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to local storage") - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) if err != nil { recoverMode(err, "Error backing up database") return diff --git a/pkg/config.go b/pkg/config.go index b24040d..e02532e 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -78,7 +78,7 @@ type BackupConfig struct { storage string cronExpression string all bool - singleFile bool + allInOne bool } type FTPConfig struct { host string @@ -253,13 +253,17 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig { remotePath := utils.GetEnvVariable("REMOTE_PATH", "SSH_REMOTE_PATH") storage = utils.GetEnv(cmd, "storage", "STORAGE") prune := false + configFile := os.Getenv("BACKUP_CONFIG_FILE") backupRetention := utils.GetIntEnv("BACKUP_RETENTION_DAYS") if backupRetention > 0 { prune = true } disableCompression, _ = cmd.Flags().GetBool("disable-compression") - all, _ := cmd.Flags().GetBool("all") - singleFile, _ := cmd.Flags().GetBool("single-file") + all, _ := cmd.Flags().GetBool("all-databases") + allInOne, _ := cmd.Flags().GetBool("all-in-one") + if allInOne { + all = true + } _, _ = cmd.Flags().GetString("mode") passphrase := os.Getenv("GPG_PASSPHRASE") _ = utils.GetEnv(cmd, "path", "AWS_S3_PATH") @@ -273,6 +277,10 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig { encryption = true usingKey = false } + dbName := os.Getenv("DB_NAME") + if dbName == "" && !all && configFile == "" { + utils.Fatal("Database name is required, use DB_NAME environment variable or -d flag") + } // Initialize backup configs config := BackupConfig{} config.backupRetention = backupRetention @@ -286,7 +294,7 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig { config.usingKey = usingKey config.cronExpression = cronExpression config.all = all - config.singleFile = singleFile + config.allInOne = allInOne return &config } diff --git a/pkg/helper.go b/pkg/helper.go index c971970..4019467 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -37,7 +37,7 @@ import ( ) func intro() { - fmt.Println("Starting MySQL Backup...") + fmt.Println("Starting MYSQL-BKUP...") fmt.Printf("Version: %s\n", utils.Version) fmt.Println("Copyright (c) 2024 Jonas Kaninda") } diff --git a/pkg/remote.go b/pkg/remote.go index 84272d9..2989b21 100644 --- a/pkg/remote.go +++ b/pkg/remote.go @@ -39,7 +39,7 @@ import ( func sshBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Remote server") // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) if err != nil { recoverMode(err, "Error backing up database") return @@ -160,7 +160,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to the remote FTP server") // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) if err != nil { recoverMode(err, "Error backing up database") return diff --git a/pkg/restore.go b/pkg/restore.go index dd0c799..42633fc 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -57,11 +57,17 @@ func StartRestore(cmd *cobra.Command) { } func localRestore(dbConf *dbConfig, restoreConf *RestoreConfig) { utils.Info("Restore database from local") + basePath := filepath.Dir(restoreConf.file) + fileName := filepath.Base(restoreConf.file) + restoreConf.file = fileName + if basePath == "" || basePath == "." { + basePath = storagePath + } localStorage := local.NewStorage(local.Config{ - RemotePath: storagePath, + RemotePath: basePath, LocalPath: tmpPath, }) - err := localStorage.CopyFrom(restoreConf.file) + err := localStorage.CopyFrom(fileName) if err != nil { utils.Fatal("Error copying backup file: %s", err) } @@ -69,87 +75,79 @@ func localRestore(dbConf *dbConfig, restoreConf *RestoreConfig) { } -// RestoreDatabase restore database +// RestoreDatabase restores the database from a backup file func RestoreDatabase(db *dbConfig, conf *RestoreConfig) { if conf.file == "" { utils.Fatal("Error, file required") } - extension := filepath.Ext(filepath.Join(tmpPath, conf.file)) - rFile, err := os.ReadFile(filepath.Join(tmpPath, conf.file)) - outputFile := RemoveLastExtension(filepath.Join(tmpPath, conf.file)) + + filePath := filepath.Join(tmpPath, conf.file) + rFile, err := os.ReadFile(filePath) if err != nil { - utils.Fatal("Error reading backup file: %s ", err) + utils.Fatal("Error reading backup file: %v", err) } + extension := filepath.Ext(filePath) + outputFile := RemoveLastExtension(filePath) + if extension == ".gpg" { - - if conf.usingKey { - utils.Info("Decrypting backup using private key...") - utils.Warn("Backup decryption using a private key is not fully supported") - prKey, err := os.ReadFile(conf.privateKey) - if err != nil { - utils.Fatal("Error reading public key: %s ", err) - } - err = encryptor.DecryptWithPrivateKey(rFile, outputFile, prKey, conf.passphrase) - if err != nil { - utils.Fatal("error during decrypting backup %v", err) - } - utils.Info("Decrypting backup using private key...done") - } else { - if conf.passphrase == "" { - utils.Error("Error, passphrase or private key required") - utils.Fatal("Your file seems to be a GPG file.\nYou need to provide GPG keys. GPG_PASSPHRASE or GPG_PRIVATE_KEY environment variable is required.") - } else { - utils.Info("Decrypting backup using passphrase...") - // decryptWithGPG file - err := encryptor.Decrypt(rFile, outputFile, conf.passphrase) - if err != nil { - utils.Fatal("Error decrypting file %s %v", file, err) - } - utils.Info("Decrypting backup using passphrase...done") - // Update file name - conf.file = RemoveLastExtension(file) - } - } - + decryptBackup(conf, rFile, outputFile) } - if utils.FileExists(filepath.Join(tmpPath, conf.file)) { - err = testDatabaseConnection(db) + restorationFile := filepath.Join(tmpPath, conf.file) + if !utils.FileExists(restorationFile) { + utils.Fatal("File not found: %s", restorationFile) + } + + if err := testDatabaseConnection(db); err != nil { + utils.Fatal("Error connecting to the database: %v", err) + } + + utils.Info("Restoring database...") + restoreDatabaseFile(db, restorationFile) +} + +func decryptBackup(conf *RestoreConfig, rFile []byte, outputFile string) { + if conf.usingKey { + utils.Info("Decrypting backup using private key...") + prKey, err := os.ReadFile(conf.privateKey) if err != nil { - utils.Fatal("Error connecting to the database %v", err) + utils.Fatal("Error reading private key: %v", err) } - utils.Info("Restoring database...") - - extension := filepath.Ext(filepath.Join(tmpPath, conf.file)) - // Restore from a compressed file / .sql.gz - if extension == ".gz" { - str := "zcat " + filepath.Join(tmpPath, conf.file) + " | mariadb " + fmt.Sprintf("--defaults-file=%s", mysqlClientConfig) + " " + db.dbName - output, err := exec.Command("sh", "-c", str).Output() - if err != nil { - utils.Fatal("Error, in restoring the database %v output: %v", err, output) - } - utils.Info("Restoring database... done") - utils.Info("Database has been restored") - // Delete temp - deleteTemp() - - } else if extension == ".sql" { - // Restore from SQL file - str := "cat " + filepath.Join(tmpPath, conf.file) + " | mariadb " + fmt.Sprintf("--defaults-file=%s", mysqlClientConfig) + " " + db.dbName - output, err := exec.Command("sh", "-c", str).Output() - if err != nil { - utils.Fatal("Error, in restoring the database %v output: %v", err, output) - } - utils.Info("Restoring database... done") - utils.Info("Database has been restored") - // Delete temp - deleteTemp() - } else { - utils.Fatal("Unknown file extension %s", extension) + if err := encryptor.DecryptWithPrivateKey(rFile, outputFile, prKey, conf.passphrase); err != nil { + utils.Fatal("Error decrypting backup: %v", err) } - } else { - utils.Fatal("File not found in %s", filepath.Join(tmpPath, conf.file)) + if conf.passphrase == "" { + utils.Fatal("Passphrase or private key required for GPG file.") + } + utils.Info("Decrypting backup using passphrase...") + if err := encryptor.Decrypt(rFile, outputFile, conf.passphrase); err != nil { + utils.Fatal("Error decrypting file: %v", err) + } + conf.file = RemoveLastExtension(conf.file) } } + +func restoreDatabaseFile(db *dbConfig, restorationFile string) { + extension := filepath.Ext(restorationFile) + var cmdStr string + + switch extension { + case ".gz": + cmdStr = fmt.Sprintf("zcat %s | mariadb --defaults-file=%s %s", restorationFile, mysqlClientConfig, db.dbName) + case ".sql": + cmdStr = fmt.Sprintf("cat %s | mariadb --defaults-file=%s %s", restorationFile, mysqlClientConfig, db.dbName) + default: + utils.Fatal("Unknown file extension: %s", extension) + } + + cmd := exec.Command("sh", "-c", cmdStr) + output, err := cmd.CombinedOutput() + if err != nil { + utils.Fatal("Error restoring database: %v\nOutput: %s", err, string(output)) + } + + utils.Info("Database has been restored successfully.") + deleteTemp() +} diff --git a/pkg/s3.go b/pkg/s3.go index 8efc5b7..be80481 100644 --- a/pkg/s3.go +++ b/pkg/s3.go @@ -39,7 +39,7 @@ func s3Backup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to s3 storage") // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.allInOne) if err != nil { recoverMode(err, "Error backing up database") return diff --git a/pkg/var.go b/pkg/var.go index 4a15b1e..a00ecd7 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -54,7 +54,6 @@ var dbHVars = []string{ "DB_HOST", "DB_PASSWORD", "DB_USERNAME", - "DB_NAME", } var tdbRVars = []string{ "TARGET_DB_HOST",