diff --git a/cmd/backup.go b/cmd/backup.go index afe12eb..d792772 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -51,5 +51,6 @@ func init() { BackupCmd.PersistentFlags().StringP("config", "c", "", "Configuration file for multi database backup. (e.g: `/backup/config.yaml`)") BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression") BackupCmd.PersistentFlags().BoolP("all", "a", false, "Backup all databases") + BackupCmd.PersistentFlags().BoolP("single-file", "", false, "Backup all databases in a single file") } diff --git a/pkg/azure.go b/pkg/azure.go index 6278bed..195699c 100644 --- a/pkg/azure.go +++ b/pkg/azure.go @@ -39,7 +39,7 @@ func azureBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Azure Blob Storage") // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) if err != nil { recoverMode(err, "Error backing up database") return diff --git a/pkg/backup.go b/pkg/backup.go index cbfe9ac..1336aac 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -26,6 +26,7 @@ SOFTWARE. package pkg import ( + "bytes" "fmt" "github.com/jkaninda/encryptor" "github.com/jkaninda/go-storage/pkg/local" @@ -36,6 +37,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" ) @@ -48,7 +50,7 @@ func StartBackup(cmd *cobra.Command) { if err != nil { dbConf = initDbConfig(cmd) if config.cronExpression == "" { - BackupTask(dbConf, config) + createBackupTask(dbConf, config) } else { if utils.IsValidCronExpression(config.cronExpression) { scheduledMode(dbConf, config) @@ -82,7 +84,7 @@ func scheduledMode(db *dbConfig, config *BackupConfig) { c := cron.New() _, err = c.AddFunc(config.cronExpression, func() { - BackupTask(db, config) + createBackupTask(db, config) utils.Info("Next backup time is: %v", utils.CronNextTime(config.cronExpression).Format(timeFormat)) }) @@ -104,16 +106,41 @@ func multiBackupTask(databases []Database, bkConfig *BackupConfig) { if db.Path != "" { bkConfig.remotePath = db.Path } - BackupTask(getDatabase(db), bkConfig) + createBackupTask(getDatabase(db), bkConfig) } } -// BackupTask backups database -func BackupTask(db *dbConfig, config *BackupConfig) { +// createBackupTask backup task +func createBackupTask(db *dbConfig, config *BackupConfig) { + if config.all && !config.singleFile { + backupAll(db, config) + } else { + backupTask(db, config) + } +} + +// backupAll backup all databases +func backupAll(db *dbConfig, config *BackupConfig) { + databases, err := listDatabases(*db) + if err != nil { + utils.Fatal("Error listing databases: %s", err) + } + for _, dbName := range databases { + if dbName == "information_schema" || dbName == "performance_schema" || dbName == "mysql" || dbName == "sys" || dbName == "innodb" || dbName == "Database" { + continue + } + db.dbName = dbName + config.backupFileName = fmt.Sprintf("%s_%s.sql.gz", dbName, time.Now().Format("20060102_150405")) + backupTask(db, config) + } + +} + +func backupTask(db *dbConfig, config *BackupConfig) { utils.Info("Starting backup task...") startTime = time.Now() prefix := db.dbName - if config.all { + if config.all && config.singleFile { prefix = "all_databases" } // Generate file name @@ -199,7 +226,7 @@ func startMultiBackup(bkConfig *BackupConfig, configFile string) { } // BackupDatabase backup database -func BackupDatabase(db *dbConfig, backupFileName string, disableCompression, all bool) error { +func BackupDatabase(db *dbConfig, backupFileName string, disableCompression, all, singleFile bool) error { storagePath = os.Getenv("STORAGE_PATH") utils.Info("Starting database backup...") @@ -208,7 +235,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression, all } dumpArgs := []string{fmt.Sprintf("--defaults-file=%s", mysqlClientConfig)} - if all { + if all && singleFile { dumpArgs = append(dumpArgs, "--all-databases", "--single-transaction", "--routines", "--triggers") } else { dumpArgs = append(dumpArgs, db.dbName) @@ -267,7 +294,7 @@ func runCommandWithCompression(command string, args []string, outputPath string) } func localBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to local storage") - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) if err != nil { recoverMode(err, "Error backing up database") return @@ -346,6 +373,31 @@ func encryptBackup(config *BackupConfig) { } } + +// listDatabases list all databases +func listDatabases(db dbConfig) ([]string, error) { + databases := []string{} + // Create the mysql client config file + if err := createMysqlClientConfigFile(db); err != nil { + return databases, fmt.Errorf(err.Error()) + } + utils.Info("Listing databases...") + // Step 1: List all databases + cmd := exec.Command("mariadb", fmt.Sprintf("--defaults-file=%s", mysqlClientConfig), "-e", "SHOW DATABASES;") + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return databases, fmt.Errorf("failed to list databases: %s", err) + } + // Step 2: Parse the output + for _, _db := range strings.Split(out.String(), "\n") { + if _db != "" { + databases = append(databases, _db) + } + } + return databases, nil +} func recoverMode(err error, msg string) { if err != nil { if backupRescueMode { diff --git a/pkg/config.go b/pkg/config.go index ce401f9..b24040d 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -78,6 +78,7 @@ type BackupConfig struct { storage string cronExpression string all bool + singleFile bool } type FTPConfig struct { host string @@ -258,6 +259,7 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig { } disableCompression, _ = cmd.Flags().GetBool("disable-compression") all, _ := cmd.Flags().GetBool("all") + singleFile, _ := cmd.Flags().GetBool("single-file") _, _ = cmd.Flags().GetString("mode") passphrase := os.Getenv("GPG_PASSPHRASE") _ = utils.GetEnv(cmd, "path", "AWS_S3_PATH") @@ -284,6 +286,7 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig { config.usingKey = usingKey config.cronExpression = cronExpression config.all = all + config.singleFile = singleFile return &config } diff --git a/pkg/helper.go b/pkg/helper.go index eb03af7..c971970 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -195,8 +195,8 @@ func createMysqlClientConfigFile(db dbConfig) error { sslMode := goutils.GetStringEnvWithDefault("DB_SSL_MODE", "0") // Create the mysql client config file mysqlClientConfigFile := filepath.Join(tmpPath, "my.cnf") - mysqlClientConfig := fmt.Sprintf("[client]\nhost=%s\nport=%s\nuser=%s\npassword=%s\nssl-ca=%s\nssl=%s\n", db.dbHost, db.dbPort, db.dbUserName, db.dbPassword, caCertPath, sslMode) - if err := os.WriteFile(mysqlClientConfigFile, []byte(mysqlClientConfig), 0644); err != nil { + mysqlCl := fmt.Sprintf("[client]\nhost=%s\nport=%s\nuser=%s\npassword=%s\nssl-ca=%s\nssl=%s\n", db.dbHost, db.dbPort, db.dbUserName, db.dbPassword, caCertPath, sslMode) + if err := os.WriteFile(mysqlClientConfigFile, []byte(mysqlCl), 0644); err != nil { return fmt.Errorf("failed to create mysql client config file: %v", err) } return nil diff --git a/pkg/migrate.go b/pkg/migrate.go index 41c9229..e6d7fb7 100644 --- a/pkg/migrate.go +++ b/pkg/migrate.go @@ -51,7 +51,7 @@ func StartMigration(cmd *cobra.Command) { conf := &RestoreConfig{} conf.file = backupFileName // Backup source Database - err := BackupDatabase(dbConf, backupFileName, true, false) + err := BackupDatabase(dbConf, backupFileName, true, false, false) if err != nil { utils.Fatal("Error backing up database: %s", err) } diff --git a/pkg/remote.go b/pkg/remote.go index 5545882..a92cf94 100644 --- a/pkg/remote.go +++ b/pkg/remote.go @@ -39,7 +39,7 @@ import ( func sshBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Remote server") // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) if err != nil { recoverMode(err, "Error backing up database") return @@ -160,7 +160,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to the remote FTP server") // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) if err != nil { recoverMode(err, "Error backing up database") return diff --git a/pkg/s3.go b/pkg/s3.go index 6327fa5..c3853b7 100644 --- a/pkg/s3.go +++ b/pkg/s3.go @@ -39,7 +39,7 @@ func s3Backup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to s3 storage") // Backup database - err := BackupDatabase(db, config.backupFileName, disableCompression, config.all) + err := BackupDatabase(db, config.backupFileName, disableCompression, config.all, config.singleFile) if err != nil { recoverMode(err, "Error backing up database") return