diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9242f3d..3530887 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: uses: docker/build-push-action@v3 with: push: true - file: "./docker/Dockerfile" + file: "./Dockerfile" platforms: linux/amd64,linux/arm64,linux/arm/v7 build-args: | appVersion=develop-${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cb417ee..c6ea3c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: uses: docker/build-push-action@v3 with: push: true - file: "./docker/Dockerfile" + file: "./Dockerfile" platforms: linux/amd64,linux/arm64,linux/arm/v7 build-args: | appVersion=${{ env.TAG_NAME }} diff --git a/docker/Dockerfile b/Dockerfile similarity index 100% rename from docker/Dockerfile rename to Dockerfile diff --git a/docs/how-tos/deprecated-configs.md b/docs/how-tos/deprecated-configs.md new file mode 100644 index 0000000..1813ff2 --- /dev/null +++ b/docs/how-tos/deprecated-configs.md @@ -0,0 +1,6 @@ +--- +title: Update deprecated configurations +layout: default +parent: How Tos +nav_order: 11 +--- \ No newline at end of file diff --git a/docs/how-tos/mutli-backup.md b/docs/how-tos/mutli-backup.md new file mode 100644 index 0000000..1a033a8 --- /dev/null +++ b/docs/how-tos/mutli-backup.md @@ -0,0 +1,40 @@ +--- +title: Run multiple database backup schedules in the same container +layout: default +parent: How Tos +nav_order: 11 +--- + +Multiple backup schedules with different configuration can be configured by mounting a configuration file into `/config/config.yaml` `/config/config.yml` or by defining an environment variable `BACKUP_CONFIG_FILE=/backup/config.yaml`. + +## Configuration file + +```yaml +#cronExpression: "@every 20m" //Optional, for scheduled backups +cronExpression: "" +databases: + - host: postgres1 + port: 5432 + name: database1 + user: database1 + password: password + path: /s3-path/database1 #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres2 + port: 5432 + name: lldap + user: lldap + password: password + path: /s3-path/lldap #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres3 + port: 5432 + name: keycloak + user: keycloak + password: password + path: /s3-path/keycloak #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres4 + port: 5432 + name: joplin + user: joplin + password: password + path: /s3-path/joplin #For SSH or FTP you need to define the full path (/home/toto/backup/) +``` \ No newline at end of file diff --git a/go.mod b/go.mod index 8ae07ba..876dd6d 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.8.0 golang.org/x/crypto v0.28.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/pkg/backup.go b/pkg/backup.go index abca1ac..6851564 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -20,23 +20,28 @@ import ( func StartBackup(cmd *cobra.Command) { intro() - dbConf = initDbConfig(cmd) //Initialize backup configs config := initBackupConfig(cmd) - - if config.cronExpression == "" { - BackupTask(dbConf, config) - } else { - if utils.IsValidCronExpression(config.cronExpression) { - scheduledMode(dbConf, config) + //Load backup configuration file + configFile, err := loadConfigFile() + if err != nil { + dbConf = initDbConfig(cmd) + if config.cronExpression == "" { + BackupTask(dbConf, config) } else { - utils.Fatal("Cron expression is not valid: %s", config.cronExpression) + if utils.IsValidCronExpression(config.cronExpression) { + scheduledMode(dbConf, config) + } else { + utils.Fatal("Cron expression is not valid: %s", config.cronExpression) + } } + } else { + startMultiBackup(config, configFile) } } -// Run in scheduled mode +// scheduledMode Runs backup in scheduled mode func scheduledMode(db *dbConfig, config *BackupConfig) { utils.Info("Running in Scheduled mode") utils.Info("Backup cron expression: %s", config.cronExpression) @@ -63,6 +68,17 @@ func scheduledMode(db *dbConfig, config *BackupConfig) { defer c.Stop() select {} } + +// multiBackupTask backup multi database +func multiBackupTask(databases []Database, bkConfig *BackupConfig) { + for _, db := range databases { + //Check if path is defined in config file + if db.Path != "" { + bkConfig.remotePath = db.Path + } + BackupTask(getDatabase(db), bkConfig) + } +} func BackupTask(db *dbConfig, config *BackupConfig) { utils.Info("Starting backup task...") //Generate file name @@ -85,9 +101,54 @@ func BackupTask(db *dbConfig, config *BackupConfig) { localBackup(db, config) } } -func intro() { - utils.Info("Starting PostgreSQL Backup...") - utils.Info("Copyright (c) 2024 Jonas Kaninda ") +func startMultiBackup(bkConfig *BackupConfig, configFile string) { + utils.Info("Starting multiple backup job...") + var conf = &Config{} + conf, err := readConf(configFile) + if err != nil { + utils.Fatal("Error reading config file: %s", err) + } + //Check if cronExpression is defined in config file + if conf.CronExpression != "" { + bkConfig.cronExpression = conf.CronExpression + } + // Check if cronExpression is defined + if bkConfig.cronExpression == "" { + multiBackupTask(conf.Databases, bkConfig) + } else { + // Check if cronExpression is valid + if utils.IsValidCronExpression(bkConfig.cronExpression) { + utils.Info("Running MultiBackup in Scheduled mode") + utils.Info("Backup cron expression: %s", bkConfig.cronExpression) + utils.Info("Storage type %s ", bkConfig.storage) + + //Test backup + utils.Info("Testing backup configurations...") + multiBackupTask(conf.Databases, bkConfig) + utils.Info("Testing backup configurations...done") + utils.Info("Creating multi backup job...") + // Create a new cron instance + c := cron.New() + + _, err := c.AddFunc(bkConfig.cronExpression, func() { + // Create a channel + multiBackupTask(conf.Databases, bkConfig) + }) + if err != nil { + return + } + // Start the cron scheduler + c.Start() + utils.Info("Creating multi backup job...done") + utils.Info("Backup job started") + defer c.Stop() + select {} + + } else { + utils.Fatal("Cron expression is not valid: %s", bkConfig.cronExpression) + } + } + } // BackupDatabase backup database @@ -119,7 +180,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool log.Fatal(err) } // save output - file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + file, err := os.Create(filepath.Join(tmpPath, backupFileName)) if err != nil { log.Fatal(err) } @@ -145,7 +206,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool gzipCmd := exec.Command("gzip") gzipCmd.Stdin = stdout // save output - gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName)) gzipCmd.Start() if err != nil { log.Fatal(err) @@ -180,6 +241,7 @@ func localBackup(db *dbConfig, config *BackupConfig) { } //Delete temp deleteTemp() + utils.Info("Backup completed successfully") } func s3Backup(db *dbConfig, config *BackupConfig) { @@ -220,6 +282,8 @@ func s3Backup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") + } func sshBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Remote server") @@ -255,6 +319,8 @@ func sshBackup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") + } func ftpBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to the remote FTP server") @@ -290,6 +356,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") } func encryptBackup(config *BackupConfig) { diff --git a/pkg/config.go b/pkg/config.go index d898523..5015fce 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -14,7 +14,17 @@ import ( "strconv" ) +type Database struct { + Host string `yaml:"host"` + Port string `yaml:"port"` + Name string `yaml:"name"` + User string `yaml:"user"` + Password string `yaml:"password"` + Path string `yaml:"path"` +} type Config struct { + Databases []Database `yaml:"databases"` + CronExpression string `yaml:"cronExpression"` } type dbConfig struct { @@ -92,6 +102,16 @@ func initDbConfig(cmd *cobra.Command) *dbConfig { return &dConf } +func getDatabase(database Database) *dbConfig { + return &dbConfig{ + dbHost: database.Host, + dbPort: database.Port, + dbName: database.Name, + dbUserName: database.User, + dbPassword: database.Password, + } +} + // loadSSHConfig loads the SSH configuration from environment variables func loadSSHConfig() (*SSHConfig, error) { utils.GetEnvVariable("SSH_HOST", "SSH_HOST_NAME") @@ -245,3 +265,10 @@ func initTargetDbConfig() *targetDbConfig { } return &tdbConfig } +func loadConfigFile() (string, error) { + backupConfigFile, err := checkConfigFile(os.Getenv("BACKUP_CONFIG_FILE")) + if err == nil { + return backupConfigFile, nil + } + return "", fmt.Errorf("backup config file not found") +} diff --git a/pkg/helper.go b/pkg/helper.go index 4f236ee..8586631 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -10,12 +10,18 @@ import ( "bytes" "fmt" "github.com/jkaninda/pg-bkup/utils" + "gopkg.in/yaml.v3" "os" "os/exec" "path/filepath" "time" ) +func intro() { + utils.Info("Starting PostgreSQL Backup...") + utils.Info("Copyright (c) 2024 Jonas Kaninda ") +} + // copyToTmp copy file to temporary directory func copyToTmp(sourcePath string, backupFileName string) { //Copy backup from storage to /tmp @@ -139,6 +145,8 @@ func testDatabaseConnection(db *dbConfig) { utils.Info("Successfully connected to %s database", db.dbName) } + +// checkPubKeyFile checks gpg public key func checkPubKeyFile(pubKey string) (string, error) { // Define possible key file names keyFiles := []string{filepath.Join(gpgHome, "public_key.asc"), filepath.Join(gpgHome, "public_key.gpg"), pubKey} @@ -160,6 +168,8 @@ func checkPubKeyFile(pubKey string) (string, error) { // Return an error if neither file exists return "", fmt.Errorf("no public key file found") } + +// checkPrKeyFile checks private key func checkPrKeyFile(prKey string) (string, error) { // Define possible key file names keyFiles := []string{filepath.Join(gpgHome, "private_key.asc"), filepath.Join(gpgHome, "private_key.gpg"), prKey} @@ -181,3 +191,45 @@ func checkPrKeyFile(prKey string) (string, error) { // Return an error if neither file exists return "", fmt.Errorf("no public key file found") } + +// readConf reads config file and returns Config +func readConf(configFile string) (*Config, error) { + if utils.FileExists(configFile) { + buf, err := os.ReadFile(configFile) + if err != nil { + return nil, err + } + + c := &Config{} + err = yaml.Unmarshal(buf, c) + if err != nil { + return nil, fmt.Errorf("in file %q: %w", configFile, err) + } + + return c, err + } + return nil, fmt.Errorf("config file %q not found", configFile) +} + +// checkConfigFile checks config files and returns one config file +func checkConfigFile(filePath string) (string, error) { + // Define possible config file names + configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath} + + // Loop through config file names and check if they exist + for _, configFile := range configFiles { + if _, err := os.Stat(configFile); err == nil { + // File exists + return configFile, nil + } else if os.IsNotExist(err) { + // File does not exist, continue to the next one + continue + } else { + // An unexpected error occurred + return "", err + } + } + + // Return an error if neither file exists + return "", fmt.Errorf("no config file found") +} diff --git a/pkg/var.go b/pkg/var.go index 2702bd5..1541f94 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -16,6 +16,7 @@ var ( file = "" storagePath = "/backup" + workingDir = "/config" disableCompression = false encryption = false usingKey = false diff --git a/utils/utils.go b/utils/utils.go index 8ecb140..15392b8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -20,6 +20,7 @@ import ( "strconv" ) +// FileExists checks if the file does exist func FileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) {