From 01c5a061c5fb23c678c787cb85a3211fa751bf2f Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Mon, 21 Oct 2024 08:02:57 +0200 Subject: [PATCH 1/4] refactor: clean up code --- pkg/backup.go | 6 +++--- pkg/helper.go | 6 +++--- pkg/restore.go | 4 ++-- pkg/s3.go | 9 ++++++--- utils/logger.go | 10 ---------- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/pkg/backup.go b/pkg/backup.go index e233f61..c88dfb7 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -308,7 +308,7 @@ func s3Backup(db *dbConfig, config *BackupConfig) { utils.Fatal("Error deleting old backup from S3: %s ", err) } } - utils.Done("Uploading backup archive to remote storage S3 ... done ") + utils.Info("Uploading backup archive to remote storage S3 ... done ") //Send notification utils.NotifySuccess(&utils.NotificationData{ File: finalFileName, @@ -360,7 +360,7 @@ func sshBackup(db *dbConfig, config *BackupConfig) { } - utils.Done("Uploading backup archive to remote storage ... done ") + utils.Info("Uploading backup archive to remote storage ... done ") //Send notification utils.NotifySuccess(&utils.NotificationData{ File: finalFileName, @@ -412,7 +412,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { } - utils.Done("Uploading backup archive to the remote FTP server ... done ") + utils.Info("Uploading backup archive to the remote FTP server ... done ") //Send notification utils.NotifySuccess(&utils.NotificationData{ diff --git a/pkg/helper.go b/pkg/helper.go index 2095e0a..1c6274b 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -45,7 +45,7 @@ func moveToBackup(backupFileName string, destinationPath string) { utils.Error("Error deleting file: %s", err) } - utils.Done("Database has been backed up and copied to %s", filepath.Join(destinationPath, backupFileName)) + utils.Info("Database has been backed up and copied to %s", filepath.Join(destinationPath, backupFileName)) } func deleteOldBackup(retentionDays int) { utils.Info("Deleting old backups...") @@ -60,7 +60,7 @@ func deleteOldBackup(retentionDays int) { if err != nil { utils.Fatal("Error:", err) } else { - utils.Done("File %s deleted successfully", filePath) + utils.Info("File %s deleted successfully", filePath) } return err } @@ -87,7 +87,7 @@ func deleteOldBackup(retentionDays int) { utils.Fatal("Error:", err) return } - utils.Done("Deleting old backups...done") + utils.Info("Deleting old backups...done") } func deleteTemp() { utils.Info("Deleting %s ...", tmpPath) diff --git a/pkg/restore.go b/pkg/restore.go index f950095..82ae4ff 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -127,7 +127,7 @@ func RestoreDatabase(db *dbConfig, conf *RestoreConfig) { utils.Fatal("Error, in restoring the database %v", err) } utils.Info("Restoring database... done") - utils.Done("Database has been restored") + utils.Info("Database has been restored") //Delete temp deleteTemp() @@ -139,7 +139,7 @@ func RestoreDatabase(db *dbConfig, conf *RestoreConfig) { utils.Fatal("Error in restoring the database %v", err) } utils.Info("Restoring database... done") - utils.Done("Database has been restored") + utils.Info("Database has been restored") //Delete temp deleteTemp() } else { diff --git a/pkg/s3.go b/pkg/s3.go index e69556b..abb70c1 100644 --- a/pkg/s3.go +++ b/pkg/s3.go @@ -106,6 +106,8 @@ func DownloadFile(destinationPath, key, bucket, prefix string) error { return nil } func DeleteOldBackup(bucket, prefix string, retention int) error { + utils.Info("Deleting old backups...") + utils.Info("Bucket %s Prefix: %s Retention: %d", bucket, prefix, retention) sess, err := CreateSession() if err != nil { return err @@ -113,7 +115,7 @@ func DeleteOldBackup(bucket, prefix string, retention int) error { svc := s3.New(sess) - // Get the current time and the time threshold for 7 days ago + // Get the current time now := time.Now() backupRetentionDays := now.AddDate(0, 0, -retention) @@ -125,6 +127,7 @@ func DeleteOldBackup(bucket, prefix string, retention int) error { err = svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool { for _, object := range page.Contents { if object.LastModified.Before(backupRetentionDays) { + utils.Info("Deleting old backup: %s", *object.Key) // Object is older than retention days, delete it _, err := svc.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(bucket), @@ -133,7 +136,7 @@ func DeleteOldBackup(bucket, prefix string, retention int) error { if err != nil { utils.Info("Failed to delete object %s: %v", *object.Key, err) } else { - utils.Info("Deleted object %s\n", *object.Key) + utils.Info("Deleted object %s", *object.Key) } } } @@ -143,6 +146,6 @@ func DeleteOldBackup(bucket, prefix string, retention int) error { utils.Error("Failed to list objects: %v", err) } - utils.Info("Finished deleting old files.") + utils.Info("Deleting old backups...done") return nil } diff --git a/utils/logger.go b/utils/logger.go index 2d1a41b..219196b 100644 --- a/utils/logger.go +++ b/utils/logger.go @@ -44,16 +44,6 @@ func Error(msg string, args ...any) { fmt.Printf("%s ERROR: %s\n", currentTime, formattedMessage) } } -func Done(msg string, args ...any) { - var currentTime = time.Now().Format("2006/01/02 15:04:05") - formattedMessage := fmt.Sprintf(msg, args...) - if len(args) == 0 { - fmt.Printf("%s INFO: %s\n", currentTime, msg) - } else { - fmt.Printf("%s INFO: %s\n", currentTime, formattedMessage) - } -} - func Fatal(msg string, args ...any) { var currentTime = time.Now().Format("2006/01/02 15:04:05") // Fatal logs an error message and exits the program. From f09c1d6fd0c8a4dacc89ee5d4c14ea03d4211cb7 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Mon, 21 Oct 2024 08:18:39 +0200 Subject: [PATCH 2/4] refactor: clean up Dockerfile --- Dockerfile | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 770b559..6747d0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,19 +22,12 @@ LABEL version=${appVersion} LABEL github="github.com/jkaninda/pg-bkup" RUN apk --update add --no-cache postgresql-client tzdata ca-certificates -RUN mkdir $WORKDIR -RUN mkdir $BACKUPDIR -RUN mkdir $TEMPLATES_DIR -RUN mkdir -p $BACKUP_TMP_DIR -RUN chmod 777 $WORKDIR -RUN chmod 777 $BACKUPDIR -RUN chmod 777 $BACKUP_TMP_DIR -RUN chmod 777 $WORKDIR +RUN mkdir -p $WORKDIR $BACKUPDIR $TEMPLATES_DIR $BACKUP_TMP_DIR && \ + chmod a+rw $WORKDIR $BACKUPDIR $BACKUP_TMP_DIR COPY --from=build /app/pg-bkup /usr/local/bin/pg-bkup COPY ./templates/* $TEMPLATES_DIR/ -RUN chmod +x /usr/local/bin/pg-bkup - -RUN ln -s /usr/local/bin/pg-bkup /usr/local/bin/bkup +RUN chmod +x /usr/local/bin/pg-bkup && \ + ln -s /usr/local/bin/pg-bkup /usr/local/bin/bkup # Create the backup script and make it executable RUN printf '#!/bin/sh\n/usr/local/bin/pg-bkup backup "$@"' > /usr/local/bin/backup && \ From 3b75f0432c50ca49bff4637c88adae152498894b Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Mon, 21 Oct 2024 08:30:24 +0200 Subject: [PATCH 3/4] doc: update deployment using s3 storage --- docs/how-tos/backup-to-s3.md | 3 ++- examples/docker-compose.s3.yaml | 1 + examples/docker-compose.scheduled.s3.yaml | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/how-tos/backup-to-s3.md b/docs/how-tos/backup-to-s3.md index a9ae1ae..45c8622 100644 --- a/docs/how-tos/backup-to-s3.md +++ b/docs/how-tos/backup-to-s3.md @@ -37,7 +37,7 @@ services: - AWS_SECRET_KEY=xxxxx ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true - AWS_DISABLE_SSL="false" - - AWS_FORCE_PATH_STYLE="false" + - AWS_FORCE_PATH_STYLE=false # true for S3 alternative such as Minio # pg-bkup container must be connected to the same network with your database networks: @@ -78,6 +78,7 @@ services: #- BACKUP_RETENTION_DAYS=7 ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true - AWS_DISABLE_SSL="false" + - AWS_FORCE_PATH_STYLE=true # true for S3 alternative such as Minio # pg-bkup container must be connected to the same network with your database networks: - web diff --git a/examples/docker-compose.s3.yaml b/examples/docker-compose.s3.yaml index 8d4d6ea..2409f5b 100644 --- a/examples/docker-compose.s3.yaml +++ b/examples/docker-compose.s3.yaml @@ -21,6 +21,7 @@ services: - AWS_SECRET_KEY=xxxxx ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true - AWS_DISABLE_SSL="false" + - AWS_FORCE_PATH_STYLE=false # true for S3 alternative such as Minio # pg-bkup container must be connected to the same network with your database networks: - web diff --git a/examples/docker-compose.scheduled.s3.yaml b/examples/docker-compose.scheduled.s3.yaml index ea1ef2b..07e9127 100644 --- a/examples/docker-compose.scheduled.s3.yaml +++ b/examples/docker-compose.scheduled.s3.yaml @@ -21,6 +21,7 @@ services: - AWS_SECRET_KEY=xxxxx ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true - AWS_DISABLE_SSL="false" + - AWS_FORCE_PATH_STYLE=false # true for S3 alternative such as Minio # Check https://jkaninda.github.io/pg-bkup/reference/#predefined-schedules - BACKUP_CRON_EXPRESSION=@daily #@every 5m|@weekly | @monthly |0 1 * * * # pg-bkup container must be connected to the same network with your database From 97c709489c69042288a42c54d23d4c4be42c4037 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Mon, 21 Oct 2024 08:33:01 +0200 Subject: [PATCH 4/4] chore: add ssh, ftp to storage list --- cmd/backup.go | 2 +- cmd/restore.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 19f8b11..15acfc2 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -28,7 +28,7 @@ var BackupCmd = &cobra.Command{ func init() { //Backup - BackupCmd.PersistentFlags().StringP("storage", "s", "local", "Storage. local or s3") + BackupCmd.PersistentFlags().StringP("storage", "s", "local", "Define storage: local, s3, ssh, ftp") BackupCmd.PersistentFlags().StringP("path", "P", "", "AWS S3 path without file name. eg: /custom_path or ssh remote path `/home/foo/backup`") BackupCmd.PersistentFlags().StringP("cron-expression", "", "", "Backup cron expression") BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression") diff --git a/cmd/restore.go b/cmd/restore.go index 270c50c..6e4069a 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -30,7 +30,7 @@ var RestoreCmd = &cobra.Command{ func init() { //Restore RestoreCmd.PersistentFlags().StringP("file", "f", "", "File name of database") - RestoreCmd.PersistentFlags().StringP("storage", "s", "local", "Storage. local or s3") + RestoreCmd.PersistentFlags().StringP("storage", "s", "local", "Define storage: local, s3, ssh, ftp") RestoreCmd.PersistentFlags().StringP("path", "P", "", "AWS S3 path without file name. eg: /custom_path or ssh remote path `/home/foo/backup`") }