fix: Fix AWS S3 and SSH backup in scheduled mode on Docker and Docker Swarm

2024-08-04 11:19:11 +02:00
parent df2757fb1d
commit 012f6392a7
9 changed files with 67 additions and 32 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -0,0 +1,32 @@
+name: Build
+on:
+  push:
+    branches: ['develop']
+env:
+  BUILDKIT_IMAGE: jkaninda/pg-bkup
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          file: "./docker/Dockerfile"
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          tags: |
+            "${{env.BUILDKIT_IMAGE}}:develop-${{ github.sha }}"
+
--- a/8
+++ b/8
@@ -29,10 +29,10 @@ docker-run-scheduled: docker-build


 docker-run-scheduled-s3: docker-build
-	docker run --rm --network web --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}"  ${IMAGE_NAME}  bkup backup --storage s3 --mode scheduled --path /custom-path --period "* * * * *"
+	docker run --rm --network web  --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}"  ${IMAGE_NAME}  bkup backup --storage s3 --mode scheduled --path /custom-path --period "* * * * *"

 docker-run-s3: docker-build
-	docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}"  ${IMAGE_NAME}  bkup backup --storage s3  #--path /custom-path
+	docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "S3_ENDPOINT=${AWS_S3_ENDPOINT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}"  ${IMAGE_NAME}  bkup backup --storage s3 --mode scheduled --path custom-path  --period "* * * * *"


 docker-restore-s3: docker-build
@@ -41,8 +41,10 @@ docker-restore-s3: docker-build
 docker-run-ssh: docker-build
 	docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME}  bkup backup --storage ssh

+docker-run-scheduled-ssh: docker-build
+	docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME}  bkup backup --storage ssh --mode scheduled --period "* * * * *"
 docker-restore-ssh: docker-build
-	docker run --rm --network web --name pg-bkup  -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}"  ${IMAGE_NAME}  bkup restore --storage ssh -f data_20240731_200104.sql.gz.gpg
+	docker run --rm --network web --name pg-bkup  -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}"  ${IMAGE_NAME}  bkup restore --storage ssh -f ${FILE_NAME}

 run-docs:
 	cd docs && bundle exec jekyll serve -H 0.0.0.0 -t
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -20,6 +20,7 @@ ENV AWS_S3_ENDPOINT=""
 ENV AWS_S3_BUCKET_NAME=""
 ENV AWS_ACCESS_KEY=""
 ENV AWS_SECRET_KEY=""
+ENV AWS_S3_PATH=""
 ENV AWS_REGION="us-west-2"
 ENV AWS_DISABLE_SSL="false"
 ENV GPG_PASSPHRASE=""
--- a/docker/supervisord.conf
+++ b/docker/supervisord.conf
@@ -1,13 +1,11 @@
 [supervisord]
-nodaemon=true
+nodemon=true
 user=root
 logfile=/var/log/supervisor/supervisord.log
-pidfile=/var/run/supervisord.pid
+pipfile=/var/run/supervisord.pid

 [program:cron]
 command = /bin/bash -c "declare -p | grep -Ev '^declare -[[:alpha:]]*r' > /run/supervisord.env && /usr/sbin/cron -f -L 15"
 autostart=true
-autorestart=true
-user = root
 stderr_logfile=/var/log/cron.err.log
 stdout_logfile=/var/log/cron.out.log
--- a/pkg/backup.go
+++ b/pkg/backup.go
@@ -25,7 +25,6 @@ func StartBackup(cmd *cobra.Command) {
 	utils.GetEnv(cmd, "period", "SCHEDULE_PERIOD")

 	//Get flag value and set env
-	s3Path := utils.GetEnv(cmd, "path", "AWS_S3_PATH")
 	remotePath := utils.GetEnv(cmd, "path", "SSH_REMOTE_PATH")
 	storage = utils.GetEnv(cmd, "storage", "STORAGE")
 	file = utils.GetEnv(cmd, "file", "FILE_NAME")
@@ -35,6 +34,8 @@ func StartBackup(cmd *cobra.Command) {
 	executionMode, _ = cmd.Flags().GetString("mode")
 	dbName = os.Getenv("DB_NAME")
 	gpgPassphrase := os.Getenv("GPG_PASSPHRASE")
+	_ = utils.GetEnv(cmd, "path", "AWS_S3_PATH")
+
 	//
 	if gpgPassphrase != "" {
 		encryption = true
@@ -49,7 +50,7 @@ func StartBackup(cmd *cobra.Command) {
 	if executionMode == "default" {
 		switch storage {
 		case "s3":
-			s3Backup(backupFileName, s3Path, disableCompression, prune, backupRetention, encryption)
+			s3Backup(backupFileName, disableCompression, prune, backupRetention, encryption)
 		case "local":
 			localBackup(backupFileName, disableCompression, prune, backupRetention, encryption)
 		case "ssh", "remote":
@@ -61,7 +62,7 @@ func StartBackup(cmd *cobra.Command) {
 		}

 	} else if executionMode == "scheduled" {
-		scheduledMode()
+		scheduledMode(storage)
 	} else {
 		utils.Fatal("Error, unknown execution mode!")
 	}
@@ -69,7 +70,7 @@ func StartBackup(cmd *cobra.Command) {
 }

 // Run in scheduled mode
-func scheduledMode() {
+func scheduledMode(storage string) {

 	fmt.Println()
 	fmt.Println("**********************************")
@@ -77,6 +78,7 @@ func scheduledMode() {
 	fmt.Println("***********************************")
 	utils.Info("Running in Scheduled mode")
 	utils.Info("Execution period %s ", os.Getenv("SCHEDULE_PERIOD"))
+	utils.Info("Storage type %s ", storage)

 	//Test database connexion
 	utils.TestDatabaseConnection()
@@ -101,8 +103,9 @@ func scheduledMode() {
 			utils.Info("Supervisor stopped.")
 		}
 	}()
+
 	if _, err := os.Stat(cronLogFile); os.IsNotExist(err) {
-		utils.Fatal("Log file %s does not exist.", cronLogFile)
+		utils.Fatal(fmt.Sprintf("Log file %s does not exist.", cronLogFile))
 	}
 	t, err := tail.TailFile(cronLogFile, tail.Config{Follow: true})
 	if err != nil {
@@ -213,8 +216,9 @@ func localBackup(backupFileName string, disableCompression bool, prune bool, bac
 	}
 }

-func s3Backup(backupFileName string, s3Path string, disableCompression bool, prune bool, backupRetention int, encrypt bool) {
+func s3Backup(backupFileName string, disableCompression bool, prune bool, backupRetention int, encrypt bool) {
 	bucket := utils.GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME")
+	s3Path := utils.GetEnvVariable("AWS_S3_PATH", "S3_PATH")
 	utils.Info("Backup database to s3 storage")
 	//Backup database
 	BackupDatabase(backupFileName, disableCompression)
@@ -256,7 +260,7 @@ func sshBackup(backupFileName, remotePath string, disableCompression bool, prune
 		finalFileName = fmt.Sprintf("%s.%s", backupFileName, "gpg")
 	}
 	utils.Info("Uploading backup file to remote server...")
-	utils.Info("Backup name is ", backupFileName)
+	utils.Info("Backup name is %s", backupFileName)
 	err := CopyToRemote(finalFileName, remotePath)
 	if err != nil {
 		utils.Fatal("Error uploading file to the remote server: %s ", err)
@@ -266,7 +270,7 @@ func sshBackup(backupFileName, remotePath string, disableCompression bool, prune
 	//Delete backup file from tmp folder
 	err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName))
 	if err != nil {
-		utils.Error("Error deleting file:", err)
+		utils.Error("Error deleting file: %v", err)

 	}
 	if prune {
@@ -282,7 +286,7 @@ func encryptBackup(backupFileName string) {
 	gpgPassphrase := os.Getenv("GPG_PASSPHRASE")
 	err := Encrypt(filepath.Join(tmpPath, backupFileName), gpgPassphrase)
 	if err != nil {
-		utils.Fatal("Error during encrypting backup %s", err)
+		utils.Fatal("Error during encrypting backup %v", err)
 	}

 }
--- a/pkg/scripts.go
+++ b/pkg/scripts.go
@@ -24,17 +24,10 @@ func CreateCrontabScript(disableCompression bool, storage string) {

 	var scriptContent string

-	if storage == "s3" {
-		scriptContent = fmt.Sprintf(`#!/usr/bin/env bash
+	scriptContent = fmt.Sprintf(`#!/usr/bin/env bash
 set -e
-bkup backup --dbname %s --port %s --storage s3 --path %s %v
-`, os.Getenv("DB_NAME"), os.Getenv("DB_PORT"), os.Getenv("S3_PATH"), disableC)
-	} else {
-		scriptContent = fmt.Sprintf(`#!/usr/bin/env bash
-set -e
-bkup backup --dbname %s --port %s %v
-`, os.Getenv("DB_NAME"), os.Getenv("DB_PORT"), disableC)
-	}
+bkup backup --dbname %s --port %s --storage %s %v
+`, os.Getenv("DB_NAME"), os.Getenv("DB_PORT"), storage, disableC)

 	if err := utils.WriteToFile(backupCronFile, scriptContent); err != nil {
 		utils.Fatal("Error writing to %s: %v\n", backupCronFile, err)
--- a/pkg/var.go
+++ b/pkg/var.go
@@ -1,7 +1,5 @@
 package pkg

-const s3MountPath string = "/s3mnt"
-const s3fsPasswdFile string = "/etc/passwd-s3fs"
 const cronLogFile = "/var/log/pg-bkup.log"
 const tmpPath = "/tmp/backup"
 const backupCronFile = "/usr/local/bin/backup_cron.sh"
@@ -37,3 +35,12 @@ var sshVars = []string{
 	"SSH_HOST_NAME",
 	"SSH_PORT",
 }
+
+// AwsVars Required environment variables for AWS S3 storage
+var awsVars = []string{
+	"AWS_S3_ENDPOINT",
+	"AWS_S3_BUCKET_NAME",
+	"AWS_ACCESS_KEY",
+	"AWS_SECRET_KEY",
+	"AWS_REGION",
+}
--- a/utils/logger.go
+++ b/utils/logger.go
@@ -55,4 +55,5 @@ func Fatal(msg string, args ...any) {
 		fmt.Printf("%s ERROR: %s\n", currentTime, formattedMessage)
 	}
 	os.Exit(1)
+	os.Kill.Signal()
 }
--- a/utils/s3.go
+++ b/utils/s3.go
@@ -24,8 +24,6 @@ func CreateSession() (*session.Session, error) {
 		"AWS_ACCESS_KEY",
 		"AWS_SECRET_KEY",
 		"AWS_REGION",
-		"AWS_REGION",
-		"AWS_REGION",
 	}

 	endPoint := GetEnvVariable("AWS_S3_ENDPOINT", "S3_ENDPOINT")
@@ -41,8 +39,7 @@ func CreateSession() (*session.Session, error) {

 	err = CheckEnvVars(awsVars)
 	if err != nil {
-		Error("Error checking environment variables\n: %s", err)
-		os.Exit(1)
+		Fatal("Error checking environment variables\n: %s", err)
 	}
 	// Configure to use MinIO Server
 	s3Config := &aws.Config{