From df2757fb1dd6d2b6769f13d214b85239f83c35fe Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Sun, 4 Aug 2024 01:20:30 +0200 Subject: [PATCH] Fix log , refactoring of code --- .github/workflows/release.yml | 20 +++-- Makefile | 20 +++-- go.mod | 2 +- pkg/backup.go | 150 +++++++++++++++++----------------- pkg/encrypt.go | 5 +- pkg/helper.go | 11 ++- pkg/restore.go | 77 ++++++++--------- pkg/scp.go | 25 ++++-- pkg/scripts.go | 14 ++-- pkg/var.go | 16 ++++ utils/logger.go | 58 +++++++++++++ utils/s3.go | 35 ++++++-- utils/utils.go | 47 ++++++----- 13 files changed, 293 insertions(+), 187 deletions(-) create mode 100644 utils/logger.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cfc94f4..fb81ed4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Release on: push: tags: - - v** + - v1.** env: BUILDKIT_IMAGE: jkaninda/pg-bkup jobs: @@ -12,13 +12,13 @@ jobs: packages: write contents: read steps: - - + - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - + - name: Login to DockerHub uses: docker/login-action@v3 with: @@ -30,6 +30,10 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - + name: Get the tag name + id: get_tag_name + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - name: Build and push uses: docker/build-push-action@v3 @@ -38,8 +42,8 @@ jobs: file: "./docker/Dockerfile" platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | - "${{env.BUILDKIT_IMAGE}}:${{env.GITHUB_REF_NAME}}" - "${{env.BUILDKIT_IMAGE}}:latest" - "ghcr.io/${{env.BUILDKIT_IMAGE}}:${{env.GITHUB_REF_NAME}}" - "ghcr.io/${{env.BUILDKIT_IMAGE}}:latest" + "${{env.BUILDKIT_IMAGE}}:${{ env.TAG_NAME }}" + "${{env.BUILDKIT_IMAGE}}:latest" + "ghcr.io/${{env.BUILDKIT_IMAGE}}:${{ env.TAG_NAME }}" + "ghcr.io/${{env.BUILDKIT_IMAGE}}:latest" diff --git a/Makefile b/Makefile index e2823c3..03bd69e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ BINARY_NAME=pg-bkup +IMAGE_NAME=jkaninda/pg-bkup +#IMAGE_NAME=ghcr.io/jkaninda/pg-bkup:v1.0 include .env export run: @@ -14,33 +16,33 @@ compile: GOOS=linux GOARCH=amd64 go build -o bin/${BINARY_NAME}-linux-amd64 . docker-build: - docker build -f docker/Dockerfile -t jkaninda/pg-bkup:latest . + docker build -f docker/Dockerfile -t ${IMAGE_NAME}:latest . docker-run: docker-build - docker run --rm --network internal --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup backup --prune --keep-last 2 + docker run --rm --network web --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --prune --keep-last 2 docker-restore: docker-build - docker run --rm --network internal --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup restore -f ${FILE_NAME} + docker run --rm --network web --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup restore -f ${FILE_NAME} docker-run-scheduled: docker-build - docker run --rm --network internal --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --mode scheduled --period "* * * * *" + docker run --rm --network web --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --mode scheduled --period "* * * * *" docker-run-scheduled-s3: docker-build - docker run --rm --network internal --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage s3 --mode scheduled --path /custom-path --period "* * * * *" + docker run --rm --network web --user 1000:1000 --name pg-bkup -v "./backup:/backup" -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --storage s3 --mode scheduled --path /custom-path --period "* * * * *" docker-run-s3: docker-build - docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage s3 #--path /custom-path + docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --storage s3 #--path /custom-path docker-restore-s3: docker-build - docker run --rm --network internal --privileged --device /dev/fuse --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup restore --storage s3 -f ${FILE_NAME} #--path /custom-path + docker run --rm --network web --privileged --device /dev/fuse --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "ACCESS_KEY=${ACCESS_KEY}" -e "SECRET_KEY=${SECRET_KEY}" -e "BUCKET_NAME=${AWS_S3_BUCKET_NAME}" -e "S3_ENDPOINT=${S3_ENDPOINT}" -e "AWS_REGION=eu2" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup restore --storage s3 -f ${FILE_NAME} #--path /custom-path docker-run-ssh: docker-build - docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" jkaninda/pg-bkup bkup backup --storage ssh + docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" ${IMAGE_NAME} bkup backup --storage ssh docker-restore-ssh: docker-build - docker run --rm --network internal --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" jkaninda/pg-bkup bkup restore --storage ssh -f data_20240731_200104.sql.gz.gpg + docker run --rm --network web --name pg-bkup -e "DB_HOST=${DB_HOST}" -e "DB_NAME=${DB_NAME}" -e "DB_USERNAME=${DB_USERNAME}" -e "DB_PASSWORD=${DB_PASSWORD}" -e "SSH_USER=${SSH_USER}" -e "SSH_HOST_NAME=${SSH_HOST_NAME}" -e "SSH_REMOTE_PATH=${SSH_REMOTE_PATH}" -e "SSH_PASSWORD=${SSH_PASSWORD}" -e "SSH_PORT=${SSH_PORT}" -e "GPG_PASSPHRASE=${GPG_PASSPHRASE}" -e "SSH_IDENTIFY_FILE=${SSH_IDENTIFY_FILE}" ${IMAGE_NAME} bkup restore --storage ssh -f data_20240731_200104.sql.gz.gpg run-docs: cd docs && bundle exec jekyll serve -H 0.0.0.0 -t \ No newline at end of file diff --git a/go.mod b/go.mod index 5333b0e..9700d93 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,8 @@ require ( github.com/hpcloud/tail v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect golang.org/x/crypto v0.18.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/sys v0.22.0 // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect ) + diff --git a/pkg/backup.go b/pkg/backup.go index d994944..3d2b30c 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -55,7 +55,7 @@ func StartBackup(cmd *cobra.Command) { case "ssh", "remote": sshBackup(backupFileName, remotePath, disableCompression, prune, backupRetention, encryption) case "ftp": - utils.Fatalf("Not supported storage type: %s", storage) + utils.Fatal("Not supported storage type: %s", storage) default: localBackup(backupFileName, disableCompression, prune, backupRetention, encryption) } @@ -76,7 +76,7 @@ func scheduledMode() { fmt.Println(" Starting PostgreSQL Bkup... ") fmt.Println("***********************************") utils.Info("Running in Scheduled mode") - utils.Info("Execution period ", os.Getenv("SCHEDULE_PERIOD")) + utils.Info("Execution period %s ", os.Getenv("SCHEDULE_PERIOD")) //Test database connexion utils.TestDatabaseConnection() @@ -93,6 +93,7 @@ func scheduledMode() { utils.Fatal("Failed to start supervisord: %v", err) } utils.Info("Backup job started") + defer func() { if err := cmd.Process.Kill(); err != nil { utils.Info("Failed to kill supervisord process: %v", err) @@ -105,13 +106,14 @@ func scheduledMode() { } t, err := tail.TailFile(cronLogFile, tail.Config{Follow: true}) if err != nil { - utils.Fatalf("Failed to tail file: %v", err) + utils.Fatal("Failed to tail file: %v", err) } // Read and print new lines from the log file for line := range t.Lines { fmt.Println(line.Text) } + } // BackupDatabase backup database @@ -124,73 +126,75 @@ func BackupDatabase(backupFileName string, disableCompression bool) { storagePath = os.Getenv("STORAGE_PATH") utils.Info("Starting database backup...") - if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" { - utils.Fatal("Please make sure all required environment variables for database are set") - } else { - err := os.Setenv("PGPASSWORD", dbPassword) + + err := utils.CheckEnvVars(dbHVars) + if err != nil { + utils.Error("Please make sure all required environment variables for database are set") + utils.Fatal("Error checking environment variables: %s", err) + } + + err = os.Setenv("PGPASSWORD", dbPassword) + if err != nil { + return + } + utils.TestDatabaseConnection() + // Backup Database database + utils.Info("Backing up database...") + + // Verify is compression is disabled + if disableCompression { + // Execute pg_dump + cmd := exec.Command("pg_dump", + "-h", dbHost, + "-p", dbPort, + "-U", dbUserName, + "-d", dbName, + ) + output, err := cmd.Output() if err != nil { - return + log.Fatal(err) } - utils.TestDatabaseConnection() - // Backup Database database - utils.Info("Backing up database...") - - // Verify is compression is disabled - if disableCompression { - // Execute pg_dump - cmd := exec.Command("pg_dump", - "-h", dbHost, - "-p", dbPort, - "-U", dbUserName, - "-d", dbName, - ) - output, err := cmd.Output() - if err != nil { - log.Fatal(err) - } - // save output - file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) - if err != nil { - log.Fatal(err) - } - defer file.Close() - - _, err = file.Write(output) - if err != nil { - log.Fatal(err) - } - - } else { - // Execute pg_dump - cmd := exec.Command("pg_dump", - "-h", dbHost, - "-p", dbPort, - "-U", dbUserName, - "-d", dbName, - ) - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - gzipCmd := exec.Command("gzip") - gzipCmd.Stdin = stdout - // save output - gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) - gzipCmd.Start() - if err != nil { - log.Fatal(err) - } - if err := cmd.Run(); err != nil { - log.Fatal(err) - } - if err := gzipCmd.Wait(); err != nil { - log.Fatal(err) - } - + // save output + file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + _, err = file.Write(output) + if err != nil { + log.Fatal(err) + } + + } else { + // Execute pg_dump + cmd := exec.Command("pg_dump", + "-h", dbHost, + "-p", dbPort, + "-U", dbUserName, + "-d", dbName, + ) + stdout, err := cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + gzipCmd := exec.Command("gzip") + gzipCmd.Stdin = stdout + // save output + gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + gzipCmd.Start() + if err != nil { + log.Fatal(err) + } + if err := cmd.Run(); err != nil { + log.Fatal(err) + } + if err := gzipCmd.Wait(); err != nil { + log.Fatal(err) } - utils.Info("Database has been backed up") } + utils.Info("Database has been backed up") } func localBackup(backupFileName string, disableCompression bool, prune bool, backupRetention int, encrypt bool) { @@ -201,7 +205,7 @@ func localBackup(backupFileName string, disableCompression bool, prune bool, bac encryptBackup(backupFileName) finalFileName = fmt.Sprintf("%s.%s", backupFileName, gpgExtension) } - utils.Info("Backup name is ", finalFileName) + utils.Info("Backup name is %s", finalFileName) moveToBackup(finalFileName, storagePath) //Delete old backup if prune { @@ -220,24 +224,24 @@ func s3Backup(backupFileName string, s3Path string, disableCompression bool, pru finalFileName = fmt.Sprintf("%s.%s", backupFileName, "gpg") } utils.Info("Uploading backup file to S3 storage...") - utils.Info("Backup name is ", finalFileName) + utils.Info("Backup name is %s", finalFileName) err := utils.UploadFileToS3(tmpPath, finalFileName, bucket, s3Path) if err != nil { - utils.Fatalf("Error uploading file to S3: %s ", err) + utils.Fatal("Error uploading file to S3: %s ", err) } //Delete backup file from tmp folder err = utils.DeleteFile(filepath.Join(tmpPath, backupFileName)) if err != nil { - fmt.Println("Error deleting file:", err) + fmt.Println("Error deleting file: ", err) } // Delete old backup if prune { err := utils.DeleteOldBackup(bucket, s3Path, backupRetention) if err != nil { - utils.Fatalf("Error deleting old backup from S3: %s ", err) + utils.Fatal("Error deleting old backup from S3: %s ", err) } } utils.Done("Database has been backed up and uploaded to s3 ") @@ -255,14 +259,14 @@ func sshBackup(backupFileName, remotePath string, disableCompression bool, prune utils.Info("Backup name is ", backupFileName) err := CopyToRemote(finalFileName, remotePath) if err != nil { - utils.Fatalf("Error uploading file to the remote server: %s ", err) + utils.Fatal("Error uploading file to the remote server: %s ", err) } //Delete backup file from tmp folder err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) if err != nil { - fmt.Println("Error deleting file:", err) + utils.Error("Error deleting file:", err) } if prune { @@ -278,7 +282,7 @@ func encryptBackup(backupFileName string) { gpgPassphrase := os.Getenv("GPG_PASSPHRASE") err := Encrypt(filepath.Join(tmpPath, backupFileName), gpgPassphrase) if err != nil { - utils.Fatalf("Error during encrypting backup %s", err) + utils.Fatal("Error during encrypting backup %s", err) } } diff --git a/pkg/encrypt.go b/pkg/encrypt.go index ea74108..017e6b0 100644 --- a/pkg/encrypt.go +++ b/pkg/encrypt.go @@ -1,7 +1,6 @@ package pkg import ( - "fmt" "github.com/jkaninda/pg-bkup/utils" "os" "os/exec" @@ -9,14 +8,13 @@ import ( ) func Decrypt(inputFile string, passphrase string) error { - utils.Info("Decrypting backup file: " + inputFile + " ...") + utils.Info("Decrypting backup file: %s...", inputFile) cmd := exec.Command("gpg", "--batch", "--passphrase", passphrase, "--output", RemoveLastExtension(inputFile), "--decrypt", inputFile) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err := cmd.Run() if err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) return err } @@ -32,7 +30,6 @@ func Encrypt(inputFile string, passphrase string) error { err := cmd.Run() if err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) return err } diff --git a/pkg/helper.go b/pkg/helper.go index a6cabaa..3f86ff2 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -1,7 +1,6 @@ package pkg import ( - "fmt" "github.com/jkaninda/pg-bkup/utils" "os" "path/filepath" @@ -12,7 +11,7 @@ func copyToTmp(sourcePath string, backupFileName string) { //Copy backup from storage to /tmp err := utils.CopyFile(filepath.Join(sourcePath, backupFileName), filepath.Join(tmpPath, backupFileName)) if err != nil { - utils.Fatal("Error copying file ", backupFileName, err) + utils.Fatal("Error copying file %s %v", backupFileName, err) } } @@ -20,16 +19,16 @@ func moveToBackup(backupFileName string, destinationPath string) { //Copy backup from tmp folder to storage destination err := utils.CopyFile(filepath.Join(tmpPath, backupFileName), filepath.Join(destinationPath, backupFileName)) if err != nil { - utils.Fatal("Error copying file ", backupFileName, err) + utils.Fatal("Error copying file %s %v", backupFileName, err) } //Delete backup file from tmp folder err = utils.DeleteFile(filepath.Join(tmpPath, backupFileName)) if err != nil { - fmt.Println("Error deleting file:", err) + utils.Error("Error deleting file: %s", err) } - utils.Done("Database has been backed up and copied to ", filepath.Join(destinationPath, backupFileName)) + utils.Done("Database has been backed up and copied to %s", filepath.Join(destinationPath, backupFileName)) } func deleteOldBackup(retentionDays int) { utils.Info("Deleting old backups...") @@ -44,7 +43,7 @@ func deleteOldBackup(retentionDays int) { if err != nil { utils.Fatal("Error:", err) } else { - utils.Done("File ", filePath, " deleted successfully") + utils.Done("File %s deleted successfully", filePath) } return err } diff --git a/pkg/restore.go b/pkg/restore.go index 042478d..d24fd0a 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -33,7 +33,7 @@ func StartRestore(cmd *cobra.Command) { case "ssh": restoreFromRemote(file, remotePath) case "ftp": - utils.Fatalf("Restore from FTP is not yet supported") + utils.Fatal("Restore from FTP is not yet supported") default: utils.Info("Restore database from local") RestoreDatabase(file) @@ -44,7 +44,7 @@ func restoreFromS3(file, bucket, s3Path string) { utils.Info("Restore database from s3") err := utils.DownloadFile(tmpPath, file, bucket, s3Path) if err != nil { - utils.Fatal("Error download file from s3 ", file, err) + utils.Fatal("Error download file from s3 %s %v ", file, err) } RestoreDatabase(file) } @@ -52,7 +52,7 @@ func restoreFromRemote(file, remotePath string) { utils.Info("Restore database from remote server") err := CopyFromRemote(file, remotePath) if err != nil { - utils.Fatal("Error download file from remote server: ", filepath.Join(remotePath, file), err) + utils.Fatal("Error download file from remote server: %s %v", filepath.Join(remotePath, file), err) } RestoreDatabase(file) } @@ -77,7 +77,7 @@ func RestoreDatabase(file string) { //Decrypt file err := Decrypt(filepath.Join(tmpPath, file), gpgPassphrase) if err != nil { - utils.Fatal("Error decrypting file ", file, err) + utils.Fatal("Error decrypting file %s %v", file, err) } //Update file name file = RemoveLastExtension(file) @@ -85,42 +85,43 @@ func RestoreDatabase(file string) { } - if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" || file == "" { - utils.Fatal("Please make sure all required environment variables are set") - } else { + err := utils.CheckEnvVars(dbHVars) + if err != nil { + utils.Error("Please make sure all required environment variables for database are set") + utils.Fatal("Error checking environment variables: %s", err) + } - if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, file)) { + if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, file)) { - err := os.Setenv("PGPASSWORD", dbPassword) - if err != nil { - return - } - utils.TestDatabaseConnection() - - extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file)) - // Restore from compressed file / .sql.gz - if extension == ".gz" { - str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME") - _, err := exec.Command("bash", "-c", str).Output() - if err != nil { - utils.Fatal("Error, in restoring the database ", err) - } - utils.Done("Database has been restored") - - } else if extension == ".sql" { - //Restore from sql file - str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME") - _, err := exec.Command("bash", "-c", str).Output() - if err != nil { - utils.Fatal("Error in restoring the database", err) - } - utils.Done("Database has been restored") - } else { - utils.Fatal("Unknown file extension ", extension) - } - - } else { - utils.Fatal("File not found in ", fmt.Sprintf("%s/%s", tmpPath, file)) + err := os.Setenv("PGPASSWORD", dbPassword) + if err != nil { + return } + utils.TestDatabaseConnection() + + extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file)) + // Restore from compressed file / .sql.gz + if extension == ".gz" { + str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME") + _, err := exec.Command("bash", "-c", str).Output() + if err != nil { + utils.Fatal("Error, in restoring the database %v", err) + } + utils.Done("Database has been restored") + + } else if extension == ".sql" { + //Restore from sql file + str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME") + _, err := exec.Command("bash", "-c", str).Output() + if err != nil { + utils.Fatal("Error in restoring the database %v", err) + } + utils.Done("Database has been restored") + } else { + utils.Fatal("Unknown file extension: %s", extension) + } + + } else { + utils.Fatal("File not found in %s", fmt.Sprintf("%s/%s", tmpPath, file)) } } diff --git a/pkg/scp.go b/pkg/scp.go index ce73e12..f91b658 100644 --- a/pkg/scp.go +++ b/pkg/scp.go @@ -8,7 +8,6 @@ import ( "github.com/bramvdbogaerde/go-scp/auth" "github.com/jkaninda/pg-bkup/utils" "golang.org/x/crypto/ssh" - "golang.org/x/exp/slog" "os" "path/filepath" ) @@ -20,15 +19,20 @@ func CopyToRemote(fileName, remotePath string) error { sshPort := os.Getenv("SSH_PORT") sshIdentifyFile := os.Getenv("SSH_IDENTIFY_FILE") + err := utils.CheckEnvVars(sshVars) + if err != nil { + utils.Fatal(fmt.Sprintf("Error checking environment variables\n: %s", err)) + } + clientConfig, _ := auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey()) if sshIdentifyFile != "" && utils.FileExists(sshIdentifyFile) { clientConfig, _ = auth.PrivateKey(sshUser, sshIdentifyFile, ssh.InsecureIgnoreHostKey()) } else { if sshPassword == "" { - return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty\n") + return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty") } - slog.Warn("Accessing the remote server using password, password is not recommended\n") + utils.Warn("Accessing the remote server using password, password is not recommended") clientConfig, _ = auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey()) } @@ -36,9 +40,9 @@ func CopyToRemote(fileName, remotePath string) error { client := scp.NewClient(fmt.Sprintf("%s:%s", sshHostName, sshPort), &clientConfig) // Connect to the remote server - err := client.Connect() + err = client.Connect() if err != nil { - return errors.New("Couldn't establish a connection to the remote server\n") + return errors.New("couldn't establish a connection to the remote server") } // Open a file @@ -64,6 +68,11 @@ func CopyFromRemote(fileName, remotePath string) error { sshPort := os.Getenv("SSH_PORT") sshIdentifyFile := os.Getenv("SSH_IDENTIFY_FILE") + err := utils.CheckEnvVars(sshVars) + if err != nil { + utils.Fatal("Error checking environment variables\n: %s", err) + } + clientConfig, _ := auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey()) if sshIdentifyFile != "" && utils.FileExists(sshIdentifyFile) { clientConfig, _ = auth.PrivateKey(sshUser, sshIdentifyFile, ssh.InsecureIgnoreHostKey()) @@ -72,7 +81,7 @@ func CopyFromRemote(fileName, remotePath string) error { if sshPassword == "" { return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty\n") } - slog.Warn("Accessing the remote server using password, password is not recommended\n") + utils.Warn("Accessing the remote server using password, password is not recommended\n") clientConfig, _ = auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey()) } @@ -80,7 +89,7 @@ func CopyFromRemote(fileName, remotePath string) error { client := scp.NewClient(fmt.Sprintf("%s:%s", sshHostName, sshPort), &clientConfig) // Connect to the remote server - err := client.Connect() + err = client.Connect() if err != nil { return errors.New("Couldn't establish a connection to the remote server\n") } @@ -96,7 +105,7 @@ func CopyFromRemote(fileName, remotePath string) error { err = client.CopyFromRemote(context.Background(), file, filepath.Join(remotePath, fileName)) if err != nil { - fmt.Println("Error while copying file ", err) + utils.Error("Error while copying file %s ", err) return err } return nil diff --git a/pkg/scripts.go b/pkg/scripts.go index b056804..8370273 100644 --- a/pkg/scripts.go +++ b/pkg/scripts.go @@ -15,7 +15,7 @@ func CreateCrontabScript(disableCompression bool, storage string) { //task := "/usr/local/bin/backup_cron.sh" touchCmd := exec.Command("touch", backupCronFile) if err := touchCmd.Run(); err != nil { - utils.Fatalf("Error creating file %s: %v\n", backupCronFile, err) + utils.Fatal("Error creating file %s: %v\n", backupCronFile, err) } var disableC = "" if disableCompression { @@ -37,36 +37,36 @@ bkup backup --dbname %s --port %s %v } if err := utils.WriteToFile(backupCronFile, scriptContent); err != nil { - utils.Fatalf("Error writing to %s: %v\n", backupCronFile, err) + utils.Fatal("Error writing to %s: %v\n", backupCronFile, err) } chmodCmd := exec.Command("chmod", "+x", "/usr/local/bin/backup_cron.sh") if err := chmodCmd.Run(); err != nil { - utils.Fatalf("Error changing permissions of %s: %v\n", backupCronFile, err) + utils.Fatal("Error changing permissions of %s: %v\n", backupCronFile, err) } lnCmd := exec.Command("ln", "-s", "/usr/local/bin/backup_cron.sh", "/usr/local/bin/backup_cron") if err := lnCmd.Run(); err != nil { - utils.Fatalf("Error creating symbolic link: %v\n", err) + utils.Fatal("Error creating symbolic link: %v\n", err) } touchLogCmd := exec.Command("touch", cronLogFile) if err := touchLogCmd.Run(); err != nil { - utils.Fatalf("Error creating file %s: %v\n", cronLogFile, err) + utils.Fatal("Error creating file %s: %v\n", cronLogFile, err) } cronJob := "/etc/cron.d/backup_cron" touchCronCmd := exec.Command("touch", cronJob) if err := touchCronCmd.Run(); err != nil { - utils.Fatalf("Error creating file %s: %v\n", cronJob, err) + utils.Fatal("Error creating file %s: %v\n", cronJob, err) } cronContent := fmt.Sprintf(`%s root exec /bin/bash -c ". /run/supervisord.env; /usr/local/bin/backup_cron.sh >> %s" `, os.Getenv("SCHEDULE_PERIOD"), cronLogFile) if err := utils.WriteToFile(cronJob, cronContent); err != nil { - utils.Fatalf("Error writing to %s: %v\n", cronJob, err) + utils.Fatal("Error writing to %s: %v\n", cronJob, err) } utils.ChangePermission("/etc/cron.d/backup_cron", 0644) diff --git a/pkg/var.go b/pkg/var.go index 542ed95..fb2619a 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -21,3 +21,19 @@ var ( disableCompression = false encryption = false ) + +// dbHVars Required environment variables for database +var dbHVars = []string{ + "DB_HOST", + "DB_PASSWORD", + "DB_USERNAME", + "DB_NAME", +} + +// sshVars Required environment variables for SSH remote server storage +var sshVars = []string{ + "SSH_USER", + "SSH_REMOTE_PATH", + "SSH_HOST_NAME", + "SSH_PORT", +} diff --git a/utils/logger.go b/utils/logger.go new file mode 100644 index 0000000..990638a --- /dev/null +++ b/utils/logger.go @@ -0,0 +1,58 @@ +package utils + +import ( + "fmt" + "os" + "time" +) + +var currentTime = time.Now().Format("2006/01/02 15:04:05") + +// Info message +func Info(msg string, args ...any) { + formattedMessage := fmt.Sprintf(msg, args...) + if len(args) == 0 { + fmt.Printf("%s INFO: %s\n", currentTime, msg) + } else { + fmt.Printf("%s INFO: %s\n", currentTime, formattedMessage) + } +} + +// Warn Warning message +func Warn(msg string, args ...any) { + formattedMessage := fmt.Sprintf(msg, args...) + if len(args) == 0 { + fmt.Printf("%s WARN: %s\n", currentTime, msg) + } else { + fmt.Printf("%s WARN: %s\n", currentTime, formattedMessage) + } +} + +// Error error message +func Error(msg string, args ...any) { + formattedMessage := fmt.Sprintf(msg, args...) + if len(args) == 0 { + fmt.Printf("%s ERROR: %s\n", currentTime, msg) + } else { + fmt.Printf("%s ERROR: %s\n", currentTime, formattedMessage) + } +} +func Done(msg string, args ...any) { + formattedMessage := fmt.Sprintf(msg, args...) + if len(args) == 0 { + fmt.Printf("%s INFO: %s\n", currentTime, msg) + } else { + fmt.Printf("%s INFO: %s\n", currentTime, formattedMessage) + } +} + +func Fatal(msg string, args ...any) { + // Fatal logs an error message and exits the program. + formattedMessage := fmt.Sprintf(msg, args...) + if len(args) == 0 { + fmt.Printf("%s ERROR: %s\n", currentTime, msg) + } else { + fmt.Printf("%s ERROR: %s\n", currentTime, formattedMessage) + } + os.Exit(1) +} diff --git a/utils/s3.go b/utils/s3.go index 984a505..5ff27bd 100644 --- a/utils/s3.go +++ b/utils/s3.go @@ -2,13 +2,11 @@ package utils import ( "bytes" - "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" - "log" "net/http" "os" "path/filepath" @@ -19,13 +17,32 @@ import ( // CreateSession creates a new AWS session func CreateSession() (*session.Session, error) { + // AwsVars Required environment variables for AWS S3 storage + var awsVars = []string{ + "AWS_S3_ENDPOINT", + "AWS_S3_BUCKET_NAME", + "AWS_ACCESS_KEY", + "AWS_SECRET_KEY", + "AWS_REGION", + "AWS_REGION", + "AWS_REGION", + } + endPoint := GetEnvVariable("AWS_S3_ENDPOINT", "S3_ENDPOINT") accessKey := GetEnvVariable("AWS_ACCESS_KEY", "ACCESS_KEY") secretKey := GetEnvVariable("AWS_SECRET_KEY", "SECRET_KEY") + _ = GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME") + region := os.Getenv("AWS_REGION") awsDisableSsl, err := strconv.ParseBool(os.Getenv("AWS_DISABLE_SSL")) if err != nil { - Fatalf("Unable to parse AWS_DISABLE_SSL env var: %s", err) + Fatal("Unable to parse AWS_DISABLE_SSL env var: %s", err) + } + + err = CheckEnvVars(awsVars) + if err != nil { + Error("Error checking environment variables\n: %s", err) + os.Exit(1) } // Configure to use MinIO Server s3Config := &aws.Config{ @@ -88,7 +105,7 @@ func DownloadFile(destinationPath, key, bucket, prefix string) error { Info("Download backup from S3 storage...") file, err := os.Create(filepath.Join(destinationPath, key)) if err != nil { - fmt.Println("Failed to create file", err) + Error("Failed to create file", err) return err } defer file.Close() @@ -102,7 +119,7 @@ func DownloadFile(destinationPath, key, bucket, prefix string) error { Key: aws.String(objectKey), }) if err != nil { - fmt.Println("Failed to download file", err) + Error("Failed to download file", err) return err } Info("Backup downloaded: ", file.Name(), " bytes size ", numBytes) @@ -135,18 +152,18 @@ func DeleteOldBackup(bucket, prefix string, retention int) error { Key: object.Key, }) if err != nil { - log.Printf("Failed to delete object %s: %v", *object.Key, err) + Info("Failed to delete object %s: %v", *object.Key, err) } else { - fmt.Printf("Deleted object %s\n", *object.Key) + Info("Deleted object %s\n", *object.Key) } } } return !lastPage }) if err != nil { - log.Fatalf("Failed to list objects: %v", err) + Error("Failed to list objects: %v", err) } - fmt.Println("Finished deleting old files.") + Info("Finished deleting old files.") return nil } diff --git a/utils/utils.go b/utils/utils.go index ecae7cd..f208b2f 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -10,31 +10,12 @@ import ( "bytes" "fmt" "github.com/spf13/cobra" - "golang.org/x/exp/slog" "io" "io/fs" "os" "os/exec" ) -func Info(v ...any) { - fmt.Println("⒤ ", fmt.Sprint(v...)) -} -func Worn(msg string, v ...any) { - slog.Warn(fmt.Sprintf(msg, v)) -} -func Done(v ...any) { - fmt.Println("✔ ", fmt.Sprint(v...)) -} -func Fatal(v ...any) { - fmt.Println("✘ ", fmt.Sprint(v...)) - os.Exit(1) -} -func Fatalf(msg string, v ...any) { - fmt.Printf("✘ "+msg, v...) - os.Exit(1) -} - func FileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) { @@ -91,7 +72,7 @@ func CopyFile(src, dst string) error { } func ChangePermission(filePath string, mod int) { if err := os.Chmod(filePath, fs.FileMode(mod)); err != nil { - Fatalf("Error changing permissions of %s: %v\n", filePath, err) + Fatal("Error changing permissions of %s: %v\n", filePath, err) } } @@ -145,7 +126,7 @@ func TestDatabaseConnection() { // Run the command and capture any errors err = cmd.Run() if err != nil { - fmt.Printf("Error running psql command: %v\nOutput: %s\n", err, out.String()) + Error("Error running psql command: %v\nOutput: %s\n", err, out.String()) return } Info("Successfully connected to database") @@ -187,11 +168,29 @@ func GetEnvVariable(envName, oldEnvName string) string { if value == "" { value = os.Getenv(oldEnvName) if value != "" { - slog.Warn(fmt.Sprintf("%s is deprecated, please use %s instead!\n", oldEnvName, envName)) - + err := os.Setenv(envName, value) + if err != nil { + return value + } + Warn("%s is deprecated, please use %s instead! ", oldEnvName, envName) } } return value } -func ShowHistory() { + +// CheckEnvVars checks if all the specified environment variables are set +func CheckEnvVars(vars []string) error { + missingVars := []string{} + + for _, v := range vars { + if os.Getenv(v) == "" { + missingVars = append(missingVars, v) + } + } + + if len(missingVars) > 0 { + return fmt.Errorf("missing environment variables: %v", missingVars) + } + + return nil }