jkaninda/pg-bkup
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Fix log , refactoring of code

Browse Source
This commit is contained in:
Jonas Kaninda
2024-08-04 01:20:30 +02:00
parent a81c7aa7b4
commit df2757fb1d
13 changed files with 293 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
pkg/backup.go
View File

@@ -55,7 +55,7 @@ func StartBackup(cmd *cobra.Command) {
case "ssh", "remote":
sshBackup(backupFileName, remotePath, disableCompression, prune, backupRetention, encryption)
case "ftp":
utils.Fatalf("Not supported storage type: %s", storage)
utils.Fatal("Not supported storage type: %s", storage)
default:
localBackup(backupFileName, disableCompression, prune, backupRetention, encryption)
}
@@ -76,7 +76,7 @@ func scheduledMode() {
fmt.Println(" Starting PostgreSQL Bkup... ")
fmt.Println("***********************************")
utils.Info("Running in Scheduled mode")
utils.Info("Execution period ", os.Getenv("SCHEDULE_PERIOD"))
utils.Info("Execution period %s ", os.Getenv("SCHEDULE_PERIOD"))
//Test database connexion
utils.TestDatabaseConnection()
@@ -93,6 +93,7 @@ func scheduledMode() {
utils.Fatal("Failed to start supervisord: %v", err)
}
utils.Info("Backup job started")
defer func() {
if err := cmd.Process.Kill(); err != nil {
utils.Info("Failed to kill supervisord process: %v", err)
@@ -105,13 +106,14 @@ func scheduledMode() {
}
t, err := tail.TailFile(cronLogFile, tail.Config{Follow: true})
if err != nil {
utils.Fatalf("Failed to tail file: %v", err)
utils.Fatal("Failed to tail file: %v", err)
}
// Read and print new lines from the log file
for line := range t.Lines {
fmt.Println(line.Text)
}
}
// BackupDatabase backup database
@@ -124,73 +126,75 @@ func BackupDatabase(backupFileName string, disableCompression bool) {
storagePath = os.Getenv("STORAGE_PATH")
utils.Info("Starting database backup...")
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" {
utils.Fatal("Please make sure all required environment variables for database are set")
} else {
err := os.Setenv("PGPASSWORD", dbPassword)
err := utils.CheckEnvVars(dbHVars)
if err != nil {
utils.Error("Please make sure all required environment variables for database are set")
utils.Fatal("Error checking environment variables: %s", err)
}
err = os.Setenv("PGPASSWORD", dbPassword)
if err != nil {
return
}
utils.TestDatabaseConnection()
// Backup Database database
utils.Info("Backing up database...")
// Verify is compression is disabled
if disableCompression {
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
"-p", dbPort,
"-U", dbUserName,
"-d", dbName,
)
output, err := cmd.Output()
if err != nil {
return
log.Fatal(err)
}
utils.TestDatabaseConnection()
// Backup Database database
utils.Info("Backing up database...")
// Verify is compression is disabled
if disableCompression {
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
"-p", dbPort,
"-U", dbUserName,
"-d", dbName,
)
output, err := cmd.Output()
if err != nil {
log.Fatal(err)
}
// save output
file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
if err != nil {
log.Fatal(err)
}
defer file.Close()
_, err = file.Write(output)
if err != nil {
log.Fatal(err)
}
} else {
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
"-p", dbPort,
"-U", dbUserName,
"-d", dbName,
)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
gzipCmd := exec.Command("gzip")
gzipCmd.Stdin = stdout
// save output
gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
gzipCmd.Start()
if err != nil {
log.Fatal(err)
}
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
if err := gzipCmd.Wait(); err != nil {
log.Fatal(err)
}
// save output
file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
if err != nil {
log.Fatal(err)
}
defer file.Close()
_, err = file.Write(output)
if err != nil {
log.Fatal(err)
}
} else {
// Execute pg_dump
cmd := exec.Command("pg_dump",
"-h", dbHost,
"-p", dbPort,
"-U", dbUserName,
"-d", dbName,
)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatal(err)
}
gzipCmd := exec.Command("gzip")
gzipCmd.Stdin = stdout
// save output
gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName))
gzipCmd.Start()
if err != nil {
log.Fatal(err)
}
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
if err := gzipCmd.Wait(); err != nil {
log.Fatal(err)
}
utils.Info("Database has been backed up")
}
utils.Info("Database has been backed up")
}
func localBackup(backupFileName string, disableCompression bool, prune bool, backupRetention int, encrypt bool) {
@@ -201,7 +205,7 @@ func localBackup(backupFileName string, disableCompression bool, prune bool, bac
encryptBackup(backupFileName)
finalFileName = fmt.Sprintf("%s.%s", backupFileName, gpgExtension)
}
utils.Info("Backup name is ", finalFileName)
utils.Info("Backup name is %s", finalFileName)
moveToBackup(finalFileName, storagePath)
//Delete old backup
if prune {
@@ -220,24 +224,24 @@ func s3Backup(backupFileName string, s3Path string, disableCompression bool, pru
finalFileName = fmt.Sprintf("%s.%s", backupFileName, "gpg")
}
utils.Info("Uploading backup file to S3 storage...")
utils.Info("Backup name is ", finalFileName)
utils.Info("Backup name is %s", finalFileName)
err := utils.UploadFileToS3(tmpPath, finalFileName, bucket, s3Path)
if err != nil {
utils.Fatalf("Error uploading file to S3: %s ", err)
utils.Fatal("Error uploading file to S3: %s ", err)
}
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, backupFileName))
if err != nil {
fmt.Println("Error deleting file:", err)
fmt.Println("Error deleting file: ", err)
}
// Delete old backup
if prune {
err := utils.DeleteOldBackup(bucket, s3Path, backupRetention)
if err != nil {
utils.Fatalf("Error deleting old backup from S3: %s ", err)
utils.Fatal("Error deleting old backup from S3: %s ", err)
}
}
utils.Done("Database has been backed up and uploaded to s3 ")
@@ -255,14 +259,14 @@ func sshBackup(backupFileName, remotePath string, disableCompression bool, prune
utils.Info("Backup name is ", backupFileName)
err := CopyToRemote(finalFileName, remotePath)
if err != nil {
utils.Fatalf("Error uploading file to the remote server: %s ", err)
utils.Fatal("Error uploading file to the remote server: %s ", err)
}
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName))
if err != nil {
fmt.Println("Error deleting file:", err)
utils.Error("Error deleting file:", err)
}
if prune {
@@ -278,7 +282,7 @@ func encryptBackup(backupFileName string) {
gpgPassphrase := os.Getenv("GPG_PASSPHRASE")
err := Encrypt(filepath.Join(tmpPath, backupFileName), gpgPassphrase)
if err != nil {
utils.Fatalf("Error during encrypting backup %s", err)
utils.Fatal("Error during encrypting backup %s", err)
}
}

5
pkg/encrypt.go
View File

@@ -1,7 +1,6 @@
package pkg
import (
"fmt"
"github.com/jkaninda/pg-bkup/utils"
"os"
"os/exec"
@@ -9,14 +8,13 @@ import (
)
func Decrypt(inputFile string, passphrase string) error {
utils.Info("Decrypting backup file: " + inputFile + " ...")
utils.Info("Decrypting backup file: %s...", inputFile)
cmd := exec.Command("gpg", "--batch", "--passphrase", passphrase, "--output", RemoveLastExtension(inputFile), "--decrypt", inputFile)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
return err
}
@@ -32,7 +30,6 @@ func Encrypt(inputFile string, passphrase string) error {
err := cmd.Run()
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
return err
}

11
pkg/helper.go
View File

@@ -1,7 +1,6 @@
package pkg
import (
"fmt"
"github.com/jkaninda/pg-bkup/utils"
"os"
"path/filepath"
@@ -12,7 +11,7 @@ func copyToTmp(sourcePath string, backupFileName string) {
//Copy backup from storage to /tmp
err := utils.CopyFile(filepath.Join(sourcePath, backupFileName), filepath.Join(tmpPath, backupFileName))
if err != nil {
utils.Fatal("Error copying file ", backupFileName, err)
utils.Fatal("Error copying file %s %v", backupFileName, err)
}
}
@@ -20,16 +19,16 @@ func moveToBackup(backupFileName string, destinationPath string) {
//Copy backup from tmp folder to storage destination
err := utils.CopyFile(filepath.Join(tmpPath, backupFileName), filepath.Join(destinationPath, backupFileName))
if err != nil {
utils.Fatal("Error copying file ", backupFileName, err)
utils.Fatal("Error copying file %s %v", backupFileName, err)
}
//Delete backup file from tmp folder
err = utils.DeleteFile(filepath.Join(tmpPath, backupFileName))
if err != nil {
fmt.Println("Error deleting file:", err)
utils.Error("Error deleting file: %s", err)
}
utils.Done("Database has been backed up and copied to ", filepath.Join(destinationPath, backupFileName))
utils.Done("Database has been backed up and copied to %s", filepath.Join(destinationPath, backupFileName))
}
func deleteOldBackup(retentionDays int) {
utils.Info("Deleting old backups...")
@@ -44,7 +43,7 @@ func deleteOldBackup(retentionDays int) {
if err != nil {
utils.Fatal("Error:", err)
} else {
utils.Done("File ", filePath, " deleted successfully")
utils.Done("File %s deleted successfully", filePath)
}
return err
}

77
pkg/restore.go
View File

@@ -33,7 +33,7 @@ func StartRestore(cmd *cobra.Command) {
case "ssh":
restoreFromRemote(file, remotePath)
case "ftp":
utils.Fatalf("Restore from FTP is not yet supported")
utils.Fatal("Restore from FTP is not yet supported")
default:
utils.Info("Restore database from local")
RestoreDatabase(file)
@@ -44,7 +44,7 @@ func restoreFromS3(file, bucket, s3Path string) {
utils.Info("Restore database from s3")
err := utils.DownloadFile(tmpPath, file, bucket, s3Path)
if err != nil {
utils.Fatal("Error download file from s3 ", file, err)
utils.Fatal("Error download file from s3 %s %v ", file, err)
}
RestoreDatabase(file)
}
@@ -52,7 +52,7 @@ func restoreFromRemote(file, remotePath string) {
utils.Info("Restore database from remote server")
err := CopyFromRemote(file, remotePath)
if err != nil {
utils.Fatal("Error download file from remote server: ", filepath.Join(remotePath, file), err)
utils.Fatal("Error download file from remote server: %s %v", filepath.Join(remotePath, file), err)
}
RestoreDatabase(file)
}
@@ -77,7 +77,7 @@ func RestoreDatabase(file string) {
//Decrypt file
err := Decrypt(filepath.Join(tmpPath, file), gpgPassphrase)
if err != nil {
utils.Fatal("Error decrypting file ", file, err)
utils.Fatal("Error decrypting file %s %v", file, err)
}
//Update file name
file = RemoveLastExtension(file)
@@ -85,42 +85,43 @@ func RestoreDatabase(file string) {
}
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" || file == "" {
utils.Fatal("Please make sure all required environment variables are set")
} else {
err := utils.CheckEnvVars(dbHVars)
if err != nil {
utils.Error("Please make sure all required environment variables for database are set")
utils.Fatal("Error checking environment variables: %s", err)
}
if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, file)) {
if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, file)) {
err := os.Setenv("PGPASSWORD", dbPassword)
if err != nil {
return
}
utils.TestDatabaseConnection()
extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file))
// Restore from compressed file / .sql.gz
if extension == ".gz" {
str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error, in restoring the database ", err)
}
utils.Done("Database has been restored")
} else if extension == ".sql" {
//Restore from sql file
str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error in restoring the database", err)
}
utils.Done("Database has been restored")
} else {
utils.Fatal("Unknown file extension ", extension)
}
} else {
utils.Fatal("File not found in ", fmt.Sprintf("%s/%s", tmpPath, file))
err := os.Setenv("PGPASSWORD", dbPassword)
if err != nil {
return
}
utils.TestDatabaseConnection()
extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file))
// Restore from compressed file / .sql.gz
if extension == ".gz" {
str := "zcat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error, in restoring the database %v", err)
}
utils.Done("Database has been restored")
} else if extension == ".sql" {
//Restore from sql file
str := "cat " + fmt.Sprintf("%s/%s", tmpPath, file) + " | psql -h " + os.Getenv("DB_HOST") + " -p " + os.Getenv("DB_PORT") + " -U " + os.Getenv("DB_USERNAME") + " -v -d " + os.Getenv("DB_NAME")
_, err := exec.Command("bash", "-c", str).Output()
if err != nil {
utils.Fatal("Error in restoring the database %v", err)
}
utils.Done("Database has been restored")
} else {
utils.Fatal("Unknown file extension: %s", extension)
}
} else {
utils.Fatal("File not found in %s", fmt.Sprintf("%s/%s", tmpPath, file))
}
}

25
pkg/scp.go
View File

@@ -8,7 +8,6 @@ import (
"github.com/bramvdbogaerde/go-scp/auth"
"github.com/jkaninda/pg-bkup/utils"
"golang.org/x/crypto/ssh"
"golang.org/x/exp/slog"
"os"
"path/filepath"
)
@@ -20,15 +19,20 @@ func CopyToRemote(fileName, remotePath string) error {
sshPort := os.Getenv("SSH_PORT")
sshIdentifyFile := os.Getenv("SSH_IDENTIFY_FILE")
err := utils.CheckEnvVars(sshVars)
if err != nil {
utils.Fatal(fmt.Sprintf("Error checking environment variables\n: %s", err))
}
clientConfig, _ := auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey())
if sshIdentifyFile != "" && utils.FileExists(sshIdentifyFile) {
clientConfig, _ = auth.PrivateKey(sshUser, sshIdentifyFile, ssh.InsecureIgnoreHostKey())
} else {
if sshPassword == "" {
return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty\n")
return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty")
}
slog.Warn("Accessing the remote server using password, password is not recommended\n")
utils.Warn("Accessing the remote server using password, password is not recommended")
clientConfig, _ = auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey())
}
@@ -36,9 +40,9 @@ func CopyToRemote(fileName, remotePath string) error {
client := scp.NewClient(fmt.Sprintf("%s:%s", sshHostName, sshPort), &clientConfig)
// Connect to the remote server
err := client.Connect()
err = client.Connect()
if err != nil {
return errors.New("Couldn't establish a connection to the remote server\n")
return errors.New("couldn't establish a connection to the remote server")
}
// Open a file
@@ -64,6 +68,11 @@ func CopyFromRemote(fileName, remotePath string) error {
sshPort := os.Getenv("SSH_PORT")
sshIdentifyFile := os.Getenv("SSH_IDENTIFY_FILE")
err := utils.CheckEnvVars(sshVars)
if err != nil {
utils.Fatal("Error checking environment variables\n: %s", err)
}
clientConfig, _ := auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey())
if sshIdentifyFile != "" && utils.FileExists(sshIdentifyFile) {
clientConfig, _ = auth.PrivateKey(sshUser, sshIdentifyFile, ssh.InsecureIgnoreHostKey())
@@ -72,7 +81,7 @@ func CopyFromRemote(fileName, remotePath string) error {
if sshPassword == "" {
return errors.New("SSH_PASSWORD environment variable is required if SSH_IDENTIFY_FILE is empty\n")
}
slog.Warn("Accessing the remote server using password, password is not recommended\n")
utils.Warn("Accessing the remote server using password, password is not recommended\n")
clientConfig, _ = auth.PasswordKey(sshUser, sshPassword, ssh.InsecureIgnoreHostKey())
}
@@ -80,7 +89,7 @@ func CopyFromRemote(fileName, remotePath string) error {
client := scp.NewClient(fmt.Sprintf("%s:%s", sshHostName, sshPort), &clientConfig)
// Connect to the remote server
err := client.Connect()
err = client.Connect()
if err != nil {
return errors.New("Couldn't establish a connection to the remote server\n")
}
@@ -96,7 +105,7 @@ func CopyFromRemote(fileName, remotePath string) error {
err = client.CopyFromRemote(context.Background(), file, filepath.Join(remotePath, fileName))
if err != nil {
fmt.Println("Error while copying file ", err)
utils.Error("Error while copying file %s ", err)
return err
}
return nil

14
pkg/scripts.go
View File

@@ -15,7 +15,7 @@ func CreateCrontabScript(disableCompression bool, storage string) {
//task := "/usr/local/bin/backup_cron.sh"
touchCmd := exec.Command("touch", backupCronFile)
if err := touchCmd.Run(); err != nil {
utils.Fatalf("Error creating file %s: %v\n", backupCronFile, err)
utils.Fatal("Error creating file %s: %v\n", backupCronFile, err)
}
var disableC = ""
if disableCompression {
@@ -37,36 +37,36 @@ bkup backup --dbname %s --port %s %v
}
if err := utils.WriteToFile(backupCronFile, scriptContent); err != nil {
utils.Fatalf("Error writing to %s: %v\n", backupCronFile, err)
utils.Fatal("Error writing to %s: %v\n", backupCronFile, err)
}
chmodCmd := exec.Command("chmod", "+x", "/usr/local/bin/backup_cron.sh")
if err := chmodCmd.Run(); err != nil {
utils.Fatalf("Error changing permissions of %s: %v\n", backupCronFile, err)
utils.Fatal("Error changing permissions of %s: %v\n", backupCronFile, err)
}
lnCmd := exec.Command("ln", "-s", "/usr/local/bin/backup_cron.sh", "/usr/local/bin/backup_cron")
if err := lnCmd.Run(); err != nil {
utils.Fatalf("Error creating symbolic link: %v\n", err)
utils.Fatal("Error creating symbolic link: %v\n", err)
}
touchLogCmd := exec.Command("touch", cronLogFile)
if err := touchLogCmd.Run(); err != nil {
utils.Fatalf("Error creating file %s: %v\n", cronLogFile, err)
utils.Fatal("Error creating file %s: %v\n", cronLogFile, err)
}
cronJob := "/etc/cron.d/backup_cron"
touchCronCmd := exec.Command("touch", cronJob)
if err := touchCronCmd.Run(); err != nil {
utils.Fatalf("Error creating file %s: %v\n", cronJob, err)
utils.Fatal("Error creating file %s: %v\n", cronJob, err)
}
cronContent := fmt.Sprintf(`%s root exec /bin/bash -c ". /run/supervisord.env; /usr/local/bin/backup_cron.sh >> %s"
`, os.Getenv("SCHEDULE_PERIOD"), cronLogFile)
if err := utils.WriteToFile(cronJob, cronContent); err != nil {
utils.Fatalf("Error writing to %s: %v\n", cronJob, err)
utils.Fatal("Error writing to %s: %v\n", cronJob, err)
}
utils.ChangePermission("/etc/cron.d/backup_cron", 0644)

16
pkg/var.go
View File

@@ -21,3 +21,19 @@ var (
disableCompression = false
encryption = false
)
// dbHVars Required environment variables for database
var dbHVars = []string{
"DB_HOST",
"DB_PASSWORD",
"DB_USERNAME",
"DB_NAME",
}
// sshVars Required environment variables for SSH remote server storage
var sshVars = []string{
"SSH_USER",
"SSH_REMOTE_PATH",
"SSH_HOST_NAME",
"SSH_PORT",
}