mirror of
https://github.com/jkaninda/mysql-bkup.git
synced 2025-12-06 21:49:40 +01:00
refactor: move backup, restore, s3fs tasks in pkg folder
This commit is contained in:
310
main.go
310
main.go
@@ -8,14 +8,11 @@ package main
|
||||
**/
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/jkaninda/mysql-bkup/pkg"
|
||||
"github.com/jkaninda/mysql-bkup/utils"
|
||||
flag "github.com/spf13/pflag"
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
var appVersion string = os.Getenv("VERSION")
|
||||
@@ -41,25 +38,25 @@ var (
|
||||
s3fsPasswdFile string = "/etc/passwd-s3fs"
|
||||
disableCompression bool = false
|
||||
startBackup bool = true
|
||||
outputContent string = ""
|
||||
timeout int = 30
|
||||
period string = "0 1 * * *"
|
||||
|
||||
timeout int = 30
|
||||
period string = "0 1 * * *"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var (
|
||||
operationFlag = flag.StringP("operation", "o", "backup", "Set operation")
|
||||
storageFlag = flag.StringP("storage", "s", "local", "Set storage. local or s3")
|
||||
fileFlag = flag.StringP("file", "f", "", "Set file name")
|
||||
pathFlag = flag.StringP("path", "P", "/mysql-bkup", "Set s3 path, without file name")
|
||||
dbnameFlag = flag.StringP("dbname", "d", "", "Set database name")
|
||||
modeFlag = flag.StringP("mode", "m", "default", "Set execution mode. default or scheduled")
|
||||
periodFlag = flag.StringP("period", "", "0 1 * * *", "Set schedule period time")
|
||||
timeoutFlag = flag.IntP("timeout", "t", 30, "Set timeout")
|
||||
operationFlag = flag.StringP("operation", "o", "backup", "Operation")
|
||||
storageFlag = flag.StringP("storage", "s", "local", "Storage, local or s3")
|
||||
fileFlag = flag.StringP("file", "f", "", "File name")
|
||||
pathFlag = flag.StringP("path", "P", "/mysql-bkup", "S3 path, without file name")
|
||||
dbnameFlag = flag.StringP("dbname", "d", "", "Database name")
|
||||
modeFlag = flag.StringP("mode", "m", "default", "Execution mode. default or scheduled")
|
||||
periodFlag = flag.StringP("period", "", "0 1 * * *", "Schedule period time")
|
||||
timeoutFlag = flag.IntP("timeout", "t", 30, "Timeout (in seconds) to stop database connexion")
|
||||
disableCompressionFlag = flag.BoolP("disable-compression", "", false, "Disable backup compression")
|
||||
portFlag = flag.IntP("port", "p", 3306, "Set database port")
|
||||
portFlag = flag.IntP("port", "p", 3306, "Database port")
|
||||
helpFlag = flag.BoolP("help", "h", false, "Print this help message")
|
||||
versionFlag = flag.BoolP("version", "v", false, "shows version information")
|
||||
versionFlag = flag.BoolP("version", "v", false, "Version information")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
@@ -75,7 +72,8 @@ func init() {
|
||||
disableCompression = *disableCompressionFlag
|
||||
|
||||
flag.Usage = func() {
|
||||
fmt.Print("Usage: bkup -o backup -s s3 -d databasename --path /my_path ...\n")
|
||||
fmt.Print("MySQL Backup and Restoration tool. Backup database to AWS S3 storage or any S3 Alternatives for Object Storage.\n\n")
|
||||
fmt.Print("Usage: bkup --operation backup -storage s3 --dbname databasename --path /my_path ...\n")
|
||||
fmt.Print(" bkup -o backup -d databasename --disable-compression ...\n")
|
||||
fmt.Print(" Restore: bkup -o restore -d databasename -f db_20231217_051339.sql.gz ...\n\n")
|
||||
flag.PrintDefaults()
|
||||
@@ -131,18 +129,17 @@ func init() {
|
||||
return
|
||||
}
|
||||
}
|
||||
dbHost = os.Getenv("DB_HOST")
|
||||
dbPassword = os.Getenv("DB_PASSWORD")
|
||||
dbUserName = os.Getenv("DB_USERNAME")
|
||||
dbName = os.Getenv("DB_NAME")
|
||||
dbPort = os.Getenv("DB_PORT")
|
||||
period = os.Getenv("SCHEDULE_PERIOD")
|
||||
//dbHost = os.Getenv("DB_HOST")
|
||||
//dbPassword = os.Getenv("DB_PASSWORD")
|
||||
//dbUserName = os.Getenv("DB_USERNAME")
|
||||
//dbName = os.Getenv("DB_NAME")
|
||||
//dbPort = os.Getenv("DB_PORT")
|
||||
//period = os.Getenv("SCHEDULE_PERIOD")
|
||||
storage = os.Getenv("STORAGE")
|
||||
|
||||
accessKey = os.Getenv("ACCESS_KEY")
|
||||
secretKey = os.Getenv("SECRET_KEY")
|
||||
bucketName = os.Getenv("BUCKETNAME")
|
||||
s3Endpoint = os.Getenv("S3_ENDPOINT")
|
||||
err := os.Setenv("STORAGE_PATH", storagePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -151,6 +148,8 @@ func version() {
|
||||
fmt.Print()
|
||||
}
|
||||
func main() {
|
||||
//cmd.Execute()
|
||||
|
||||
err := os.Setenv("STORAGE_PATH", storagePath)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -167,7 +166,7 @@ func start() {
|
||||
if operation != "backup" {
|
||||
if storage != "s3" {
|
||||
utils.Info("Restore from local")
|
||||
restore()
|
||||
pkg.Restore(file)
|
||||
} else {
|
||||
utils.Info("Restore from s3")
|
||||
s3Restore()
|
||||
@@ -175,7 +174,7 @@ func start() {
|
||||
} else {
|
||||
if storage != "s3" {
|
||||
utils.Info("Backup to local storage")
|
||||
backup()
|
||||
pkg.Backup(disableCompression)
|
||||
} else {
|
||||
utils.Info("Backup to s3 storage")
|
||||
s3Backup()
|
||||
@@ -187,116 +186,11 @@ func start() {
|
||||
utils.Fatal("Error, unknown execution mode!")
|
||||
}
|
||||
}
|
||||
func backup() {
|
||||
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" {
|
||||
utils.Fatal("Please make sure all required environment variables for database are set")
|
||||
} else {
|
||||
testDatabaseConnection()
|
||||
// Backup database
|
||||
utils.Info("Backing up database...")
|
||||
bkFileName := fmt.Sprintf("%s_%s.sql.gz", dbName, time.Now().Format("20060102_150405"))
|
||||
|
||||
if disableCompression {
|
||||
bkFileName = fmt.Sprintf("%s_%s.sql", dbName, time.Now().Format("20060102_150405"))
|
||||
cmd := exec.Command("mysqldump",
|
||||
"-h", dbHost,
|
||||
"-P", dbPort,
|
||||
"-u", dbUserName,
|
||||
"--password="+dbPassword,
|
||||
dbName,
|
||||
)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
file, err := os.Create(fmt.Sprintf("%s/%s", storagePath, bkFileName))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = file.Write(output)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
utils.Info("Database has been backed up")
|
||||
|
||||
} else {
|
||||
cmd := exec.Command("mysqldump", "-h", dbHost, "-P", dbPort, "-u", dbUserName, "--password="+dbPassword, dbName)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
gzipCmd := exec.Command("gzip")
|
||||
gzipCmd.Stdin = stdout
|
||||
gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", storagePath, bkFileName))
|
||||
gzipCmd.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := gzipCmd.Wait(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
utils.Info("Database has been backed up")
|
||||
|
||||
}
|
||||
|
||||
historyFile, err := os.OpenFile(fmt.Sprintf("%s/history.txt", storagePath), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer historyFile.Close()
|
||||
if _, err := historyFile.WriteString(bkFileName + "\n"); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
func restore() {
|
||||
if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" || file == "" {
|
||||
utils.Fatal("Please make sure all required environment variables are set")
|
||||
} else {
|
||||
|
||||
if utils.FileExists(fmt.Sprintf("%s/%s", storagePath, file)) {
|
||||
testDatabaseConnection()
|
||||
|
||||
extension := filepath.Ext(fmt.Sprintf("%s/%s", storagePath, file))
|
||||
// GZ compressed file
|
||||
if extension == ".gz" {
|
||||
str := "zcat " + fmt.Sprintf("%s/%s", storagePath, file) + " | mysql -h " + os.Getenv("DB_HOST") + " -P " + os.Getenv("DB_PORT") + " -u " + os.Getenv("DB_USERNAME") + " --password=" + os.Getenv("DB_PASSWORD") + " " + os.Getenv("DB_NAME")
|
||||
output, err := exec.Command("bash", "-c", str).Output()
|
||||
if err != nil {
|
||||
utils.Fatal("Error, in restoring the database")
|
||||
}
|
||||
outputContent = string(output)
|
||||
utils.Info("Database has been restored")
|
||||
|
||||
} else if extension == ".sql" {
|
||||
//SQL file
|
||||
str := "cat " + fmt.Sprintf("%s/%s", storagePath, file) + " | mysql -h " + os.Getenv("DB_HOST") + " -P " + os.Getenv("DB_PORT") + " -u " + os.Getenv("DB_USERNAME") + " --password=" + os.Getenv("DB_PASSWORD") + " " + os.Getenv("DB_NAME")
|
||||
output, err := exec.Command("bash", "-c", str).Output()
|
||||
if err != nil {
|
||||
utils.Fatal("Error, in restoring the database", err)
|
||||
}
|
||||
outputContent = string(output)
|
||||
utils.Info("Database has been restored")
|
||||
} else {
|
||||
utils.Fatal("Unknown file extension ", extension)
|
||||
}
|
||||
|
||||
} else {
|
||||
utils.Fatal("File not found in ", fmt.Sprintf("%s/%s", storagePath, file))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
func s3Backup() {
|
||||
// Implement S3 backup logic
|
||||
s3Mount()
|
||||
backup()
|
||||
// Backup to S3 storage
|
||||
pkg.MountS3Storage(s3Path)
|
||||
pkg.Backup(disableCompression)
|
||||
}
|
||||
|
||||
// Run in scheduled mode
|
||||
@@ -311,9 +205,13 @@ func scheduledMode() {
|
||||
utils.Info("Running in Scheduled mode")
|
||||
utils.Info("Log file in /var/log/mysql-bkup.log")
|
||||
utils.Info("Execution period ", os.Getenv("SCHEDULE_PERIOD"))
|
||||
testDatabaseConnection()
|
||||
|
||||
//Test database connexion
|
||||
utils.TestDatabaseConnection()
|
||||
|
||||
utils.Info("Creating backup job...")
|
||||
createCrontabScript()
|
||||
pkg.CreateCrontabScript(disableCompression, storage)
|
||||
|
||||
supervisordCmd := exec.Command("supervisord", "-c", "/etc/supervisor/supervisord.conf")
|
||||
if err := supervisordCmd.Run(); err != nil {
|
||||
utils.Fatalf("Error starting supervisord: %v\n", err)
|
||||
@@ -323,128 +221,8 @@ func scheduledMode() {
|
||||
}
|
||||
}
|
||||
|
||||
// Mount s3 using s3fs
|
||||
func s3Mount() {
|
||||
if accessKey == "" || secretKey == "" || bucketName == "" {
|
||||
utils.Fatal("Please make sure all environment variables are set")
|
||||
} else {
|
||||
storagePath = fmt.Sprintf("%s%s", s3MountPath, s3Path)
|
||||
err := os.Setenv("STORAGE_PATH", storagePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
//Write file
|
||||
err = utils.WriteToFile(s3fsPasswdFile, fmt.Sprintf("%s:%s", accessKey, secretKey))
|
||||
if err != nil {
|
||||
utils.Fatal("Error creating file")
|
||||
}
|
||||
//Change file permission
|
||||
utils.ChangePermission(s3fsPasswdFile, 0600)
|
||||
utils.Info("Mounting Object storage in", s3MountPath)
|
||||
if isEmpty, _ := utils.IsDirEmpty(s3MountPath); isEmpty {
|
||||
cmd := exec.Command("s3fs", bucketName, s3MountPath,
|
||||
"-o", "passwd_file="+s3fsPasswdFile,
|
||||
"-o", "use_cache=/tmp/s3cache",
|
||||
"-o", "allow_other",
|
||||
"-o", "url="+s3Endpoint,
|
||||
"-o", "use_path_request_style",
|
||||
)
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
utils.Fatal("Error mounting Object storage:", err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(storagePath, os.ModePerm); err != nil {
|
||||
utils.Fatalf("Error creating directory %v %v", storagePath, err)
|
||||
}
|
||||
|
||||
} else {
|
||||
utils.Info("Object storage already mounted in " + s3MountPath)
|
||||
if err := os.MkdirAll(storagePath, os.ModePerm); err != nil {
|
||||
utils.Fatal("Error creating directory "+storagePath, err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
func s3Restore() {
|
||||
// Implement S3 restore logic\
|
||||
s3Mount()
|
||||
restore()
|
||||
}
|
||||
|
||||
func createCrontabScript() {
|
||||
task := "/usr/local/bin/backup_cron.sh"
|
||||
touchCmd := exec.Command("touch", task)
|
||||
if err := touchCmd.Run(); err != nil {
|
||||
utils.Fatalf("Error creating file %s: %v\n", task, err)
|
||||
}
|
||||
var disableC = ""
|
||||
if disableCompression {
|
||||
disableC = "--disable-compression"
|
||||
}
|
||||
|
||||
var scriptContent string
|
||||
|
||||
if storage == "s3" {
|
||||
scriptContent = fmt.Sprintf(`#!/usr/bin/env bash
|
||||
set -e
|
||||
bkup --operation backup --dbname %s --port %s --storage s3 --path %s %v
|
||||
`, os.Getenv("DB_NAME"), os.Getenv("DB_PORT"), os.Getenv("S3_PATH"), disableC)
|
||||
} else {
|
||||
scriptContent = fmt.Sprintf(`#!/usr/bin/env bash
|
||||
set -e
|
||||
bkup --operation backup --dbname %s --port %s %v
|
||||
`, os.Getenv("DB_NAME"), os.Getenv("DB_PORT"), disableC)
|
||||
}
|
||||
|
||||
if err := utils.WriteToFile(task, scriptContent); err != nil {
|
||||
utils.Fatalf("Error writing to %s: %v\n", task, err)
|
||||
}
|
||||
|
||||
chmodCmd := exec.Command("chmod", "+x", "/usr/local/bin/backup_cron.sh")
|
||||
if err := chmodCmd.Run(); err != nil {
|
||||
utils.Fatalf("Error changing permissions of %s: %v\n", task, err)
|
||||
}
|
||||
|
||||
lnCmd := exec.Command("ln", "-s", "/usr/local/bin/backup_cron.sh", "/usr/local/bin/backup_cron")
|
||||
if err := lnCmd.Run(); err != nil {
|
||||
utils.Fatalf("Error creating symbolic link: %v\n", err)
|
||||
|
||||
}
|
||||
|
||||
cronJob := "/etc/cron.d/backup_cron"
|
||||
touchCronCmd := exec.Command("touch", cronJob)
|
||||
if err := touchCronCmd.Run(); err != nil {
|
||||
utils.Fatalf("Error creating file %s: %v\n", cronJob, err)
|
||||
}
|
||||
|
||||
cronContent := fmt.Sprintf(`%s root exec /bin/bash -c ". /run/supervisord.env; /usr/local/bin/backup_cron.sh >> /var/log/mysql-bkup.log"
|
||||
`, os.Getenv("SCHEDULE_PERIOD"))
|
||||
|
||||
if err := utils.WriteToFile(cronJob, cronContent); err != nil {
|
||||
utils.Fatalf("Error writing to %s: %v\n", cronJob, err)
|
||||
}
|
||||
utils.ChangePermission("/etc/cron.d/backup_cron", 0644)
|
||||
|
||||
crontabCmd := exec.Command("crontab", "/etc/cron.d/backup_cron")
|
||||
if err := crontabCmd.Run(); err != nil {
|
||||
utils.Fatal("Error updating crontab: ", err)
|
||||
}
|
||||
utils.Info("Starting backup in scheduled mode")
|
||||
}
|
||||
|
||||
// testDatabaseConnection tests the database connection
|
||||
func testDatabaseConnection() {
|
||||
utils.Info("Testing database connection...")
|
||||
// Test database connection
|
||||
cmd := exec.Command("mysql", "-h", os.Getenv("DB_HOST"), "-P", os.Getenv("DB_PORT"), "-u", os.Getenv("DB_USERNAME"), "--password="+os.Getenv("DB_PASSWORD"), os.Getenv("DB_NAME"), "-e", "quit")
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
utils.Fatal("Error testing database connection:", err)
|
||||
|
||||
}
|
||||
|
||||
// Restore database from S3
|
||||
pkg.MountS3Storage(s3Path)
|
||||
pkg.Restore(file)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user