From 82987c82f00b39727a79b61f9617089a62f514dd Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Sat, 20 Jan 2024 14:03:06 +0100 Subject: [PATCH] feat: replace --operation flag by backup and restore command --- README.md | 35 ++-- cmd/backup.go | 28 +++ cmd/history.go | 14 ++ cmd/restore.go | 28 +++ cmd/root.go | 55 ++--- cmd/s3mount.go | 14 ++ cmd/version.go | 7 +- examples/docker-compose.s3.yaml | 2 +- examples/docker-compose.scheduled.local.yaml | 2 +- examples/docker-compose.scheduled.s3.yaml | 2 +- examples/docker-compose.yaml | 2 +- examples/k8s-job.yaml | 2 +- go.mod | 8 +- go.sum | 2 + main.go | 210 +------------------ pkg/backup.go | 79 ++++++- pkg/restore.go | 32 +++ pkg/s3fs.go | 11 +- pkg/scripts.go | 4 +- pkg/var.go | 18 ++ scripts/backup_script.sh | 2 +- utils/constant.go | 16 ++ utils/utils.go | 35 +++- 23 files changed, 332 insertions(+), 276 deletions(-) create mode 100644 cmd/backup.go create mode 100644 cmd/history.go create mode 100644 cmd/restore.go create mode 100644 cmd/s3mount.go create mode 100644 pkg/var.go create mode 100644 utils/constant.go diff --git a/README.md b/README.md index 7ced2ce..ac6f253 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,9 @@ PostgreSQL Backup and Restoration tool. Backup database to AWS S3 storage or any | Options | Shorts | Usage | |---------------|--------|------------------------------------| | pg-bkup | bkup | CLI utility | -| --operation | -o | Set operation. backup or restore (default: backup) | +| backup | | Backup database operation | +| restore | | Restore database operation | +| history | | Show the history of backup | | --storage | -s | Set storage. local or s3 (default: local) | | --file | -f | Set file name for restoration | | --path | | Set s3 path without file name. eg: /custom_path | @@ -67,12 +69,12 @@ bkup -o backup ### S3 ```sh -bkup --operation backup --storage s3 +bkup backup --storage s3 ``` ## Docker run: ```sh -docker run --rm --network your_network_name --name pg-bkup -v $PWD/backup:/backup/ -e "DB_HOST=database_host_name" -e "DB_USERNAME=username" -e "DB_PASSWORD=password" jkaninda/pg-bkup bkup -o backup -d database_name +docker run --rm --network your_network_name --name pg-bkup -v $PWD/backup:/backup/ -e "DB_HOST=database_host_name" -e "DB_USERNAME=username" -e "DB_PASSWORD=password" jkaninda/pg-bkup bkup backup -d database_name ``` ## Docker compose file: @@ -82,7 +84,6 @@ services: postgres: image: postgres:14.5 container_name: postgres - pull_policy: if_not_present restart: unless-stopped volumes: - ./postgres:/var/lib/postgresql/data @@ -98,7 +99,7 @@ services: command: - /bin/sh - -c - - bkup --operation backup -d bkup + - bkup backup -d bkup volumes: - ./backup:/backup environment: @@ -113,22 +114,22 @@ services: Simple database restore operation usage ```sh -bkup --operation restore --file database_20231217_115621.sql --dbname database_name +bkup restore --file database_20231217_115621.sql --dbname database_name ``` ```sh -bkup -o restore -f database_20231217_115621.sql -d database_name +bkup restore -f database_20231217_115621.sql -d database_name ``` ### S3 ```sh -bkup --operation restore --storage s3 --file database_20231217_115621.sql --dbname database_name +bkup restore --storage s3 --file database_20231217_115621.sql --dbname database_name ``` ## Docker run: ```sh -docker run --rm --network your_network_name --name pg-bkup -v $PWD/backup:/backup/ -e "DB_HOST=database_host_name" -e "DB_USERNAME=username" -e "DB_PASSWORD=password" jkaninda/pg-bkup bkup -o restore -d database_name -f napata_20231219_022941.sql.gz +docker run --rm --network your_network_name --name pg-bkup -v $PWD/backup:/backup/ -e "DB_HOST=database_host_name" -e "DB_USERNAME=username" -e "DB_PASSWORD=password" jkaninda/pg-bkup bkup restore -d database_name -f napata_20231219_022941.sql.gz ``` ## Docker compose file: @@ -142,7 +143,7 @@ services: command: - /bin/sh - -c - - bkup --operation restore --file database_20231217_115621.sql -d database_name + - bkup restore --file database_20231217_115621.sql -d database_name volumes: - ./backup:/backup environment: @@ -160,14 +161,14 @@ docker-compose up -d ## Backup to S3 ```sh -docker run --rm --privileged --device /dev/fuse --name pg-bkup -e "DB_HOST=db_hostname" -e "DB_USERNAME=username" -e "DB_PASSWORD=password" -e "ACCESS_KEY=your_access_key" -e "SECRET_KEY=your_secret_key" -e "BUCKETNAME=your_bucket_name" -e "S3_ENDPOINT=https://s3.us-west-2.amazonaws.com" jkaninda/pg-bkup bkup -o backup -s s3 -d database_name +docker run --rm --privileged --device /dev/fuse --name pg-bkup -e "DB_HOST=db_hostname" -e "DB_USERNAME=username" -e "DB_PASSWORD=password" -e "ACCESS_KEY=your_access_key" -e "SECRET_KEY=your_secret_key" -e "BUCKETNAME=your_bucket_name" -e "S3_ENDPOINT=https://s3.us-west-2.amazonaws.com" jkaninda/pg-bkup bkup backup -s s3 -d database_name ``` > To change s3 backup path add this flag : --path /mycustomPath . default path is /pg-bkup Simple S3 backup usage ```sh -bkup --operation backup --storage s3 --dbname mydatabase +bkup backup --storage s3 --dbname mydatabase ``` ```yaml pg-bkup: @@ -180,7 +181,7 @@ bkup --operation backup --storage s3 --dbname mydatabase command: - /bin/sh - -c - - pg-bkup --operation restore --storage s3 -f database_20231217_115621.sql.gz --dbname database_name + - pg-bkup restore --storage s3 -f database_20231217_115621.sql.gz --dbname database_name environment: - DB_PORT=5432 - DB_HOST=postgress @@ -246,7 +247,7 @@ Easy to remember format: > Docker run : ```sh -docker run --rm --name pg-bkup -v $BACKUP_DIR:/backup/ -e "DB_HOST=$DB_HOST" -e "DB_USERNAME=$DB_USERNAME" -e "DB_PASSWORD=$DB_PASSWORD" jkaninda/pg-bkup bkup --operation backup --dbname $DB_NAME --mode scheduled --period "0 1 * * *" +docker run --rm --name pg-bkup -v $BACKUP_DIR:/backup/ -e "DB_HOST=$DB_HOST" -e "DB_USERNAME=$DB_USERNAME" -e "DB_PASSWORD=$DB_PASSWORD" jkaninda/pg-bkup bkup backup --dbname $DB_NAME --mode scheduled --period "0 1 * * *" ``` > With Docker compose @@ -263,7 +264,7 @@ services: command: - /bin/sh - -c - - bkup --operation backup --storage s3 --path /mys3_custome_path --dbname database_name --mode scheduled --period "*/30 * * * *" + - bkup backup --storage s3 --path /mys3_custome_path --dbname database_name --mode scheduled --period "*/30 * * * *" environment: - DB_PORT=5432 - DB_HOST=postgreshost @@ -277,7 +278,7 @@ services: ## Kubernetes CronJob -For Kubernetes you don't need to run it in scheduled mode. +For Kubernetes, you don't need to run it in scheduled mode. Simple Kubernetes CronJob usage: @@ -300,7 +301,7 @@ spec: command: - /bin/sh - -c - - bkup --operation backup -s s3 --path /custom_path + - bkup backup -s s3 --path /custom_path env: - name: DB_PORT value: "5432" diff --git a/cmd/backup.go b/cmd/backup.go new file mode 100644 index 0000000..456ac27 --- /dev/null +++ b/cmd/backup.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "github.com/jkaninda/pg-bkup/pkg" + "github.com/jkaninda/pg-bkup/utils" + "github.com/spf13/cobra" +) + +var BackupCmd = &cobra.Command{ + Use: "backup ", + Short: "Backup database operation", + Example: utils.BackupExample, + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + pkg.StartBackup(cmd) + } else { + utils.Fatal("Error, no argument required") + } + }, +} + +func init() { + //Backup + BackupCmd.PersistentFlags().StringP("mode", "m", "default", "Set execution mode. default or scheduled") + BackupCmd.PersistentFlags().StringP("period", "", "0 1 * * *", "Set schedule period time") + BackupCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression") + +} diff --git a/cmd/history.go b/cmd/history.go new file mode 100644 index 0000000..cf232e7 --- /dev/null +++ b/cmd/history.go @@ -0,0 +1,14 @@ +package cmd + +import ( + "github.com/jkaninda/pg-bkup/utils" + "github.com/spf13/cobra" +) + +var HistoryCmd = &cobra.Command{ + Use: "history", + Short: "Show the history of backup", + Run: func(cmd *cobra.Command, args []string) { + utils.ShowHistory() + }, +} diff --git a/cmd/restore.go b/cmd/restore.go new file mode 100644 index 0000000..4e275ef --- /dev/null +++ b/cmd/restore.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "github.com/jkaninda/pg-bkup/pkg" + "github.com/jkaninda/pg-bkup/utils" + "github.com/spf13/cobra" +) + +var RestoreCmd = &cobra.Command{ + Use: "restore", + Short: "Restore database operation", + Example: utils.RestoreExample, + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + pkg.StartRestore(cmd) + } else { + utils.Fatal("Error, no argument required") + + } + + }, +} + +func init() { + //Restore + RestoreCmd.PersistentFlags().StringP("file", "f", "", "File name of database") + +} diff --git a/cmd/root.go b/cmd/root.go index 1874620..180fde1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,20 +5,32 @@ Copyright © 2024 Jonas Kaninda package cmd import ( - "os" - + "fmt" + "github.com/jkaninda/pg-bkup/utils" "github.com/spf13/cobra" + "os" ) // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ - Use: "pg-bkup", - Short: "PostgreSQL Backup tool, backup database to S3 or Object Storage", - Long: `PostgreSQL Backup and Restoration tool. Backup database to AWS S3 storage or any S3 Alternatives for Object Storage.`, - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, + Use: "pg-bkup [Command]", + Short: "PostgreSQL Backup tool, backup database to S3 or Object Storage", + Long: `PostgreSQL Database backup and restoration tool. Backup database to AWS S3 storage or any S3 Alternatives for Object Storage.`, + Example: utils.MainExample, + Version: appVersion, + //TODO: To remove + //For old user || To remove + Run: func(cmd *cobra.Command, args []string) { + if operation != "" { + if operation == "backup" || operation == "restore" { + fmt.Println(utils.Notice) + utils.Fatal("New config required, please check --help") + } + } + }, } +var operation = "" +var s3Path = "/pg-bkup" // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. @@ -30,26 +42,21 @@ func Execute() { } func init() { - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.mysql-bkup.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.PersistentFlags().StringP("operation", "o", "backup", "Set operation") rootCmd.PersistentFlags().StringP("storage", "s", "local", "Set storage. local or s3") - rootCmd.PersistentFlags().StringP("file", "f", "", "Set file name") - rootCmd.PersistentFlags().StringP("path", "P", "/mysql-bkup", "Set s3 path, without file name") + rootCmd.PersistentFlags().StringP("path", "P", s3Path, "Set s3 path, without file name. for S3 storage only") rootCmd.PersistentFlags().StringP("dbname", "d", "", "Set database name") + rootCmd.PersistentFlags().IntP("timeout", "t", 30, "Set timeout") + rootCmd.PersistentFlags().IntP("port", "p", 5432, "Set database port") + rootCmd.PersistentFlags().StringVarP(&operation, "operation", "o", "", "Set operation, for old version only") + rootCmd.PersistentFlags().StringP("mode", "m", "default", "Set execution mode. default or scheduled") rootCmd.PersistentFlags().StringP("period", "", "0 1 * * *", "Set schedule period time") - rootCmd.PersistentFlags().IntP("timeout", "t", 30, "Set timeout") rootCmd.PersistentFlags().BoolP("disable-compression", "", false, "Disable backup compression") - rootCmd.PersistentFlags().IntP("port", "p", 5432, "Set database port") - rootCmd.PersistentFlags().BoolP("help", "h", false, "Print this help message") - rootCmd.PersistentFlags().BoolP("version", "v", false, "shows version information") - rootCmd.AddCommand(VersionCmd) + rootCmd.PersistentFlags().StringP("file", "f", "", "File name of database") + rootCmd.AddCommand(VersionCmd) + rootCmd.AddCommand(BackupCmd) + rootCmd.AddCommand(RestoreCmd) + rootCmd.AddCommand(S3MountCmd) + rootCmd.AddCommand(HistoryCmd) } diff --git a/cmd/s3mount.go b/cmd/s3mount.go new file mode 100644 index 0000000..c7c6007 --- /dev/null +++ b/cmd/s3mount.go @@ -0,0 +1,14 @@ +package cmd + +import ( + "github.com/jkaninda/pg-bkup/pkg" + "github.com/spf13/cobra" +) + +var S3MountCmd = &cobra.Command{ + Use: "s3mount", + Short: "Mount AWS S3 storage", + Run: func(cmd *cobra.Command, args []string) { + pkg.S3Mount() + }, +} diff --git a/cmd/version.go b/cmd/version.go index f9f7e23..1e734a5 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -1,5 +1,9 @@ package cmd +/* +Copyright © 2024 Jonas Kaninda +*/ + import ( "fmt" "github.com/spf13/cobra" @@ -16,8 +20,7 @@ var VersionCmd = &cobra.Command{ }, } -// Version display application version func Version() { fmt.Printf("Version: %s \n", appVersion) - fmt.Print() + fmt.Println() } diff --git a/examples/docker-compose.s3.yaml b/examples/docker-compose.s3.yaml index 96e3032..3ae15b4 100644 --- a/examples/docker-compose.s3.yaml +++ b/examples/docker-compose.s3.yaml @@ -9,7 +9,7 @@ services: command: - /bin/sh - -c - - bkup --operation backup --storage s3 --path /mys3_custom_path --dbname database_name + - bkup backup --storage s3 --path /mys3_custom_path --dbname database_name environment: - DB_PORT=5432 - DB_HOST=postgress diff --git a/examples/docker-compose.scheduled.local.yaml b/examples/docker-compose.scheduled.local.yaml index cf5462f..6e6f239 100644 --- a/examples/docker-compose.scheduled.local.yaml +++ b/examples/docker-compose.scheduled.local.yaml @@ -6,7 +6,7 @@ services: command: - /bin/sh - -c - - bkup --operation backup --dbname database_name --mode scheduled --period "0 1 * * *" + - bkup backup --dbname database_name --mode scheduled --period "0 1 * * *" volumes: - ./backup:/backup environment: diff --git a/examples/docker-compose.scheduled.s3.yaml b/examples/docker-compose.scheduled.s3.yaml index 15cb628..4106232 100644 --- a/examples/docker-compose.scheduled.s3.yaml +++ b/examples/docker-compose.scheduled.s3.yaml @@ -9,7 +9,7 @@ services: command: - /bin/sh - -c - - bkup --operation backup --storage s3 --path /mys3_custom_path --dbname database_name --mode scheduled --period "0 1 * * *" + - bkup backup --storage s3 --path /mys3_custom_path --dbname database_name --mode scheduled --period "0 1 * * *" environment: - DB_PORT=5432 - DB_HOST=postgress diff --git a/examples/docker-compose.yaml b/examples/docker-compose.yaml index 433eb3e..4407057 100644 --- a/examples/docker-compose.yaml +++ b/examples/docker-compose.yaml @@ -6,7 +6,7 @@ services: command: - /bin/sh - -c - - bkup --operation backup --dbname database_name + - bkup backup --dbname database_name volumes: - ./backup:/backup environment: diff --git a/examples/k8s-job.yaml b/examples/k8s-job.yaml index dd36a97..9afbb7c 100644 --- a/examples/k8s-job.yaml +++ b/examples/k8s-job.yaml @@ -16,7 +16,7 @@ spec: command: - /bin/sh - -c - - bkup --operation backup --storage s3 --path /custom_path + - bkup backup --storage s3 --path /custom_path env: - name: DB_PORT value: "5432" diff --git a/go.mod b/go.mod index 21ff690..145b169 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,9 @@ module github.com/jkaninda/pg-bkup go 1.21.0 -require github.com/spf13/pflag v1.0.5 - require ( - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/cobra v1.8.0 + github.com/spf13/pflag v1.0.5 ) + +require github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index aab9122..d0e8c2c 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= diff --git a/main.go b/main.go index 490ef0f..5666f92 100644 --- a/main.go +++ b/main.go @@ -1,218 +1,16 @@ package main +//main /***** -* PostgreSQL Backup & Restore +* MySQL Backup & Restore * @author Jonas Kaninda * @license MIT License * @link https://github.com/jkaninda/pg-bkup **/ -import ( - "fmt" - "github.com/jkaninda/pg-bkup/pkg" - "github.com/jkaninda/pg-bkup/utils" - flag "github.com/spf13/pflag" - "os" - "os/exec" -) +import "github.com/jkaninda/pg-bkup/cmd" -var appVersion string = os.Getenv("VERSION") - -const s3MountPath string = "/s3mnt" - -var ( - operation = "backup" - storage = "local" - file = "" - s3Path = "/pg-bkup" - dbName = "" - dbHost = "" - dbPort = "5432" - dbPassword = "" - dbUserName = "" - executionMode = "default" - storagePath = "/backup" - accessKey = "" - secretKey = "" - bucketName = "" - s3Endpoint = "" - s3fsPasswdFile = "/etc/passwd-s3fs" - disableCompression = false - startBackup = true - - timeout int = 30 - period string = "0 1 * * *" -) - -func init() { - var ( - operationFlag = flag.StringP("operation", "o", "backup", "Operation") - storageFlag = flag.StringP("storage", "s", "local", "Storage, local or s3") - fileFlag = flag.StringP("file", "f", "", "File name") - pathFlag = flag.StringP("path", "P", "/mysql-bkup", "S3 path, without file name") - dbnameFlag = flag.StringP("dbname", "d", "", "Database name") - modeFlag = flag.StringP("mode", "m", "default", "Execution mode. default or scheduled") - periodFlag = flag.StringP("period", "", "0 1 * * *", "Schedule period time") - timeoutFlag = flag.IntP("timeout", "t", 30, "Timeout (in seconds) to stop database connexion") - disableCompressionFlag = flag.BoolP("disable-compression", "", false, "Disable backup compression") - portFlag = flag.IntP("port", "p", 5432, "Database port") - helpFlag = flag.BoolP("help", "h", false, "Print this help message") - versionFlag = flag.BoolP("version", "v", false, "Version information") - ) - flag.Parse() - - operation = *operationFlag - storage = *storageFlag - file = *fileFlag - s3Path = *pathFlag - dbName = *dbnameFlag - executionMode = *modeFlag - dbPort = fmt.Sprint(*portFlag) - timeout = *timeoutFlag - period = *periodFlag - disableCompression = *disableCompressionFlag - - flag.Usage = func() { - fmt.Print("PostgreSQL Backup and Restoration tool. Backup database to AWS S3 storage or any S3 Alternatives for Object Storage.\n\n") - fmt.Print("Usage: bkup --operation backup -storage s3 --dbname databasename --path /my_path ...\n") - fmt.Print(" bkup -o backup -d databasename --disable-compression ...\n") - fmt.Print(" Restore: bkup -o restore -d databasename -f db_20231217_051339.sql.gz ...\n\n") - flag.PrintDefaults() - } - - if *helpFlag { - startBackup = false - flag.Usage() - os.Exit(0) - } - if *versionFlag { - startBackup = false - version() - os.Exit(0) - } - if *dbnameFlag != "" { - err := os.Setenv("DB_NAME", dbName) - if err != nil { - return - } - } - if *pathFlag != "" { - s3Path = *pathFlag - err := os.Setenv("S3_PATH", fmt.Sprint(*pathFlag)) - if err != nil { - return - } - - } - if *fileFlag != "" { - file = *fileFlag - err := os.Setenv("FILE_NAME", fmt.Sprint(*fileFlag)) - if err != nil { - return - } - - } - if *portFlag != 3306 { - err := os.Setenv("DB_PORT", fmt.Sprint(*portFlag)) - if err != nil { - return - } - } - if *periodFlag != "" { - err := os.Setenv("SCHEDULE_PERIOD", fmt.Sprint(*periodFlag)) - if err != nil { - return - } - } - if *storageFlag != "" { - err := os.Setenv("STORAGE", fmt.Sprint(*storageFlag)) - if err != nil { - return - } - } - storage = os.Getenv("STORAGE") - - err := os.Setenv("STORAGE_PATH", storagePath) - if err != nil { - return - } - -} - -func version() { - fmt.Printf("Version: %s \n", appVersion) - fmt.Print() -} func main() { - err := os.Setenv("STORAGE_PATH", storagePath) - if err != nil { - return - } - if startBackup { - start() - } + cmd.Execute() } -func start() { - - if executionMode == "default" { - if operation != "backup" { - if storage != "s3" { - utils.Info("Restore from local") - pkg.RestoreDatabase(file) - } else { - utils.Info("Restore from s3") - s3Restore() - } - } else { - if storage != "s3" { - utils.Info("Backup to local storage") - pkg.BackupDatabase(disableCompression) - } else { - utils.Info("Backup to s3 storage") - s3Backup() - } - } - } else if executionMode == "scheduled" { - scheduledMode() - } else { - utils.Fatal("Error, unknown execution mode!") - } -} -func s3Backup() { - pkg.MountS3Storage(s3Path) - pkg.BackupDatabase(disableCompression) -} - -// Run in scheduled mode -func scheduledMode() { - // Verify operation - if operation == "backup" { - - fmt.Println() - fmt.Println("**********************************") - fmt.Println(" Starting PostgreSQL Bkup... ") - fmt.Println("***********************************") - utils.Info("Running in Scheduled mode") - utils.Info("Log file in /var/log/pg-bkup.log") - utils.Info("Execution period ", os.Getenv("SCHEDULE_PERIOD")) - //Test database connexion - utils.TestDatabaseConnection() - - utils.Info("Creating backup job...") - pkg.CreateCrontabScript(disableCompression, storage) - - supervisordCmd := exec.Command("supervisord", "-c", "/etc/supervisor/supervisord.conf") - if err := supervisordCmd.Run(); err != nil { - utils.Fatalf("Error starting supervisord: %v\n", err) - } - } else { - utils.Fatal("Scheduled mode supports only backup operation") - } -} - -func s3Restore() { - // Restore database from S3 - pkg.MountS3Storage(s3Path) - pkg.RestoreDatabase(file) -} diff --git a/pkg/backup.go b/pkg/backup.go index e3772b3..c0660f5 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -7,20 +7,72 @@ package pkg import ( "fmt" "github.com/jkaninda/pg-bkup/utils" + "github.com/spf13/cobra" "log" "os" "os/exec" "time" ) -var ( - dbName = "" - dbHost = "" - dbPort = "" - dbPassword = "" - dbUserName = "" - storagePath = "/backup" -) +func StartBackup(cmd *cobra.Command) { + _, _ = cmd.Flags().GetString("operation") + + //Set env + utils.SetEnv("STORAGE_PATH", storagePath) + utils.GetEnv(cmd, "dbname", "DB_NAME") + utils.GetEnv(cmd, "port", "DB_PORT") + utils.GetEnv(cmd, "period", "SCHEDULE_PERIOD") + + //Get flag value and set env + s3Path = utils.GetEnv(cmd, "path", "S3_PATH") + storage = utils.GetEnv(cmd, "storage", "STORAGE") + file = utils.GetEnv(cmd, "file", "FILE_NAME") + disableCompression, _ = cmd.Flags().GetBool("disable-compression") + executionMode, _ = cmd.Flags().GetString("mode") + + if executionMode == "default" { + if storage == "s3" { + utils.Info("Backup database to s3 storage") + s3Backup(disableCompression, s3Path) + } else { + utils.Info("Backup database to local storage") + BackupDatabase(disableCompression) + + } + } else if executionMode == "scheduled" { + scheduledMode() + } else { + utils.Fatal("Error, unknown execution mode!") + } + +} + +// Run in scheduled mode +func scheduledMode() { + + fmt.Println() + fmt.Println("**********************************") + fmt.Println(" Starting PostgreSQL Bkup... ") + fmt.Println("***********************************") + utils.Info("Running in Scheduled mode") + utils.Info("Log file in /var/log/pg-bkup.log") + utils.Info("Execution period ", os.Getenv("SCHEDULE_PERIOD")) + + //Test database connexion + utils.TestDatabaseConnection() + + //Test database connexion + utils.TestDatabaseConnection() + + utils.Info("Creating backup job...") + CreateCrontabScript(disableCompression, storage) + + //Start Supervisor + supervisordCmd := exec.Command("supervisord", "-c", "/etc/supervisor/supervisord.conf") + if err := supervisordCmd.Run(); err != nil { + utils.Fatalf("Error starting supervisord: %v\n", err) + } +} // BackupDatabase backup database func BackupDatabase(disableCompression bool) { @@ -38,12 +90,13 @@ func BackupDatabase(disableCompression bool) { if err != nil { return } - //Test Database connexion utils.TestDatabaseConnection() - // Backup database + // Backup Database database utils.Info("Backing up database...") + //Generate file name bkFileName := fmt.Sprintf("%s_%s.sql.gz", dbName, time.Now().Format("20060102_150405")) + // Verify is compression is disabled if disableCompression { bkFileName = fmt.Sprintf("%s_%s.sql", dbName, time.Now().Format("20060102_150405")) // Execute pg_dump @@ -111,3 +164,9 @@ func BackupDatabase(disableCompression bool) { } } + +func s3Backup(disableCompression bool, s3Path string) { + // Backup Database to S3 storage + MountS3Storage(s3Path) + BackupDatabase(disableCompression) +} diff --git a/pkg/restore.go b/pkg/restore.go index a78f30f..81d1ad1 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -3,11 +3,35 @@ package pkg import ( "fmt" "github.com/jkaninda/pg-bkup/utils" + "github.com/spf13/cobra" "os" "os/exec" "path/filepath" ) +func StartRestore(cmd *cobra.Command) { + + //Set env + utils.SetEnv("STORAGE_PATH", storagePath) + utils.GetEnv(cmd, "dbname", "DB_NAME") + utils.GetEnv(cmd, "port", "DB_PORT") + + //Get flag value and set env + s3Path = utils.GetEnv(cmd, "path", "S3_PATH") + storage = utils.GetEnv(cmd, "storage", "STORAGE") + file = utils.GetEnv(cmd, "file", "FILE_NAME") + executionMode, _ = cmd.Flags().GetString("mode") + + if storage == "s3" { + utils.Info("Restore database from s3") + s3Restore(file, s3Path) + } else { + utils.Info("Restore database from local") + RestoreDatabase(file) + + } +} + // RestoreDatabase restore database func RestoreDatabase(file string) { dbHost = os.Getenv("DB_HOST") @@ -16,6 +40,9 @@ func RestoreDatabase(file string) { dbName = os.Getenv("DB_NAME") dbPort = os.Getenv("DB_PORT") storagePath = os.Getenv("STORAGE_PATH") + if file == "" { + utils.Fatal("Error required --file") + } if os.Getenv("DB_HOST") == "" || os.Getenv("DB_NAME") == "" || os.Getenv("DB_USERNAME") == "" || os.Getenv("DB_PASSWORD") == "" || file == "" { utils.Fatal("Please make sure all required environment variables are set") @@ -56,3 +83,8 @@ func RestoreDatabase(file string) { } } } +func s3Restore(file, s3Path string) { + // Restore database from S3 + MountS3Storage(s3Path) + RestoreDatabase(file) +} diff --git a/pkg/s3fs.go b/pkg/s3fs.go index a40dd81..1ae8030 100644 --- a/pkg/s3fs.go +++ b/pkg/s3fs.go @@ -11,9 +11,6 @@ import ( "os/exec" ) -const s3MountPath string = "/s3mnt" -const s3fsPasswdFile string = "/etc/passwd-s3fs" - var ( accessKey = "" secretKey = "" @@ -21,6 +18,10 @@ var ( s3Endpoint = "" ) +func S3Mount() { + MountS3Storage(s3Path) +} + // MountS3Storage Mount s3 storage using s3fs func MountS3Storage(s3Path string) { accessKey = os.Getenv("ACCESS_KEY") @@ -44,7 +45,9 @@ func MountS3Storage(s3Path string) { } //Change file permission utils.ChangePermission(s3fsPasswdFile, 0600) - utils.Info("Mounting Object storage in", s3MountPath) + + //Mount object storage + utils.Info("Mounting Object storage in ", s3MountPath) if isEmpty, _ := utils.IsDirEmpty(s3MountPath); isEmpty { cmd := exec.Command("s3fs", bucketName, s3MountPath, "-o", "passwd_file="+s3fsPasswdFile, diff --git a/pkg/scripts.go b/pkg/scripts.go index d4e08ff..2de3a45 100644 --- a/pkg/scripts.go +++ b/pkg/scripts.go @@ -30,12 +30,12 @@ func CreateCrontabScript(disableCompression bool, storage string) { if storage == "s3" { scriptContent = fmt.Sprintf(`#!/usr/bin/env bash set -e -bkup --operation backup --dbname %s --port %s --storage s3 --path %s %v +bkup backup --dbname %s --port %s --storage s3 --path %s %v `, os.Getenv("DB_NAME"), os.Getenv("DB_PORT"), os.Getenv("S3_PATH"), disableC) } else { scriptContent = fmt.Sprintf(`#!/usr/bin/env bash set -e -bkup --operation backup --dbname %s --port %s %v +bkup backup --dbname %s --port %s %v `, os.Getenv("DB_NAME"), os.Getenv("DB_PORT"), disableC) } diff --git a/pkg/var.go b/pkg/var.go new file mode 100644 index 0000000..0ce4fd4 --- /dev/null +++ b/pkg/var.go @@ -0,0 +1,18 @@ +package pkg + +const s3MountPath string = "/s3mnt" +const s3fsPasswdFile string = "/etc/passwd-s3fs" + +var ( + storage = "local" + file = "" + s3Path = "/pg-bkup" + dbPassword = "" + dbUserName = "" + dbName = "" + dbHost = "" + dbPort = "5432" + executionMode = "default" + storagePath = "/backup" + disableCompression = false +) diff --git a/scripts/backup_script.sh b/scripts/backup_script.sh index c10fa5f..6bc5dc6 100755 --- a/scripts/backup_script.sh +++ b/scripts/backup_script.sh @@ -5,4 +5,4 @@ DB_HOST='db_hostname' DB_NAME='db_name' BACKUP_DIR="$PWD/backup" -docker run --rm --name pg-bkup -v $BACKUP_DIR:/backup/ -e "DB_HOST=$DB_HOST" -e "DB_USERNAME=$DB_USERNAME" -e "DB_PASSWORD=$DB_PASSWORD" jkaninda/pg-bkup bkup -o backup -d $DB_NAME \ No newline at end of file +docker run --rm --name pg-bkup -v $BACKUP_DIR:/backup/ -e "DB_HOST=$DB_HOST" -e "DB_USERNAME=$DB_USERNAME" -e "DB_PASSWORD=$DB_PASSWORD" jkaninda/pg-bkup bkup backup -d $DB_NAME \ No newline at end of file diff --git a/utils/constant.go b/utils/constant.go new file mode 100644 index 0000000..8b96812 --- /dev/null +++ b/utils/constant.go @@ -0,0 +1,16 @@ +package utils + +const Notice = "Please remove --operation flag.\n" + + "Use: \n" + + "- backup for database backup operation [eg: bkup backup -d database_name ...]\n" + + "- restore for database restore operation [eg. bkup restore -d database_name ...]\n" + + "Example: bkup backup --storage s3 ...( instead of < bkup --operation backup >)\n" + + "We are sorry for this inconvenient\n" +const RestoreExample = "pg-bkup restore --dbname database --file db_20231219_022941.sql.gz\n" + + "bkup restore --dbname database --storage s3 --path /custom-path --file db_20231219_022941.sql.gz" +const BackupExample = "pg-bkup backup --dbname database --disable-compression\n" + + "pg-bkup backup --dbname database --storage s3 --path /custom-path --disable-compression" + +const MainExample = "pg-bkup backup --dbname database --disable-compression\n" + + "pg-bkup backup --dbname database --storage s3 --path /custom-path\n" + + "pg-bkup restore --dbname database --file db_20231219_022941.sql.gz" diff --git a/utils/utils.go b/utils/utils.go index 053dda8..194233b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -8,6 +8,7 @@ package utils **/ import ( "fmt" + "github.com/spf13/cobra" "io/fs" "os" ) @@ -69,5 +70,37 @@ func IsDirEmpty(name string) (bool, error) { func TestDatabaseConnection() { Info("Testing database connection...") // Test database connection - +} +func GetEnv(cmd *cobra.Command, flagName, envName string) string { + value, _ := cmd.Flags().GetString(flagName) + if value != "" { + err := os.Setenv(envName, value) + if err != nil { + return value + } + } + return os.Getenv(envName) +} +func FlagGetString(cmd *cobra.Command, flagName string) string { + value, _ := cmd.Flags().GetString(flagName) + if value != "" { + return value + + } + return "" +} +func FlagGetBool(cmd *cobra.Command, flagName string) bool { + value, _ := cmd.Flags().GetBool(flagName) + return value +} + +func SetEnv(key, value string) { + + err := os.Setenv(key, value) + if err != nil { + return + } +} + +func ShowHistory() { }