From c36044144556594f441c46b5b60cc61099162ae9 Mon Sep 17 00:00:00 2001 From: Jonas Kaninda Date: Fri, 6 Dec 2024 18:27:25 +0100 Subject: [PATCH] feat: add Azure Blob storage --- README.md | 1 + docs/how-tos/azure-blob.md | 48 ++++++++ docs/index.md | 1 + go.mod | 6 +- go.sum | 10 ++ internal/azure.go | 122 +++++++++++++++++++++ internal/backup.go | 213 +----------------------------------- internal/config.go | 20 ++++ internal/remote.go | 218 +++++++++++++++++++++++++++++++++++++ internal/restore.go | 82 +------------- internal/s3.go | 135 +++++++++++++++++++++++ internal/var.go | 5 + 12 files changed, 572 insertions(+), 289 deletions(-) create mode 100644 docs/how-tos/azure-blob.md create mode 100644 internal/azure.go create mode 100644 internal/remote.go create mode 100644 internal/s3.go diff --git a/README.md b/README.md index c9f2c6f..1c94406 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ It supports a variety of storage options and ensures data security through GPG e - AWS S3 or any S3-compatible object storage - FTP - SSH-compatible storage + - Azure Blob storage - **Data Security:** - Backups can be encrypted using **GPG** to ensure confidentiality. diff --git a/docs/how-tos/azure-blob.md b/docs/how-tos/azure-blob.md new file mode 100644 index 0000000..c4bbee9 --- /dev/null +++ b/docs/how-tos/azure-blob.md @@ -0,0 +1,48 @@ +--- +title: Azure Blob storage +layout: default +parent: How Tos +nav_order: 5 +--- +# Azure Blob storage + +{: .note } +As described on local backup section, to change the storage of you backup and use Azure Blob as storage. You need to add `--storage azure` (-s azure). +You can also specify a folder where you want to save you data by adding `--path my-custom-path` flag. + + +## Backup to S3 + +```yml +services: + mysql-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/mysql-bkup/releases + # for a list of available releases. + image: jkaninda/mysql-bkup + container_name: mysql-bkup + command: backup --storage s3 -d database --path /my-custom-path + environment: + - DB_PORT=3306 + - DB_HOST=mysql + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## Azure Blob configurations + - AZURE_STORAGE_CONTAINER_NAME=backup-container + - AZURE_STORAGE_ACCOUNT_NAME=account-name + - AZURE_STORAGE_ACCOUNT_KEY=Ppby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + ## In case you are using S3 alternative such as Minio and your Minio instance is not secured, you change it to true + - AWS_DISABLE_SSL="false" + - AWS_FORCE_PATH_STYLE=true # true for S3 alternative such as Minio + + # mysql-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` + + + diff --git a/docs/index.md b/docs/index.md index 6f2d9f3..7c97978 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,6 +17,7 @@ It supports a variety of storage options and ensures data security through GPG e - AWS S3 or any S3-compatible object storage - FTP - SSH-compatible storage + - Azure Blob storage - **Data Security:** - Backups can be encrypted using **GPG** to ensure confidentiality. diff --git a/go.mod b/go.mod index 0ffcd6b..218acac 100644 --- a/go.mod +++ b/go.mod @@ -7,13 +7,16 @@ require github.com/spf13/pflag v1.0.5 // indirect require ( github.com/go-mail/mail v2.3.1+incompatible github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221 - github.com/jkaninda/go-storage v0.1.1 + github.com/jkaninda/go-storage v0.1.2 github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.8.1 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/ProtonMail/gopenpgp/v2 v2.7.5 // indirect @@ -27,6 +30,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/pkg/errors v0.9.1 // indirect golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect diff --git a/go.sum b/go.sum index f43d588..91e689c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,9 @@ +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo= github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs= github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= @@ -30,6 +36,8 @@ github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221 h1:AwkCf7el1kze github.com/jkaninda/encryptor v0.0.0-20241013064832-ed4bd6a1b221/go.mod h1:9F8ZJ+ZXE8DZBo77+aneGj8LMjrYXX6eFUCC/uqZOUo= github.com/jkaninda/go-storage v0.1.1 h1:vjpdD/fh39S5HGyfHvLE5HGYOEPIukINlOX3OnM3GW4= github.com/jkaninda/go-storage v0.1.1/go.mod h1:7VK5gQISQaLxtLfBtc+een8spcgLVSBAKTRuyF1N81I= +github.com/jkaninda/go-storage v0.1.2 h1:d7+TRPjmHXdSqO0wne3KAB8zt9ih8lf5D8aL4n7/Dds= +github.com/jkaninda/go-storage v0.1.2/go.mod h1:zVRnLprBk/9AUz2+za6Y03MgoNYrqKLy3edVtjqMaps= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -66,6 +74,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/azure.go b/internal/azure.go new file mode 100644 index 0000000..2d6cc63 --- /dev/null +++ b/internal/azure.go @@ -0,0 +1,122 @@ +/* +MIT License + +Copyright (c) 2023 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package internal + +import ( + "fmt" + "github.com/jkaninda/go-storage/pkg/azure" + "github.com/jkaninda/mysql-bkup/pkg/logger" + "github.com/jkaninda/mysql-bkup/utils" + + "os" + "path/filepath" + "time" +) + +func azureBackup(db *dbConfig, config *BackupConfig) { + logger.Info("Backup database to the remote FTP server") + startTime = time.Now().Format(utils.TimeFormat()) + + // Backup database + BackupDatabase(db, config.backupFileName, disableCompression) + finalFileName := config.backupFileName + if config.encryption { + encryptBackup(config) + finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") + } + logger.Info("Uploading backup archive to Azure Blob storage ...") + logger.Info("Backup name is %s", finalFileName) + azureConfig := loadAzureConfig() + azureStorage, err := azure.NewStorage(azure.Config{ + ContainerName: azureConfig.containerName, + AccountName: azureConfig.accountName, + AccountKey: azureConfig.accountKey, + RemotePath: config.remotePath, + LocalPath: tmpPath, + }) + if err != nil { + logger.Fatal("Error creating SSH storage: %s", err) + } + err = azureStorage.Copy(finalFileName) + if err != nil { + logger.Fatal("Error copying backup file: %s", err) + } + logger.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) + // Get backup info + fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) + if err != nil { + logger.Error("Error: %s", err) + } + backupSize = fileInfo.Size() + // Delete backup file from tmp folder + err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) + if err != nil { + logger.Error("Error deleting file: %v", err) + + } + if config.prune { + err := azureStorage.Prune(config.backupRetention) + if err != nil { + logger.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) + } + + } + + logger.Info("Uploading backup archive to Azure Blob storage ... done ") + + // Send notification + utils.NotifySuccess(&utils.NotificationData{ + File: finalFileName, + BackupSize: backupSize, + Database: db.dbName, + Storage: config.storage, + BackupLocation: filepath.Join(config.remotePath, finalFileName), + StartTime: startTime, + EndTime: time.Now().Format(utils.TimeFormat()), + }) + // Delete temp + deleteTemp() + logger.Info("Backup completed successfully") +} +func azureRestore(db *dbConfig, conf *RestoreConfig) { + logger.Info("Restore database from Azure Blob storage") + azureConfig := loadAzureConfig() + azureStorage, err := azure.NewStorage(azure.Config{ + ContainerName: azureConfig.containerName, + AccountName: azureConfig.accountName, + AccountKey: azureConfig.accountKey, + RemotePath: conf.remotePath, + LocalPath: tmpPath, + }) + if err != nil { + logger.Fatal("Error creating SSH storage: %s", err) + } + + err = azureStorage.CopyFrom(conf.file) + if err != nil { + logger.Fatal("Error downloading backup file: %s", err) + } + RestoreDatabase(db, conf) +} diff --git a/internal/backup.go b/internal/backup.go index d81be23..5a8ed6a 100644 --- a/internal/backup.go +++ b/internal/backup.go @@ -27,10 +27,7 @@ package internal import ( "fmt" "github.com/jkaninda/encryptor" - "github.com/jkaninda/go-storage/pkg/ftp" "github.com/jkaninda/go-storage/pkg/local" - "github.com/jkaninda/go-storage/pkg/s3" - "github.com/jkaninda/go-storage/pkg/ssh" "github.com/jkaninda/mysql-bkup/pkg/logger" "github.com/jkaninda/mysql-bkup/utils" "github.com/robfig/cron/v3" @@ -125,6 +122,8 @@ func BackupTask(db *dbConfig, config *BackupConfig) { sshBackup(db, config) case "ftp", "FTP": ftpBackup(db, config) + case "azure": + azureBackup(db, config) default: localBackup(db, config) } @@ -302,214 +301,6 @@ func localBackup(db *dbConfig, config *BackupConfig) { logger.Info("Backup completed successfully") } -func s3Backup(db *dbConfig, config *BackupConfig) { - - logger.Info("Backup database to s3 storage") - startTime = time.Now().Format(utils.TimeFormat()) - // Backup database - BackupDatabase(db, config.backupFileName, disableCompression) - finalFileName := config.backupFileName - if config.encryption { - encryptBackup(config) - finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") - } - logger.Info("Uploading backup archive to remote storage S3 ... ") - awsConfig := initAWSConfig() - if config.remotePath == "" { - config.remotePath = awsConfig.remotePath - } - logger.Info("Backup name is %s", finalFileName) - s3Storage, err := s3.NewStorage(s3.Config{ - Endpoint: awsConfig.endpoint, - Bucket: awsConfig.bucket, - AccessKey: awsConfig.accessKey, - SecretKey: awsConfig.secretKey, - Region: awsConfig.region, - DisableSsl: awsConfig.disableSsl, - ForcePathStyle: awsConfig.forcePathStyle, - RemotePath: awsConfig.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - logger.Fatal("Error creating s3 storage: %s", err) - } - err = s3Storage.Copy(finalFileName) - if err != nil { - logger.Fatal("Error copying backup file: %s", err) - } - // Get backup info - fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) - if err != nil { - logger.Error("Error: %s", err) - } - backupSize = fileInfo.Size() - - // Delete backup file from tmp folder - err = utils.DeleteFile(filepath.Join(tmpPath, config.backupFileName)) - if err != nil { - fmt.Println("Error deleting file: ", err) - - } - // Delete old backup - if config.prune { - err := s3Storage.Prune(config.backupRetention) - if err != nil { - logger.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) - } - } - logger.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) - logger.Info("Uploading backup archive to remote storage S3 ... done ") - // Send notification - utils.NotifySuccess(&utils.NotificationData{ - File: finalFileName, - BackupSize: backupSize, - Database: db.dbName, - Storage: config.storage, - BackupLocation: filepath.Join(config.remotePath, finalFileName), - StartTime: startTime, - EndTime: time.Now().Format(utils.TimeFormat()), - }) - // Delete temp - deleteTemp() - logger.Info("Backup completed successfully") - -} -func sshBackup(db *dbConfig, config *BackupConfig) { - logger.Info("Backup database to Remote server") - startTime = time.Now().Format(utils.TimeFormat()) - // Backup database - BackupDatabase(db, config.backupFileName, disableCompression) - finalFileName := config.backupFileName - if config.encryption { - encryptBackup(config) - finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") - } - logger.Info("Uploading backup archive to remote storage ... ") - logger.Info("Backup name is %s", finalFileName) - sshConfig, err := loadSSHConfig() - if err != nil { - logger.Fatal("Error loading ssh config: %s", err) - } - - sshStorage, err := ssh.NewStorage(ssh.Config{ - Host: sshConfig.hostName, - Port: sshConfig.port, - User: sshConfig.user, - Password: sshConfig.password, - RemotePath: config.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - logger.Fatal("Error creating SSH storage: %s", err) - } - err = sshStorage.Copy(finalFileName) - if err != nil { - logger.Fatal("Error copying backup file: %s", err) - } - // Get backup info - fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) - if err != nil { - logger.Error("Error: %s", err) - } - backupSize = fileInfo.Size() - logger.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) - - // Delete backup file from tmp folder - err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) - if err != nil { - logger.Error("Error deleting file: %v", err) - - } - if config.prune { - err := sshStorage.Prune(config.backupRetention) - if err != nil { - logger.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) - } - - } - logger.Info("Uploading backup archive to remote storage ... done ") - // Send notification - utils.NotifySuccess(&utils.NotificationData{ - File: finalFileName, - BackupSize: backupSize, - Database: db.dbName, - Storage: config.storage, - BackupLocation: filepath.Join(config.remotePath, finalFileName), - StartTime: startTime, - EndTime: time.Now().Format(utils.TimeFormat()), - }) - // Delete temp - deleteTemp() - logger.Info("Backup completed successfully") - -} -func ftpBackup(db *dbConfig, config *BackupConfig) { - logger.Info("Backup database to the remote FTP server") - startTime = time.Now().Format(utils.TimeFormat()) - - // Backup database - BackupDatabase(db, config.backupFileName, disableCompression) - finalFileName := config.backupFileName - if config.encryption { - encryptBackup(config) - finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") - } - logger.Info("Uploading backup archive to the remote FTP server ... ") - logger.Info("Backup name is %s", finalFileName) - ftpConfig := loadFtpConfig() - ftpStorage, err := ftp.NewStorage(ftp.Config{ - Host: ftpConfig.host, - Port: ftpConfig.port, - User: ftpConfig.user, - Password: ftpConfig.password, - RemotePath: config.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - logger.Fatal("Error creating SSH storage: %s", err) - } - err = ftpStorage.Copy(finalFileName) - if err != nil { - logger.Fatal("Error copying backup file: %s", err) - } - logger.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) - // Get backup info - fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) - if err != nil { - logger.Error("Error: %s", err) - } - backupSize = fileInfo.Size() - // Delete backup file from tmp folder - err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) - if err != nil { - logger.Error("Error deleting file: %v", err) - - } - if config.prune { - err := ftpStorage.Prune(config.backupRetention) - if err != nil { - logger.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) - } - - } - - logger.Info("Uploading backup archive to the remote FTP server ... done ") - - // Send notification - utils.NotifySuccess(&utils.NotificationData{ - File: finalFileName, - BackupSize: backupSize, - Database: db.dbName, - Storage: config.storage, - BackupLocation: filepath.Join(config.remotePath, finalFileName), - StartTime: startTime, - EndTime: time.Now().Format(utils.TimeFormat()), - }) - // Delete temp - deleteTemp() - logger.Info("Backup completed successfully") -} - func encryptBackup(config *BackupConfig) { backupFile, err := os.ReadFile(filepath.Join(tmpPath, config.backupFileName)) outputFile := fmt.Sprintf("%s.%s", filepath.Join(tmpPath, config.backupFileName), gpgExtension) diff --git a/internal/config.go b/internal/config.go index 879c5fc..ac99784 100644 --- a/internal/config.go +++ b/internal/config.go @@ -84,6 +84,11 @@ type FTPConfig struct { port string remotePath string } +type AzureConfig struct { + accountName string + accountKey string + containerName string +} // SSHConfig holds the SSH connection details type SSHConfig struct { @@ -164,6 +169,21 @@ func loadFtpConfig() *FTPConfig { } return &fConfig } +func loadAzureConfig() *AzureConfig { + // Initialize data configs + aConfig := AzureConfig{} + aConfig.containerName = os.Getenv("AZURE_STORAGE_CONTAINER_NAME") + aConfig.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT_NAME") + aConfig.accountKey = os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") + + err := utils.CheckEnvVars(azureVars) + if err != nil { + logger.Error("Please make sure all required environment variables for Azure Blob storage are set") + logger.Fatal("Error missing environment variables: %s", err) + } + return &aConfig +} + func initAWSConfig() *AWSConfig { // Initialize AWS configs aConfig := AWSConfig{} diff --git a/internal/remote.go b/internal/remote.go new file mode 100644 index 0000000..5ba395a --- /dev/null +++ b/internal/remote.go @@ -0,0 +1,218 @@ +/* +MIT License + +Copyright (c) 2023 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package internal + +import ( + "fmt" + "github.com/jkaninda/go-storage/pkg/ftp" + "github.com/jkaninda/go-storage/pkg/ssh" + "github.com/jkaninda/mysql-bkup/pkg/logger" + "github.com/jkaninda/mysql-bkup/utils" + + "os" + "path/filepath" + "time" +) + +func sshBackup(db *dbConfig, config *BackupConfig) { + logger.Info("Backup database to Remote server") + startTime = time.Now().Format(utils.TimeFormat()) + // Backup database + BackupDatabase(db, config.backupFileName, disableCompression) + finalFileName := config.backupFileName + if config.encryption { + encryptBackup(config) + finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") + } + logger.Info("Uploading backup archive to remote storage ... ") + logger.Info("Backup name is %s", finalFileName) + sshConfig, err := loadSSHConfig() + if err != nil { + logger.Fatal("Error loading ssh config: %s", err) + } + + sshStorage, err := ssh.NewStorage(ssh.Config{ + Host: sshConfig.hostName, + Port: sshConfig.port, + User: sshConfig.user, + Password: sshConfig.password, + RemotePath: config.remotePath, + LocalPath: tmpPath, + }) + if err != nil { + logger.Fatal("Error creating SSH storage: %s", err) + } + err = sshStorage.Copy(finalFileName) + if err != nil { + logger.Fatal("Error copying backup file: %s", err) + } + // Get backup info + fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) + if err != nil { + logger.Error("Error: %s", err) + } + backupSize = fileInfo.Size() + logger.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) + + // Delete backup file from tmp folder + err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) + if err != nil { + logger.Error("Error deleting file: %v", err) + + } + if config.prune { + err := sshStorage.Prune(config.backupRetention) + if err != nil { + logger.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) + } + + } + logger.Info("Uploading backup archive to remote storage ... done ") + // Send notification + utils.NotifySuccess(&utils.NotificationData{ + File: finalFileName, + BackupSize: backupSize, + Database: db.dbName, + Storage: config.storage, + BackupLocation: filepath.Join(config.remotePath, finalFileName), + StartTime: startTime, + EndTime: time.Now().Format(utils.TimeFormat()), + }) + // Delete temp + deleteTemp() + logger.Info("Backup completed successfully") + +} +func ftpBackup(db *dbConfig, config *BackupConfig) { + logger.Info("Backup database to the remote FTP server") + startTime = time.Now().Format(utils.TimeFormat()) + + // Backup database + BackupDatabase(db, config.backupFileName, disableCompression) + finalFileName := config.backupFileName + if config.encryption { + encryptBackup(config) + finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") + } + logger.Info("Uploading backup archive to the remote FTP server ... ") + logger.Info("Backup name is %s", finalFileName) + ftpConfig := loadFtpConfig() + ftpStorage, err := ftp.NewStorage(ftp.Config{ + Host: ftpConfig.host, + Port: ftpConfig.port, + User: ftpConfig.user, + Password: ftpConfig.password, + RemotePath: config.remotePath, + LocalPath: tmpPath, + }) + if err != nil { + logger.Fatal("Error creating SSH storage: %s", err) + } + err = ftpStorage.Copy(finalFileName) + if err != nil { + logger.Fatal("Error copying backup file: %s", err) + } + logger.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) + // Get backup info + fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) + if err != nil { + logger.Error("Error: %s", err) + } + backupSize = fileInfo.Size() + // Delete backup file from tmp folder + err = utils.DeleteFile(filepath.Join(tmpPath, finalFileName)) + if err != nil { + logger.Error("Error deleting file: %v", err) + + } + if config.prune { + err := ftpStorage.Prune(config.backupRetention) + if err != nil { + logger.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) + } + + } + + logger.Info("Uploading backup archive to the remote FTP server ... done ") + + // Send notification + utils.NotifySuccess(&utils.NotificationData{ + File: finalFileName, + BackupSize: backupSize, + Database: db.dbName, + Storage: config.storage, + BackupLocation: filepath.Join(config.remotePath, finalFileName), + StartTime: startTime, + EndTime: time.Now().Format(utils.TimeFormat()), + }) + // Delete temp + deleteTemp() + logger.Info("Backup completed successfully") +} +func remoteRestore(db *dbConfig, conf *RestoreConfig) { + logger.Info("Restore database from remote server") + sshConfig, err := loadSSHConfig() + if err != nil { + logger.Fatal("Error loading ssh config: %s", err) + } + + sshStorage, err := ssh.NewStorage(ssh.Config{ + Host: sshConfig.hostName, + Port: sshConfig.port, + User: sshConfig.user, + Password: sshConfig.password, + IdentifyFile: sshConfig.identifyFile, + RemotePath: conf.remotePath, + LocalPath: tmpPath, + }) + if err != nil { + logger.Fatal("Error creating SSH storage: %s", err) + } + err = sshStorage.CopyFrom(conf.file) + if err != nil { + logger.Fatal("Error copying backup file: %s", err) + } + RestoreDatabase(db, conf) +} +func ftpRestore(db *dbConfig, conf *RestoreConfig) { + logger.Info("Restore database from FTP server") + ftpConfig := loadFtpConfig() + ftpStorage, err := ftp.NewStorage(ftp.Config{ + Host: ftpConfig.host, + Port: ftpConfig.port, + User: ftpConfig.user, + Password: ftpConfig.password, + RemotePath: conf.remotePath, + LocalPath: tmpPath, + }) + if err != nil { + logger.Fatal("Error creating SSH storage: %s", err) + } + err = ftpStorage.CopyFrom(conf.file) + if err != nil { + logger.Fatal("Error copying backup file: %s", err) + } + RestoreDatabase(db, conf) +} diff --git a/internal/restore.go b/internal/restore.go index 56db32e..12780e6 100644 --- a/internal/restore.go +++ b/internal/restore.go @@ -26,10 +26,7 @@ SOFTWARE. */ import ( "github.com/jkaninda/encryptor" - "github.com/jkaninda/go-storage/pkg/ftp" "github.com/jkaninda/go-storage/pkg/local" - "github.com/jkaninda/go-storage/pkg/s3" - "github.com/jkaninda/go-storage/pkg/ssh" "github.com/jkaninda/mysql-bkup/pkg/logger" "github.com/jkaninda/mysql-bkup/utils" "github.com/spf13/cobra" @@ -47,11 +44,13 @@ func StartRestore(cmd *cobra.Command) { case "local": localRestore(dbConf, restoreConf) case "s3", "S3": - restoreFromS3(dbConf, restoreConf) + s3Restore(dbConf, restoreConf) case "ssh", "SSH", "remote": - restoreFromRemote(dbConf, restoreConf) + remoteRestore(dbConf, restoreConf) case "ftp", "FTP": - restoreFromFTP(dbConf, restoreConf) + ftpRestore(dbConf, restoreConf) + case "azure": + azureRestore(dbConf, restoreConf) default: localRestore(dbConf, restoreConf) } @@ -69,77 +68,6 @@ func localRestore(dbConf *dbConfig, restoreConf *RestoreConfig) { RestoreDatabase(dbConf, restoreConf) } -func restoreFromS3(db *dbConfig, conf *RestoreConfig) { - logger.Info("Restore database from s3") - awsConfig := initAWSConfig() - if conf.remotePath == "" { - conf.remotePath = awsConfig.remotePath - } - s3Storage, err := s3.NewStorage(s3.Config{ - Endpoint: awsConfig.endpoint, - Bucket: awsConfig.bucket, - AccessKey: awsConfig.accessKey, - SecretKey: awsConfig.secretKey, - Region: awsConfig.region, - DisableSsl: awsConfig.disableSsl, - ForcePathStyle: awsConfig.forcePathStyle, - RemotePath: awsConfig.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - logger.Fatal("Error creating s3 storage: %s", err) - } - err = s3Storage.CopyFrom(conf.file) - if err != nil { - logger.Fatal("Error download file from S3 storage: %s", err) - } - RestoreDatabase(db, conf) -} -func restoreFromRemote(db *dbConfig, conf *RestoreConfig) { - logger.Info("Restore database from remote server") - sshConfig, err := loadSSHConfig() - if err != nil { - logger.Fatal("Error loading ssh config: %s", err) - } - - sshStorage, err := ssh.NewStorage(ssh.Config{ - Host: sshConfig.hostName, - Port: sshConfig.port, - User: sshConfig.user, - Password: sshConfig.password, - IdentifyFile: sshConfig.identifyFile, - RemotePath: conf.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - logger.Fatal("Error creating SSH storage: %s", err) - } - err = sshStorage.CopyFrom(conf.file) - if err != nil { - logger.Fatal("Error copying backup file: %s", err) - } - RestoreDatabase(db, conf) -} -func restoreFromFTP(db *dbConfig, conf *RestoreConfig) { - logger.Info("Restore database from FTP server") - ftpConfig := loadFtpConfig() - ftpStorage, err := ftp.NewStorage(ftp.Config{ - Host: ftpConfig.host, - Port: ftpConfig.port, - User: ftpConfig.user, - Password: ftpConfig.password, - RemotePath: conf.remotePath, - LocalPath: tmpPath, - }) - if err != nil { - logger.Fatal("Error creating SSH storage: %s", err) - } - err = ftpStorage.CopyFrom(conf.file) - if err != nil { - logger.Fatal("Error copying backup file: %s", err) - } - RestoreDatabase(db, conf) -} // RestoreDatabase restore database func RestoreDatabase(db *dbConfig, conf *RestoreConfig) { diff --git a/internal/s3.go b/internal/s3.go new file mode 100644 index 0000000..6afb694 --- /dev/null +++ b/internal/s3.go @@ -0,0 +1,135 @@ +/* +MIT License + +Copyright (c) 2023 Jonas Kaninda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package internal + +import ( + "fmt" + "github.com/jkaninda/go-storage/pkg/s3" + "github.com/jkaninda/mysql-bkup/pkg/logger" + "github.com/jkaninda/mysql-bkup/utils" + + "os" + "path/filepath" + "time" +) + +func s3Backup(db *dbConfig, config *BackupConfig) { + + logger.Info("Backup database to s3 storage") + startTime = time.Now().Format(utils.TimeFormat()) + // Backup database + BackupDatabase(db, config.backupFileName, disableCompression) + finalFileName := config.backupFileName + if config.encryption { + encryptBackup(config) + finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") + } + logger.Info("Uploading backup archive to remote storage S3 ... ") + awsConfig := initAWSConfig() + if config.remotePath == "" { + config.remotePath = awsConfig.remotePath + } + logger.Info("Backup name is %s", finalFileName) + s3Storage, err := s3.NewStorage(s3.Config{ + Endpoint: awsConfig.endpoint, + Bucket: awsConfig.bucket, + AccessKey: awsConfig.accessKey, + SecretKey: awsConfig.secretKey, + Region: awsConfig.region, + DisableSsl: awsConfig.disableSsl, + ForcePathStyle: awsConfig.forcePathStyle, + RemotePath: awsConfig.remotePath, + LocalPath: tmpPath, + }) + if err != nil { + logger.Fatal("Error creating s3 storage: %s", err) + } + err = s3Storage.Copy(finalFileName) + if err != nil { + logger.Fatal("Error copying backup file: %s", err) + } + // Get backup info + fileInfo, err := os.Stat(filepath.Join(tmpPath, finalFileName)) + if err != nil { + logger.Error("Error: %s", err) + } + backupSize = fileInfo.Size() + + // Delete backup file from tmp folder + err = utils.DeleteFile(filepath.Join(tmpPath, config.backupFileName)) + if err != nil { + fmt.Println("Error deleting file: ", err) + + } + // Delete old backup + if config.prune { + err := s3Storage.Prune(config.backupRetention) + if err != nil { + logger.Fatal("Error deleting old backup from %s storage: %s ", config.storage, err) + } + } + logger.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName)) + logger.Info("Uploading backup archive to remote storage S3 ... done ") + // Send notification + utils.NotifySuccess(&utils.NotificationData{ + File: finalFileName, + BackupSize: backupSize, + Database: db.dbName, + Storage: config.storage, + BackupLocation: filepath.Join(config.remotePath, finalFileName), + StartTime: startTime, + EndTime: time.Now().Format(utils.TimeFormat()), + }) + // Delete temp + deleteTemp() + logger.Info("Backup completed successfully") + +} +func s3Restore(db *dbConfig, conf *RestoreConfig) { + logger.Info("Restore database from s3") + awsConfig := initAWSConfig() + if conf.remotePath == "" { + conf.remotePath = awsConfig.remotePath + } + s3Storage, err := s3.NewStorage(s3.Config{ + Endpoint: awsConfig.endpoint, + Bucket: awsConfig.bucket, + AccessKey: awsConfig.accessKey, + SecretKey: awsConfig.secretKey, + Region: awsConfig.region, + DisableSsl: awsConfig.disableSsl, + ForcePathStyle: awsConfig.forcePathStyle, + RemotePath: awsConfig.remotePath, + LocalPath: tmpPath, + }) + if err != nil { + logger.Fatal("Error creating s3 storage: %s", err) + } + err = s3Storage.CopyFrom(conf.file) + if err != nil { + logger.Fatal("Error download file from S3 storage: %s", err) + } + RestoreDatabase(db, conf) +} diff --git a/internal/var.go b/internal/var.go index d04f648..5f9a2a9 100644 --- a/internal/var.go +++ b/internal/var.go @@ -74,6 +74,11 @@ var ftpVars = []string{ "FTP_PASSWORD", "FTP_PORT", } +var azureVars = []string{ + "AZURE_STORAGE_CONTAINER_NAME", + "AZURE_STORAGE_ACCOUNT_NAME", + "AZURE_STORAGE_ACCOUNT_KEY", +} // AwsVars Required environment variables for AWS S3 storage var awsVars = []string{