Merge pull request #168 from jkaninda/nightly

Nightly

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23.4 AS build
+FROM golang:1.23.5 AS build
 WORKDIR /app
 ARG appVersion=""
 

docs/how-tos/mutli-backup.md

@@ -8,51 +8,62 @@ nav_order: 11
 
 # Multiple Backup Schedules
 
-You can configure multiple backup schedules with different configurations by using a configuration file. 
-
-This file can be mounted into the container at `/config/config.yaml`, `/config/config.yml`, or specified via the `BACKUP_CONFIG_FILE` environment variable.
+This tool supports running multiple database backup schedules within the same container.
+You can configure these schedules with different settings using a **configuration file**. This flexibility allows you to manage backups for multiple databases efficiently.
 
 ---
 
-## Configuration File
+## Configuration File Setup
 
-The configuration file allows you to define multiple databases and their respective backup settings. 
+The configuration file can be mounted into the container at `/config/config.yaml`, `/config/config.yml`, or specified via the `BACKUP_CONFIG_FILE` environment variable.
 
-Below is an example configuration file:
+### Key Features:
+- **Global Environment Variables**: Use these for databases that share the same configuration.
+- **Database-Specific Overrides**: Override global settings for individual databases by specifying them in the configuration file or using the database name as a suffix in the variable name (e.g., `DB_HOST_DATABASE1`).
+- **Global Cron Expression**: Define a global `cronExpression` in the configuration file to schedule backups for all databases. If omitted, backups will run immediately.
+- **Configuration File Path**: Specify the configuration file path using:
+    - The `BACKUP_CONFIG_FILE` environment variable.
+    - The `--config` or `-c` flag for the backup command.
+
+---
+
+## Configuration File Example
+
+Below is an example configuration file (`config.yaml`) that defines multiple databases and their respective backup settings:
 
 ```yaml
-# Optional: Define a global cron expression for scheduled backups
-# cronExpression: "@every 20m"
+# Optional: Define a global cron expression for scheduled backups.
+# Example: "@every 20m" (runs every 20 minutes). If omitted, backups run immediately.
 cronExpression: ""
 
 databases:
-  - host: mysql1
-    port: 3306
-    name: database1
-    user: database1
-    password: password
-    path: /s3-path/database1  # For SSH or FTP, define the full path (e.g., /home/toto/backup/)
+  - host: mysql1       # Optional: Overrides DB_HOST or uses DB_HOST_DATABASE1.
+    port: 3306            # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_DATABASE1.
+    name: database1       # Required: Database name.
+    user: database1       # Optional: Overrides DB_USERNAME or uses DB_USERNAME_DATABASE1.
+    password: password    # Optional: Overrides DB_PASSWORD or uses DB_PASSWORD_DATABASE1.
+    path: /s3-path/database1  # Required: Backup path for SSH, FTP, or S3 (e.g., /home/toto/backup/).
 
-  - host: mysql2
-    port: 3306
-    name: lldap
-    user: lldap
-    password: password
-    path: /s3-path/lldap  # For SSH or FTP, define the full path (e.g., /home/toto/backup/)
+  - host: mysql2       # Optional: Overrides DB_HOST or uses DB_HOST_LLAP.
+    port: 3306            # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_LLAP.
+    name: lldap           # Required: Database name.
+    user: lldap           # Optional: Overrides DB_USERNAME or uses DB_USERNAME_LLAP.
+    password: password    # Optional: Overrides DB_PASSWORD or uses DB_PASSWORD_LLAP.
+    path: /s3-path/lldap  # Required: Backup path for SSH, FTP, or S3 (e.g., /home/toto/backup/).
 
-  - host: mysql3
-    port: 3306
-    name: keycloak
-    user: keycloak
-    password: password
-    path: /s3-path/keycloak  # For SSH or FTP, define the full path (e.g., /home/toto/backup/)
+  - host: mysql3       # Optional: Overrides DB_HOST or uses DB_HOST_KEYCLOAK.
+    port: 3306            # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_KEYCLOAK.
+    name: keycloak        # Required: Database name.
+    user: keycloak        # Optional: Overrides DB_USERNAME or uses DB_USERNAME_KEYCLOAK.
+    password: password    # Optional: Overrides DB_PASSWORD or uses DB_PASSWORD_KEYCLOAK.
+    path: /s3-path/keycloak  # Required: Backup path for SSH, FTP, or S3 (e.g., /home/toto/backup/).
 
-  - host: mysql4
-    port: 3306
-    name: joplin
-    user: joplin
-    password: password
-    path: /s3-path/joplin  # For SSH or FTP, define the full path (e.g., /home/toto/backup/)
+  - host: mysql4       # Optional: Overrides DB_HOST or uses DB_HOST_JOPLIN.
+    port: 3306            # Optional: Default is 5432. Overrides DB_PORT or uses DB_PORT_JOPLIN.
+    name: joplin          # Required: Database name.
+    user: joplin          # Optional: Overrides DB_USERNAME or uses DB_USERNAME_JOPLIN.
+    password: password    # Optional: Overrides DB_PASSWORD or uses DB_PASSWORD_JOPLIN.
+    path: /s3-path/joplin  # Required: Backup path for SSH, FTP, or S3 (e.g., /home/toto/backup/).
 ```
 
 ---
@@ -88,9 +99,5 @@ networks:
 
 ---
 
-## Key Notes
 
-- **Global Cron Expression**: You can define a global `cronExpression` in the configuration file to schedule backups for all databases. If omitted, backups will run immediately.
-- **Database-Specific Paths**: For SSH or FTP storage, ensure the `path` field contains the full remote path (e.g., `/home/toto/backup/`).
-- **Environment Variables**: Use the `BACKUP_CONFIG_FILE` environment variable to specify the path to the configuration file.
-- **Security**: Avoid hardcoding sensitive information like passwords in the configuration file. Use environment variables or secrets management tools instead.
+

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jkaninda/go-utils v0.0.0-20250122060806-26119182077a // indirect
 	github.com/jlaffaye/ftp v0.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect

go.sum

@@ -43,6 +43,10 @@ github.com/jkaninda/encryptor v0.0.0-20241111100652-926393c9437e h1:jtFKZHt/PLGQ
 github.com/jkaninda/encryptor v0.0.0-20241111100652-926393c9437e/go.mod h1:Y1EXpPWQ9PNd7y7E6ez3xgnzZc8fuDWXwX/1/dXNCE4=
 github.com/jkaninda/go-storage v0.1.3 h1:lEpHVgFLKSvjsi/6tAek96Y07za3vxmsXF2/+jiCMZU=
 github.com/jkaninda/go-storage v0.1.3/go.mod h1:zVRnLprBk/9AUz2+za6Y03MgoNYrqKLy3edVtjqMaps=
+github.com/jkaninda/go-utils v0.0.0-20250122054739-d330fecee150 h1:AgcKk58P/Z+u4DBE2MTyZ6kCweA89YUpX7TuttHS3oQ=
+github.com/jkaninda/go-utils v0.0.0-20250122054739-d330fecee150/go.mod h1:pf0/U6k4JbxlablM2G4eSTZdQ2LFshfAsCK5Q8qNfGo=
+github.com/jkaninda/go-utils v0.0.0-20250122060806-26119182077a h1:ZTpKujQGhEF266RWkD2cXnCsafk3R2+sGtfbzCQSs1s=
+github.com/jkaninda/go-utils v0.0.0-20250122060806-26119182077a/go.mod h1:pf0/U6k4JbxlablM2G4eSTZdQ2LFshfAsCK5Q8qNfGo=
 github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
 github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=

pkg/azure.go

@@ -27,6 +27,7 @@ package pkg
 import (
 	"fmt"
 	"github.com/jkaninda/go-storage/pkg/azure"
+	goutils "github.com/jkaninda/go-utils"
 	"github.com/jkaninda/mysql-bkup/utils"
 
 	"os"
@@ -36,7 +37,6 @@ import (
 
 func azureBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Azure Blob Storage")
-	startTime = time.Now().Format(utils.TimeFormat())
 
 	// Backup database
 	BackupDatabase(db, config.backupFileName, disableCompression)
@@ -87,6 +87,8 @@ func azureBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup size: %s", utils.ConvertBytes(uint64(backupSize)))
 	utils.Info("Uploading backup archive to Azure Blob storage ... done ")
 
+	duration := goutils.FormatDuration(time.Since(startTime), 0)
+
 	// Send notification
 	utils.NotifySuccess(&utils.NotificationData{
 		File:           finalFileName,
@@ -94,12 +96,11 @@ func azureBackup(db *dbConfig, config *BackupConfig) {
 		Database:       db.dbName,
 		Storage:        config.storage,
 		BackupLocation: filepath.Join(config.remotePath, finalFileName),
-		StartTime:      startTime,
-		EndTime:        time.Now().Format(utils.TimeFormat()),
+		Duration:       duration,
 	})
 	// Delete temp
 	deleteTemp()
-	utils.Info("Backup completed successfully")
+	utils.Info("Backup successfully completed in %s", duration)
 }
 func azureRestore(db *dbConfig, conf *RestoreConfig) {
 	utils.Info("Restore database from Azure Blob storage")

pkg/backup.go

@@ -29,6 +29,7 @@ import (
 	"fmt"
 	"github.com/jkaninda/encryptor"
 	"github.com/jkaninda/go-storage/pkg/local"
+	goutils "github.com/jkaninda/go-utils"
 	"github.com/jkaninda/mysql-bkup/utils"
 	"github.com/robfig/cron/v3"
 	"github.com/spf13/cobra"
@@ -107,6 +108,7 @@ func multiBackupTask(databases []Database, bkConfig *BackupConfig) {
 // BackupTask backups database
 func BackupTask(db *dbConfig, config *BackupConfig) {
 	utils.Info("Starting backup task...")
+	startTime = time.Now()
 	// Generate file name
 	backupFileName := fmt.Sprintf("%s_%s.sql.gz", db.dbName, time.Now().Format("20060102_150405"))
 	if config.disableCompression {
@@ -118,7 +120,7 @@ func BackupTask(db *dbConfig, config *BackupConfig) {
 		localBackup(db, config)
 	case "s3", "S3":
 		s3Backup(db, config)
-	case "ssh", "SSH", "remote":
+	case "ssh", "SSH", "remote", "sftp":
 		sshBackup(db, config)
 	case "ftp", "FTP":
 		ftpBackup(db, config)
@@ -256,7 +258,6 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool
 }
 func localBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to local storage")
-	startTime = time.Now().Format(utils.TimeFormat())
 	BackupDatabase(db, config.backupFileName, disableCompression)
 	finalFileName := config.backupFileName
 	if config.encryption {
@@ -279,6 +280,8 @@ func localBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup name is %s", finalFileName)
 	utils.Info("Backup size: %s", utils.ConvertBytes(uint64(backupSize)))
 	utils.Info("Backup saved in %s", filepath.Join(storagePath, finalFileName))
+	duration := goutils.FormatDuration(time.Since(startTime), 0)
+
 	// Send notification
 	utils.NotifySuccess(&utils.NotificationData{
 		File:           finalFileName,
@@ -286,8 +289,7 @@ func localBackup(db *dbConfig, config *BackupConfig) {
 		Database:       db.dbName,
 		Storage:        config.storage,
 		BackupLocation: filepath.Join(storagePath, finalFileName),
-		StartTime:      startTime,
-		EndTime:        time.Now().Format(utils.TimeFormat()),
+		Duration:       duration,
 	})
 	// Delete old backup
 	if config.prune {
@@ -299,7 +301,7 @@ func localBackup(db *dbConfig, config *BackupConfig) {
 	}
 	// Delete temp
 	deleteTemp()
-	utils.Info("Backup completed successfully")
+	utils.Info("Backup successfully completed in %s", duration)
 }
 
 func encryptBackup(config *BackupConfig) {

pkg/config.go

@@ -30,6 +30,7 @@ import (
 	"github.com/spf13/cobra"
 	"os"
 	"strconv"
+	"strings"
 )
 
 type Database struct {
@@ -127,6 +128,11 @@ func initDbConfig(cmd *cobra.Command) *dbConfig {
 }
 
 func getDatabase(database Database) *dbConfig {
+	// Set default values from environment variables if not provided
+	database.User = getEnvOrDefault(database.User, "DB_USERNAME", database.Name, "")
+	database.Password = getEnvOrDefault(database.Password, "DB_PASSWORD", database.Name, "")
+	database.Host = getEnvOrDefault(database.Host, "DB_HOST", database.Name, "")
+	database.Port = getEnvOrDefault(database.Port, "DB_PORT", database.Name, "3306")
 	return &dbConfig{
 		dbHost:     database.Host,
 		dbPort:     database.Port,
@@ -136,6 +142,20 @@ func getDatabase(database Database) *dbConfig {
 	}
 }
 
+// Helper function to get environment variable or use a default value
+func getEnvOrDefault(currentValue, envKey, suffix, defaultValue string) string {
+	if currentValue != "" {
+		return currentValue
+	}
+	if suffix != "" {
+		envSuffix := os.Getenv(fmt.Sprintf("%s_%s", envKey, strings.ToUpper(suffix)))
+		if envSuffix != "" {
+			return envSuffix
+		}
+	}
+	return utils.EnvWithDefault(envKey, defaultValue)
+}
+
 // loadSSHConfig loads the SSH configuration from environment variables
 func loadSSHConfig() (*SSHConfig, error) {
 	utils.GetEnvVariable("SSH_HOST", "SSH_HOST_NAME")

pkg/remote.go

@@ -28,6 +28,7 @@ import (
 	"fmt"
 	"github.com/jkaninda/go-storage/pkg/ftp"
 	"github.com/jkaninda/go-storage/pkg/ssh"
+	goutils "github.com/jkaninda/go-utils"
 	"github.com/jkaninda/mysql-bkup/utils"
 
 	"os"
@@ -37,7 +38,6 @@ import (
 
 func sshBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to Remote server")
-	startTime = time.Now().Format(utils.TimeFormat())
 	// Backup database
 	BackupDatabase(db, config.backupFileName, disableCompression)
 	finalFileName := config.backupFileName
@@ -91,6 +91,8 @@ func sshBackup(db *dbConfig, config *BackupConfig) {
 
 	}
 	utils.Info("Uploading backup archive to remote storage ... done ")
+	duration := goutils.FormatDuration(time.Since(startTime), 0)
+
 	// Send notification
 	utils.NotifySuccess(&utils.NotificationData{
 		File:           finalFileName,
@@ -98,12 +100,11 @@ func sshBackup(db *dbConfig, config *BackupConfig) {
 		Database:       db.dbName,
 		Storage:        config.storage,
 		BackupLocation: filepath.Join(config.remotePath, finalFileName),
-		StartTime:      startTime,
-		EndTime:        time.Now().Format(utils.TimeFormat()),
+		Duration:       duration,
 	})
 	// Delete temp
 	deleteTemp()
-	utils.Info("Backup completed successfully")
+	utils.Info("Backup successfully completed in %s", duration)
 
 }
 func remoteRestore(db *dbConfig, conf *RestoreConfig) {
@@ -153,7 +154,6 @@ func ftpRestore(db *dbConfig, conf *RestoreConfig) {
 }
 func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup database to the remote FTP server")
-	startTime = time.Now().Format(utils.TimeFormat())
 
 	// Backup database
 	BackupDatabase(db, config.backupFileName, disableCompression)
@@ -203,6 +203,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) {
 	utils.Info("Backup name is %s", finalFileName)
 	utils.Info("Backup size: %s", utils.ConvertBytes(uint64(backupSize)))
 	utils.Info("Uploading backup archive to the remote FTP server ... done ")
+	duration := goutils.FormatDuration(time.Since(startTime), 0)
 
 	// Send notification
 	utils.NotifySuccess(&utils.NotificationData{
@@ -211,10 +212,9 @@ func ftpBackup(db *dbConfig, config *BackupConfig) {
 		Database:       db.dbName,
 		Storage:        config.storage,
 		BackupLocation: filepath.Join(config.remotePath, finalFileName),
-		StartTime:      startTime,
-		EndTime:        time.Now().Format(utils.TimeFormat()),
+		Duration:       duration,
 	})
 	// Delete temp
 	deleteTemp()
-	utils.Info("Backup completed successfully")
+	utils.Info("Backup successfully completed in %s", duration)
 }

pkg/s3.go

@@ -27,6 +27,7 @@ package pkg
 import (
 	"fmt"
 	"github.com/jkaninda/go-storage/pkg/s3"
+	goutils "github.com/jkaninda/go-utils"
 	"github.com/jkaninda/mysql-bkup/utils"
 
 	"os"
@@ -37,7 +38,6 @@ import (
 func s3Backup(db *dbConfig, config *BackupConfig) {
 
 	utils.Info("Backup database to s3 storage")
-	startTime = time.Now().Format(utils.TimeFormat())
 	// Backup database
 	BackupDatabase(db, config.backupFileName, disableCompression)
 	finalFileName := config.backupFileName
@@ -91,6 +91,7 @@ func s3Backup(db *dbConfig, config *BackupConfig) {
 	}
 	utils.Info("Backup saved in %s", filepath.Join(config.remotePath, finalFileName))
 	utils.Info("Uploading backup archive to remote storage S3 ... done ")
+	duration := goutils.FormatDuration(time.Since(startTime), 0)
 	// Send notification
 	utils.NotifySuccess(&utils.NotificationData{
 		File:           finalFileName,
@@ -98,12 +99,11 @@ func s3Backup(db *dbConfig, config *BackupConfig) {
 		Database:       db.dbName,
 		Storage:        config.storage,
 		BackupLocation: filepath.Join(config.remotePath, finalFileName),
-		StartTime:      startTime,
-		EndTime:        time.Now().Format(utils.TimeFormat()),
+		Duration:       duration,
 	})
 	// Delete temp
 	deleteTemp()
-	utils.Info("Backup completed successfully")
+	utils.Info("Backup successfully completed in %s", duration)
 
 }
 func s3Restore(db *dbConfig, conf *RestoreConfig) {

pkg/var.go

@@ -24,6 +24,8 @@ SOFTWARE.
 
 package pkg
 
+import "time"
+
 const tmpPath = "/tmp/backup"
 const gpgHome = "/config/gnupg"
 const gpgExtension = "gpg"
@@ -39,7 +41,7 @@ var (
 	encryption               = false
 	usingKey                 = false
 	backupSize         int64 = 0
-	startTime          string
+	startTime                = time.Now()
 )
 
 // dbHVars Required environment variables for database

templates/email.tmpl

@@ -52,8 +52,7 @@
         <h3>Backup Details:</h3>
         <ul>
             <li><strong>Database Name:</strong> {{.Database}}</li>
-            <li><strong>Backup Start Time:</strong> {{.StartTime}}</li>
-            <li><strong>Backup End Time:</strong> {{.EndTime}}</li>
+            <li><strong>Backup Duration:</strong> {{.Duration}}</li>
             <li><strong>Backup Storage:</strong> {{.Storage}}</li>
             <li><strong>Backup Location:</strong> {{.BackupLocation}}</li>
             <li><strong>Backup Size:</strong> {{.BackupSize}}</li>

templates/telegram.tmpl

@@ -6,8 +6,7 @@ Please find the details below:
 
 Backup Details:
 - Database Name: {{.Database}}
-- Backup Start Time: {{.StartTime}}
-- Backup EndTime: {{.EndTime}}
+- Backup Duration: {{.Duration}}
 - Backup Storage: {{.Storage}}
 - Backup Location: {{.BackupLocation}}
 - Backup Size: {{.BackupSize}}

utils/config.go

@@ -39,8 +39,7 @@ type NotificationData struct {
 	File            string
 	BackupSize      string
 	Database        string
-	StartTime       string
-	EndTime         string
+	Duration        string
 	Storage         string
 	BackupLocation  string
 	BackupReference string
@@ -84,3 +83,13 @@ func backupReference() string {
 const templatePath = "/config/templates"
 
 var DatabaseName = ""
+var vars = []string{
+	"TG_TOKEN",
+	"TG_CHAT_ID",
+}
+var mailVars = []string{
+	"MAIL_HOST",
+	"MAIL_PORT",
+	"MAIL_FROM",
+	"MAIL_TO",
+}

utils/notification.go

@@ -107,19 +107,6 @@ func sendMessage(msg string) error {
 }
 func NotifySuccess(notificationData *NotificationData) {
 	notificationData.BackupReference = backupReference()
-	var vars = []string{
-		"TG_TOKEN",
-		"TG_CHAT_ID",
-	}
-	var mailVars = []string{
-		"MAIL_HOST",
-		"MAIL_PORT",
-		"MAIL_USERNAME",
-		"MAIL_PASSWORD",
-		"MAIL_FROM",
-		"MAIL_TO",
-	}
-
 	// Email notification
 	err := CheckEnvVars(mailVars)
 	if err == nil {
@@ -147,18 +134,6 @@ func NotifySuccess(notificationData *NotificationData) {
 	}
 }
 func NotifyError(error string) {
-	var vars = []string{
-		"TG_TOKEN",
-		"TG_CHAT_ID",
-	}
-	var mailVars = []string{
-		"MAIL_HOST",
-		"MAIL_PORT",
-		"MAIL_USERNAME",
-		"MAIL_PASSWORD",
-		"MAIL_FROM",
-		"MAIL_TO",
-	}
 
 	// Email notification
 	err := CheckEnvVars(mailVars)