diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9242f3d..3530887 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: uses: docker/build-push-action@v3 with: push: true - file: "./docker/Dockerfile" + file: "./Dockerfile" platforms: linux/amd64,linux/arm64,linux/arm/v7 build-args: | appVersion=develop-${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cb417ee..c6ea3c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: uses: docker/build-push-action@v3 with: push: true - file: "./docker/Dockerfile" + file: "./Dockerfile" platforms: linux/amd64,linux/arm64,linux/arm/v7 build-args: | appVersion=${{ env.TAG_NAME }} diff --git a/docker/Dockerfile b/Dockerfile similarity index 96% rename from docker/Dockerfile rename to Dockerfile index 7f2432c..7ae797a 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -52,7 +52,7 @@ ENV VERSION=${appVersion} LABEL author="Jonas Kaninda" LABEL version=${appVersion} -RUN apk --update add --no-cache postgresql-client gnupg tzdata +RUN apk --update add --no-cache postgresql-client tzdata RUN mkdir $WORKDIR RUN mkdir $BACKUPDIR RUN mkdir -p $BACKUP_TMP_DIR diff --git a/docs/how-tos/deprecated-configs.md b/docs/how-tos/deprecated-configs.md new file mode 100644 index 0000000..1813ff2 --- /dev/null +++ b/docs/how-tos/deprecated-configs.md @@ -0,0 +1,6 @@ +--- +title: Update deprecated configurations +layout: default +parent: How Tos +nav_order: 11 +--- \ No newline at end of file diff --git a/docs/how-tos/encrypt-backup.md b/docs/how-tos/encrypt-backup.md index b228fe2..d784a50 100644 --- a/docs/how-tos/encrypt-backup.md +++ b/docs/how-tos/encrypt-backup.md @@ -6,14 +6,23 @@ nav_order: 8 --- # Encrypt backup -The image supports encrypting backups using GPG out of the box. In case a `GPG_PASSPHRASE` environment variable is set, the backup archive will be encrypted using the given key and saved as a sql.gpg file instead or sql.gz.gpg. +The image supports encrypting backups using one of two available methods: GPG with passphrase or GPG with a public key. + + +The image supports encrypting backups using GPG out of the box. In case a `GPG_PASSPHRASE` or `GPG_PUBLIC_KEY` environment variable is set, the backup archive will be encrypted using the given key and saved as a sql.gpg file instead or sql.gz.gpg. +Suppose you used a GPG public key during the backup process. In that case, you need to decrypt your backup before restoration because decryption using a `GPG private` key is not fully supported. + {: .warning } To restore an encrypted backup, you need to provide the same GPG passphrase used during backup process. - GPG home directory `/config/gnupg` - Cipher algorithm `aes256` -- + +{: .note } +The backup encrypted using `GPG passphrase` method can be restored automatically, no need to decrypt it before restoration. + + To decrypt manually, you need to install `gnupg` ```shell @@ -21,8 +30,11 @@ gpg --batch --passphrase "my-passphrase" \ --output database_20240730_044201.sql.gz \ --decrypt database_20240730_044201.sql.gz.gpg ``` - -### Backup +Using your private key +```shell +gpg --output database_20240730_044201.sql.gz --decrypt database_20240730_044201.sql.gz.gpg +``` +## Using GPG passphrase ```yml services: @@ -49,4 +61,33 @@ services: - web networks: web: -``` \ No newline at end of file +``` + +## Using GPG Public Key + +```yml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: backup -d database + volumes: + - ./backup:/backup + environment: + - DB_PORT=5432 + - DB_HOST=postgres + - DB_NAME=database + - DB_USERNAME=username + - DB_PASSWORD=password + ## Required to encrypt backup + - GPG_PUBLIC_KEY=/config/public_key.asc + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` diff --git a/docs/how-tos/mutli-backup.md b/docs/how-tos/mutli-backup.md new file mode 100644 index 0000000..45f4440 --- /dev/null +++ b/docs/how-tos/mutli-backup.md @@ -0,0 +1,64 @@ +--- +title: Run multiple database backup schedules in the same container +layout: default +parent: How Tos +nav_order: 11 +--- + +Multiple backup schedules with different configuration can be configured by mounting a configuration file into `/config/config.yaml` `/config/config.yml` or by defining an environment variable `BACKUP_CONFIG_FILE=/backup/config.yaml`. + +## Configuration file + +```yaml +#cronExpression: "@every 20m" //Optional, for scheduled backups +cronExpression: "" +databases: + - host: postgres1 + port: 5432 + name: database1 + user: database1 + password: password + path: /s3-path/database1 #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres2 + port: 5432 + name: lldap + user: lldap + password: password + path: /s3-path/lldap #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres3 + port: 5432 + name: keycloak + user: keycloak + password: password + path: /s3-path/keycloak #For SSH or FTP you need to define the full path (/home/toto/backup/) + - host: postgres4 + port: 5432 + name: joplin + user: joplin + password: password + path: /s3-path/joplin #For SSH or FTP you need to define the full path (/home/toto/backup/) +``` + +## Docker compose file + +```yaml +services: + pg-bkup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/jkaninda/pg-bkup/releases + # for a list of available releases. + image: jkaninda/pg-bkup + container_name: pg-bkup + command: backup + volumes: + - ./backup:/backup + environment: + ## Multi backup config file + - BACKUP_CONFIG_FILE=/backup/config.yaml + # pg-bkup container must be connected to the same network with your database + networks: + - web +networks: + web: +``` \ No newline at end of file diff --git a/docs/reference/index.md b/docs/reference/index.md index 43e8cb8..5fbd341 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -34,41 +34,42 @@ Backup, restore and migrate targets, schedule and retention are configured using ## Environment variables -| Name | Requirement | Description | -|------------------------|---------------------------------------------------------------|------------------------------------------------------| -| DB_PORT | Optional, default 5432 | Database port number | -| DB_HOST | Required | Database host | -| DB_NAME | Optional if it was provided from the -d flag | Database name | -| DB_USERNAME | Required | Database user name | -| DB_PASSWORD | Required | Database password | -| AWS_ACCESS_KEY | Optional, required for S3 storage | AWS S3 Access Key | -| AWS_SECRET_KEY | Optional, required for S3 storage | AWS S3 Secret Key | -| AWS_BUCKET_NAME | Optional, required for S3 storage | AWS S3 Bucket Name | -| AWS_BUCKET_NAME | Optional, required for S3 storage | AWS S3 Bucket Name | -| AWS_REGION | Optional, required for S3 storage | AWS Region | -| AWS_DISABLE_SSL | Optional, required for S3 storage | Disable SSL | -| AWS_FORCE_PATH_STYLE | Optional, required for S3 storage | Force path style | -| FILE_NAME | Optional if it was provided from the --file flag | Database file to restore (extensions: .sql, .sql.gz) | -| GPG_PASSPHRASE | Optional, required to encrypt and restore backup | GPG passphrase | -| BACKUP_CRON_EXPRESSION | Optional if it was provided from the `--cron-expression` flag | Backup cron expression for docker in scheduled mode | -| SSH_HOST | Optional, required for SSH storage | ssh remote hostname or ip | -| SSH_USER | Optional, required for SSH storage | ssh remote user | -| SSH_PASSWORD | Optional, required for SSH storage | ssh remote user's password | -| SSH_IDENTIFY_FILE | Optional, required for SSH storage | ssh remote user's private key | -| SSH_PORT | Optional, required for SSH storage | ssh remote server port | -| REMOTE_PATH | Optional, required for SSH or FTP storage | remote path (/home/toto/backup) | -| FTP_HOST | Optional, required for FTP storage | FTP host name | -| FTP_PORT | Optional, required for FTP storage | FTP server port number | -| FTP_USER | Optional, required for FTP storage | FTP user | -| FTP_PASSWORD | Optional, required for FTP storage | FTP user password | -| TARGET_DB_HOST | Optional, required for database migration | Target database host | -| TARGET_DB_PORT | Optional, required for database migration | Target database port | -| TARGET_DB_NAME | Optional, required for database migration | Target database name | -| TARGET_DB_USERNAME | Optional, required for database migration | Target database username | -| TARGET_DB_PASSWORD | Optional, required for database migration | Target database password | -| TG_TOKEN | Optional, required for Telegram notification | Telegram token (`BOT-ID:BOT-TOKEN`) | -| TG_CHAT_ID | Optional, required for Telegram notification | Telegram Chat ID | -| TZ | Optional | Time Zone | +| Name | Requirement | Description | +|------------------------|---------------------------------------------------------------|-----------------------------------------------------------------| +| DB_PORT | Optional, default 5432 | Database port number | +| DB_HOST | Required | Database host | +| DB_NAME | Optional if it was provided from the -d flag | Database name | +| DB_USERNAME | Required | Database user name | +| DB_PASSWORD | Required | Database password | +| AWS_ACCESS_KEY | Optional, required for S3 storage | AWS S3 Access Key | +| AWS_SECRET_KEY | Optional, required for S3 storage | AWS S3 Secret Key | +| AWS_BUCKET_NAME | Optional, required for S3 storage | AWS S3 Bucket Name | +| AWS_BUCKET_NAME | Optional, required for S3 storage | AWS S3 Bucket Name | +| AWS_REGION | Optional, required for S3 storage | AWS Region | +| AWS_DISABLE_SSL | Optional, required for S3 storage | Disable SSL | +| AWS_FORCE_PATH_STYLE | Optional, required for S3 storage | Force path style | +| FILE_NAME | Optional if it was provided from the --file flag | Database file to restore (extensions: .sql, .sql.gz) | +| GPG_PASSPHRASE | Optional, required to encrypt and restore backup | GPG passphrase | +| GPG_PUBLIC_KEY | Optional, required to encrypt backup | GPG public key, used to encrypt backup (/config/public_key.asc) | +| BACKUP_CRON_EXPRESSION | Optional if it was provided from the `--cron-expression` flag | Backup cron expression for docker in scheduled mode | +| SSH_HOST | Optional, required for SSH storage | ssh remote hostname or ip | +| SSH_USER | Optional, required for SSH storage | ssh remote user | +| SSH_PASSWORD | Optional, required for SSH storage | ssh remote user's password | +| SSH_IDENTIFY_FILE | Optional, required for SSH storage | ssh remote user's private key | +| SSH_PORT | Optional, required for SSH storage | ssh remote server port | +| REMOTE_PATH | Optional, required for SSH or FTP storage | remote path (/home/toto/backup) | +| FTP_HOST | Optional, required for FTP storage | FTP host name | +| FTP_PORT | Optional, required for FTP storage | FTP server port number | +| FTP_USER | Optional, required for FTP storage | FTP user | +| FTP_PASSWORD | Optional, required for FTP storage | FTP user password | +| TARGET_DB_HOST | Optional, required for database migration | Target database host | +| TARGET_DB_PORT | Optional, required for database migration | Target database port | +| TARGET_DB_NAME | Optional, required for database migration | Target database name | +| TARGET_DB_USERNAME | Optional, required for database migration | Target database username | +| TARGET_DB_PASSWORD | Optional, required for database migration | Target database password | +| TG_TOKEN | Optional, required for Telegram notification | Telegram token (`BOT-ID:BOT-TOKEN`) | +| TG_CHAT_ID | Optional, required for Telegram notification | Telegram Chat ID | +| TZ | Optional | Time Zone | --- ## Run in Scheduled mode diff --git a/go.mod b/go.mod index d3d5c5b..876dd6d 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,26 @@ module github.com/jkaninda/pg-bkup go 1.21.0 require ( + github.com/ProtonMail/gopenpgp/v2 v2.7.5 + github.com/aws/aws-sdk-go v1.55.3 + github.com/bramvdbogaerde/go-scp v1.5.0 + github.com/jlaffaye/ftp v0.2.0 + github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.8.0 - github.com/spf13/pflag v1.0.5 + golang.org/x/crypto v0.28.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/aws/aws-sdk-go v1.55.3 // indirect - github.com/bramvdbogaerde/go-scp v1.5.0 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect + github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hpcloud/tail v1.0.0 // indirect - github.com/jlaffaye/ftp v0.2.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/robfig/cron/v3 v3.0.1 // indirect - golang.org/x/crypto v0.18.0 // indirect - golang.org/x/sys v0.22.0 // indirect - gopkg.in/fsnotify.v1 v1.4.7 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect ) diff --git a/go.sum b/go.sum index 359196b..959d82b 100644 --- a/go.sum +++ b/go.sum @@ -1,68 +1,101 @@ +github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs= +github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= +github.com/ProtonMail/gopenpgp/v2 v2.7.5 h1:STOY3vgES59gNgoOt2w0nyHBjKViB/qSg7NjbQWPJkA= +github.com/ProtonMail/gopenpgp/v2 v2.7.5/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E= github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM= github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= -github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/backup.go b/pkg/backup.go index ba01beb..6851564 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -20,23 +20,28 @@ import ( func StartBackup(cmd *cobra.Command) { intro() - dbConf = initDbConfig(cmd) //Initialize backup configs config := initBackupConfig(cmd) - - if config.cronExpression == "" { - BackupTask(dbConf, config) - } else { - if utils.IsValidCronExpression(config.cronExpression) { - scheduledMode(dbConf, config) + //Load backup configuration file + configFile, err := loadConfigFile() + if err != nil { + dbConf = initDbConfig(cmd) + if config.cronExpression == "" { + BackupTask(dbConf, config) } else { - utils.Fatal("Cron expression is not valid: %s", config.cronExpression) + if utils.IsValidCronExpression(config.cronExpression) { + scheduledMode(dbConf, config) + } else { + utils.Fatal("Cron expression is not valid: %s", config.cronExpression) + } } + } else { + startMultiBackup(config, configFile) } } -// Run in scheduled mode +// scheduledMode Runs backup in scheduled mode func scheduledMode(db *dbConfig, config *BackupConfig) { utils.Info("Running in Scheduled mode") utils.Info("Backup cron expression: %s", config.cronExpression) @@ -63,6 +68,17 @@ func scheduledMode(db *dbConfig, config *BackupConfig) { defer c.Stop() select {} } + +// multiBackupTask backup multi database +func multiBackupTask(databases []Database, bkConfig *BackupConfig) { + for _, db := range databases { + //Check if path is defined in config file + if db.Path != "" { + bkConfig.remotePath = db.Path + } + BackupTask(getDatabase(db), bkConfig) + } +} func BackupTask(db *dbConfig, config *BackupConfig) { utils.Info("Starting backup task...") //Generate file name @@ -85,9 +101,54 @@ func BackupTask(db *dbConfig, config *BackupConfig) { localBackup(db, config) } } -func intro() { - utils.Info("Starting PostgreSQL Backup...") - utils.Info("Copyright (c) 2024 Jonas Kaninda ") +func startMultiBackup(bkConfig *BackupConfig, configFile string) { + utils.Info("Starting multiple backup job...") + var conf = &Config{} + conf, err := readConf(configFile) + if err != nil { + utils.Fatal("Error reading config file: %s", err) + } + //Check if cronExpression is defined in config file + if conf.CronExpression != "" { + bkConfig.cronExpression = conf.CronExpression + } + // Check if cronExpression is defined + if bkConfig.cronExpression == "" { + multiBackupTask(conf.Databases, bkConfig) + } else { + // Check if cronExpression is valid + if utils.IsValidCronExpression(bkConfig.cronExpression) { + utils.Info("Running MultiBackup in Scheduled mode") + utils.Info("Backup cron expression: %s", bkConfig.cronExpression) + utils.Info("Storage type %s ", bkConfig.storage) + + //Test backup + utils.Info("Testing backup configurations...") + multiBackupTask(conf.Databases, bkConfig) + utils.Info("Testing backup configurations...done") + utils.Info("Creating multi backup job...") + // Create a new cron instance + c := cron.New() + + _, err := c.AddFunc(bkConfig.cronExpression, func() { + // Create a channel + multiBackupTask(conf.Databases, bkConfig) + }) + if err != nil { + return + } + // Start the cron scheduler + c.Start() + utils.Info("Creating multi backup job...done") + utils.Info("Backup job started") + defer c.Stop() + select {} + + } else { + utils.Fatal("Cron expression is not valid: %s", bkConfig.cronExpression) + } + } + } // BackupDatabase backup database @@ -119,7 +180,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool log.Fatal(err) } // save output - file, err := os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + file, err := os.Create(filepath.Join(tmpPath, backupFileName)) if err != nil { log.Fatal(err) } @@ -145,7 +206,7 @@ func BackupDatabase(db *dbConfig, backupFileName string, disableCompression bool gzipCmd := exec.Command("gzip") gzipCmd.Stdin = stdout // save output - gzipCmd.Stdout, err = os.Create(fmt.Sprintf("%s/%s", tmpPath, backupFileName)) + gzipCmd.Stdout, err = os.Create(filepath.Join(tmpPath, backupFileName)) gzipCmd.Start() if err != nil { log.Fatal(err) @@ -166,9 +227,10 @@ func localBackup(db *dbConfig, config *BackupConfig) { BackupDatabase(db, config.backupFileName, disableCompression) finalFileName := config.backupFileName if config.encryption { - encryptBackup(config.backupFileName, config.passphrase) + encryptBackup(config) finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, gpgExtension) } + utils.Info("Backup name is %s", finalFileName) moveToBackup(finalFileName, storagePath) //Send notification @@ -179,6 +241,7 @@ func localBackup(db *dbConfig, config *BackupConfig) { } //Delete temp deleteTemp() + utils.Info("Backup completed successfully") } func s3Backup(db *dbConfig, config *BackupConfig) { @@ -189,7 +252,7 @@ func s3Backup(db *dbConfig, config *BackupConfig) { BackupDatabase(db, config.backupFileName, disableCompression) finalFileName := config.backupFileName if config.encryption { - encryptBackup(config.backupFileName, config.passphrase) + encryptBackup(config) finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") } utils.Info("Uploading backup archive to remote storage S3 ... ") @@ -219,6 +282,8 @@ func s3Backup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") + } func sshBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to Remote server") @@ -226,7 +291,7 @@ func sshBackup(db *dbConfig, config *BackupConfig) { BackupDatabase(db, config.backupFileName, disableCompression) finalFileName := config.backupFileName if config.encryption { - encryptBackup(config.backupFileName, config.passphrase) + encryptBackup(config) finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") } utils.Info("Uploading backup archive to remote storage ... ") @@ -254,6 +319,8 @@ func sshBackup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") + } func ftpBackup(db *dbConfig, config *BackupConfig) { utils.Info("Backup database to the remote FTP server") @@ -261,7 +328,7 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { BackupDatabase(db, config.backupFileName, disableCompression) finalFileName := config.backupFileName if config.encryption { - encryptBackup(config.backupFileName, config.passphrase) + encryptBackup(config) finalFileName = fmt.Sprintf("%s.%s", config.backupFileName, "gpg") } utils.Info("Uploading backup archive to the remote FTP server ... ") @@ -289,12 +356,21 @@ func ftpBackup(db *dbConfig, config *BackupConfig) { utils.NotifySuccess(finalFileName) //Delete temp deleteTemp() + utils.Info("Backup completed successfully") } -func encryptBackup(backupFileName, gpqPassphrase string) { - err := Encrypt(filepath.Join(tmpPath, backupFileName), gpqPassphrase) - if err != nil { - utils.Fatal("Error during encrypting backup %v", err) +func encryptBackup(config *BackupConfig) { + if config.usingKey { + err := encryptWithGPGPublicKey(filepath.Join(tmpPath, config.backupFileName), config.publicKey) + if err != nil { + utils.Fatal("error during encrypting backup %v", err) + } + } else if config.passphrase != "" { + err := encryptWithGPG(filepath.Join(tmpPath, config.backupFileName), config.passphrase) + if err != nil { + utils.Fatal("error during encrypting backup %v", err) + } + } } diff --git a/pkg/config.go b/pkg/config.go index 2792f74..5015fce 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -14,7 +14,17 @@ import ( "strconv" ) +type Database struct { + Host string `yaml:"host"` + Port string `yaml:"port"` + Name string `yaml:"name"` + User string `yaml:"user"` + Password string `yaml:"password"` + Path string `yaml:"path"` +} type Config struct { + Databases []Database `yaml:"databases"` + CronExpression string `yaml:"cronExpression"` } type dbConfig struct { @@ -40,9 +50,11 @@ type BackupConfig struct { backupRetention int disableCompression bool prune bool - encryption bool remotePath string + encryption bool + usingKey bool passphrase string + publicKey string storage string cronExpression string } @@ -90,6 +102,16 @@ func initDbConfig(cmd *cobra.Command) *dbConfig { return &dConf } +func getDatabase(database Database) *dbConfig { + return &dbConfig{ + dbHost: database.Host, + dbPort: database.Port, + dbName: database.Name, + dbUserName: database.User, + dbPassword: database.Password, + } +} + // loadSSHConfig loads the SSH configuration from environment variables func loadSSHConfig() (*SSHConfig, error) { utils.GetEnvVariable("SSH_HOST", "SSH_HOST_NAME") @@ -163,10 +185,14 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig { _ = utils.GetEnv(cmd, "path", "AWS_S3_PATH") cronExpression := os.Getenv("BACKUP_CRON_EXPRESSION") - if passphrase != "" { + publicKeyFile, err := checkPubKeyFile(os.Getenv("GPG_PUBLIC_KEY")) + if err == nil { encryption = true + usingKey = true + } else if passphrase != "" { + encryption = true + usingKey = false } - //Initialize backup configs config := BackupConfig{} config.backupRetention = backupRetention @@ -176,17 +202,21 @@ func initBackupConfig(cmd *cobra.Command) *BackupConfig { config.encryption = encryption config.remotePath = remotePath config.passphrase = passphrase + config.publicKey = publicKeyFile + config.usingKey = usingKey config.cronExpression = cronExpression return &config } type RestoreConfig struct { - s3Path string - remotePath string - storage string - file string - bucket string - gpqPassphrase string + s3Path string + remotePath string + storage string + file string + bucket string + usingKey bool + passphrase string + privateKey string } func initRestoreConfig(cmd *cobra.Command) *RestoreConfig { @@ -199,7 +229,14 @@ func initRestoreConfig(cmd *cobra.Command) *RestoreConfig { storage = utils.GetEnv(cmd, "storage", "STORAGE") file = utils.GetEnv(cmd, "file", "FILE_NAME") bucket := utils.GetEnvVariable("AWS_S3_BUCKET_NAME", "BUCKET_NAME") - gpqPassphrase := os.Getenv("GPG_PASSPHRASE") + passphrase := os.Getenv("GPG_PASSPHRASE") + privateKeyFile, err := checkPrKeyFile(os.Getenv("GPG_PRIVATE_KEY")) + if err == nil { + usingKey = true + } else if passphrase != "" { + usingKey = false + } + //Initialize restore configs rConfig := RestoreConfig{} rConfig.s3Path = s3Path @@ -208,7 +245,9 @@ func initRestoreConfig(cmd *cobra.Command) *RestoreConfig { rConfig.bucket = bucket rConfig.file = file rConfig.storage = storage - rConfig.gpqPassphrase = gpqPassphrase + rConfig.passphrase = passphrase + rConfig.usingKey = usingKey + rConfig.privateKey = privateKeyFile return &rConfig } func initTargetDbConfig() *targetDbConfig { @@ -226,3 +265,10 @@ func initTargetDbConfig() *targetDbConfig { } return &tdbConfig } +func loadConfigFile() (string, error) { + backupConfigFile, err := checkConfigFile(os.Getenv("BACKUP_CONFIG_FILE")) + if err == nil { + return backupConfigFile, nil + } + return "", fmt.Errorf("backup config file not found") +} diff --git a/pkg/encrypt.go b/pkg/encrypt.go index 1211463..cf5a7ef 100644 --- a/pkg/encrypt.go +++ b/pkg/encrypt.go @@ -7,54 +7,173 @@ package pkg import ( + "errors" + "fmt" + "github.com/ProtonMail/gopenpgp/v2/crypto" "github.com/jkaninda/pg-bkup/utils" "os" - "os/exec" "strings" ) -func Decrypt(inputFile string, passphrase string) error { - utils.Info("Decrypting backup file: %s...", inputFile) - //Create gpg home dir - err := utils.MakeDirAll(gpgHome) +// decryptWithGPG decrypts backup file using a passphrase +func decryptWithGPG(inputFile string, passphrase string) error { + utils.Info("Decrypting backup using passphrase...") + // Read the encrypted file + encFileContent, err := os.ReadFile(inputFile) if err != nil { - return err + return errors.New(fmt.Sprintf("Error reading encrypted file: %s", err)) } - utils.SetEnv("GNUPGHOME", gpgHome) - cmd := exec.Command("gpg", "--batch", "--passphrase", passphrase, "--output", RemoveLastExtension(inputFile), "--decrypt", inputFile) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - err = cmd.Run() + // Define the passphrase used to encrypt the file + _passphrase := []byte(passphrase) + // Create a PGP message object from the encrypted file content + encryptedMessage := crypto.NewPGPMessage(encFileContent) + // Decrypt the message using the passphrase + plainMessage, err := crypto.DecryptMessageWithPassword(encryptedMessage, _passphrase) if err != nil { - return err + return errors.New(fmt.Sprintf("Error decrypting file: %s", err)) } + // Save the decrypted file (restore it) + err = os.WriteFile(RemoveLastExtension(inputFile), plainMessage.GetBinary(), 0644) + if err != nil { + return errors.New(fmt.Sprintf("Error saving decrypted file: %s", err)) + } + utils.Info("Decrypting backup using passphrase...done") utils.Info("Backup file decrypted successful!") return nil } -func Encrypt(inputFile string, passphrase string) error { - utils.Info("Encrypting backup...") - //Create gpg home dir - err := utils.MakeDirAll(gpgHome) +// encryptWithGPG encrypts backup using a passphrase +func encryptWithGPG(inputFile string, passphrase string) error { + utils.Info("Encrypting backup using passphrase...") + // Read the file to be encrypted + plainFileContent, err := os.ReadFile(inputFile) if err != nil { - return err + return errors.New(fmt.Sprintf("Error reading file: %s", err)) } - utils.SetEnv("GNUPGHOME", gpgHome) - cmd := exec.Command("gpg", "--batch", "--passphrase", passphrase, "--symmetric", "--cipher-algo", algorithm, inputFile) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + // Define the passphrase to encrypt the file + _passphrase := []byte(passphrase) - err = cmd.Run() + // Create a message object from the file content + message := crypto.NewPlainMessage(plainFileContent) + // Encrypt the message using the passphrase + encryptedMessage, err := crypto.EncryptMessageWithPassword(message, _passphrase) if err != nil { - return err + return errors.New(fmt.Sprintf("Error encrypting backup file: %s", err)) } - + // Save the encrypted .tar file + err = os.WriteFile(fmt.Sprintf("%s.%s", inputFile, gpgExtension), encryptedMessage.GetBinary(), 0644) + if err != nil { + return errors.New(fmt.Sprintf("Error saving encrypted filee: %s", err)) + } + utils.Info("Encrypting backup using passphrase...done") utils.Info("Backup file encrypted successful!") return nil } +// encryptWithGPGPublicKey encrypts backup using a public key +func encryptWithGPGPublicKey(inputFile string, publicKey string) error { + utils.Info("Encrypting backup using public key...") + // Read the public key + pubKeyBytes, err := os.ReadFile(publicKey) + if err != nil { + return errors.New(fmt.Sprintf("Error reading public key: %s", err)) + } + // Create a new keyring with the public key + publicKeyObj, err := crypto.NewKeyFromArmored(string(pubKeyBytes)) + if err != nil { + return errors.New(fmt.Sprintf("Error parsing public key: %s", err)) + } + + keyRing, err := crypto.NewKeyRing(publicKeyObj) + if err != nil { + + return errors.New(fmt.Sprintf("Error creating key ring: %v", err)) + } + + // Read the file to encryptWithGPGPublicKey + fileContent, err := os.ReadFile(inputFile) + if err != nil { + return errors.New(fmt.Sprintf("Error reading file: %v", err)) + } + + // encryptWithGPG the file + message := crypto.NewPlainMessage(fileContent) + encMessage, err := keyRing.Encrypt(message, nil) + if err != nil { + return errors.New(fmt.Sprintf("Error encrypting file: %v", err)) + } + + // Save the encrypted file + err = os.WriteFile(fmt.Sprintf("%s.%s", inputFile, gpgExtension), encMessage.GetBinary(), 0644) + if err != nil { + return errors.New(fmt.Sprintf("Error saving encrypted file: %v", err)) + } + utils.Info("Encrypting backup using public key...done") + utils.Info("Backup file encrypted successful!") + return nil + +} + +// decryptWithGPGPrivateKey decrypts backup file using a private key and passphrase. +// privateKey GPG private key +// passphrase GPG passphrase +func decryptWithGPGPrivateKey(inputFile, privateKey, passphrase string) error { + utils.Info("Encrypting backup using private key...") + + // Read the private key + priKeyBytes, err := os.ReadFile(privateKey) + if err != nil { + return errors.New(fmt.Sprintf("Error reading private key: %s", err)) + } + + // Read the password for the private key (if it’s password-protected) + password := []byte(passphrase) + + // Create a key object from the armored private key + privateKeyObj, err := crypto.NewKeyFromArmored(string(priKeyBytes)) + if err != nil { + return errors.New(fmt.Sprintf("Error parsing private key: %s", err)) + } + + // Unlock the private key with the password + if passphrase != "" { + // Unlock the private key with the password + _, err = privateKeyObj.Unlock(password) + if err != nil { + return errors.New(fmt.Sprintf("Error unlocking private key: %s", err)) + } + + } + + // Create a new keyring with the private key + keyRing, err := crypto.NewKeyRing(privateKeyObj) + if err != nil { + return errors.New(fmt.Sprintf("Error creating key ring: %v", err)) + } + + // Read the encrypted file + encFileContent, err := os.ReadFile(inputFile) + if err != nil { + return errors.New(fmt.Sprintf("Error reading encrypted file: %s", err)) + } + + // decryptWithGPG the file + encryptedMessage := crypto.NewPGPMessage(encFileContent) + message, err := keyRing.Decrypt(encryptedMessage, nil, 0) + if err != nil { + return errors.New(fmt.Sprintf("Error decrypting file: %s", err)) + } + + // Save the decrypted file + err = os.WriteFile(RemoveLastExtension(inputFile), message.GetBinary(), 0644) + if err != nil { + return errors.New(fmt.Sprintf("Error saving decrypted file: %s", err)) + } + utils.Info("Encrypting backup using public key...done") + fmt.Println("File successfully decrypted!") + return nil +} func RemoveLastExtension(filename string) string { if idx := strings.LastIndex(filename, "."); idx != -1 { return filename[:idx] diff --git a/pkg/helper.go b/pkg/helper.go index 1e5e00e..8586631 100644 --- a/pkg/helper.go +++ b/pkg/helper.go @@ -8,13 +8,21 @@ package pkg import ( "bytes" + "fmt" "github.com/jkaninda/pg-bkup/utils" + "gopkg.in/yaml.v3" "os" "os/exec" "path/filepath" "time" ) +func intro() { + utils.Info("Starting PostgreSQL Backup...") + utils.Info("Copyright (c) 2024 Jonas Kaninda ") +} + +// copyToTmp copy file to temporary directory func copyToTmp(sourcePath string, backupFileName string) { //Copy backup from storage to /tmp err := utils.CopyFile(filepath.Join(sourcePath, backupFileName), filepath.Join(tmpPath, backupFileName)) @@ -137,3 +145,91 @@ func testDatabaseConnection(db *dbConfig) { utils.Info("Successfully connected to %s database", db.dbName) } + +// checkPubKeyFile checks gpg public key +func checkPubKeyFile(pubKey string) (string, error) { + // Define possible key file names + keyFiles := []string{filepath.Join(gpgHome, "public_key.asc"), filepath.Join(gpgHome, "public_key.gpg"), pubKey} + + // Loop through key file names and check if they exist + for _, keyFile := range keyFiles { + if _, err := os.Stat(keyFile); err == nil { + // File exists + return keyFile, nil + } else if os.IsNotExist(err) { + // File does not exist, continue to the next one + continue + } else { + // An unexpected error occurred + return "", err + } + } + + // Return an error if neither file exists + return "", fmt.Errorf("no public key file found") +} + +// checkPrKeyFile checks private key +func checkPrKeyFile(prKey string) (string, error) { + // Define possible key file names + keyFiles := []string{filepath.Join(gpgHome, "private_key.asc"), filepath.Join(gpgHome, "private_key.gpg"), prKey} + + // Loop through key file names and check if they exist + for _, keyFile := range keyFiles { + if _, err := os.Stat(keyFile); err == nil { + // File exists + return keyFile, nil + } else if os.IsNotExist(err) { + // File does not exist, continue to the next one + continue + } else { + // An unexpected error occurred + return "", err + } + } + + // Return an error if neither file exists + return "", fmt.Errorf("no public key file found") +} + +// readConf reads config file and returns Config +func readConf(configFile string) (*Config, error) { + if utils.FileExists(configFile) { + buf, err := os.ReadFile(configFile) + if err != nil { + return nil, err + } + + c := &Config{} + err = yaml.Unmarshal(buf, c) + if err != nil { + return nil, fmt.Errorf("in file %q: %w", configFile, err) + } + + return c, err + } + return nil, fmt.Errorf("config file %q not found", configFile) +} + +// checkConfigFile checks config files and returns one config file +func checkConfigFile(filePath string) (string, error) { + // Define possible config file names + configFiles := []string{filepath.Join(workingDir, "config.yaml"), filepath.Join(workingDir, "config.yml"), filePath} + + // Loop through config file names and check if they exist + for _, configFile := range configFiles { + if _, err := os.Stat(configFile); err == nil { + // File exists + return configFile, nil + } else if os.IsNotExist(err) { + // File does not exist, continue to the next one + continue + } else { + // An unexpected error occurred + return "", err + } + } + + // Return an error if neither file exists + return "", fmt.Errorf("no config file found") +} diff --git a/pkg/migrate.go b/pkg/migrate.go index 17908f4..7536aad 100644 --- a/pkg/migrate.go +++ b/pkg/migrate.go @@ -30,11 +30,13 @@ func StartMigration(cmd *cobra.Command) { //Generate file name backupFileName := fmt.Sprintf("%s_%s.sql", dbConf.dbName, time.Now().Format("20060102_150405")) + conf := &RestoreConfig{} + conf.file = backupFileName //Backup source Database BackupDatabase(dbConf, backupFileName, true) //Restore source database into target database utils.Info("Restoring [%s] database into [%s] database...", dbConf.dbName, targetDbConf.targetDbName) - RestoreDatabase(&newDbConfig, backupFileName) + RestoreDatabase(&newDbConfig, conf) utils.Info("[%s] database has been restored into [%s] database", dbConf.dbName, targetDbConf.targetDbName) utils.Info("Database migration completed.") } diff --git a/pkg/restore.go b/pkg/restore.go index 4feb3b3..7f5811f 100644 --- a/pkg/restore.go +++ b/pkg/restore.go @@ -7,7 +7,6 @@ package pkg import ( - "fmt" "github.com/jkaninda/pg-bkup/utils" "github.com/spf13/cobra" "os" @@ -24,75 +23,77 @@ func StartRestore(cmd *cobra.Command) { case "local": utils.Info("Restore database from local") copyToTmp(storagePath, restoreConf.file) - RestoreDatabase(dbConf, restoreConf.file) + RestoreDatabase(dbConf, restoreConf) case "s3", "S3": - restoreFromS3(dbConf, restoreConf.file, restoreConf.bucket, restoreConf.s3Path) + restoreFromS3(dbConf, restoreConf) case "ssh", "SSH", "remote": - restoreFromRemote(dbConf, restoreConf.file, restoreConf.remotePath) + restoreFromRemote(dbConf, restoreConf) case "ftp", "FTP": - restoreFromFTP(dbConf, restoreConf.file, restoreConf.remotePath) + restoreFromFTP(dbConf, restoreConf) default: utils.Info("Restore database from local") copyToTmp(storagePath, restoreConf.file) - RestoreDatabase(dbConf, restoreConf.file) + RestoreDatabase(dbConf, restoreConf) } } -func restoreFromS3(db *dbConfig, file, bucket, s3Path string) { +func restoreFromS3(db *dbConfig, conf *RestoreConfig) { utils.Info("Restore database from s3") - err := DownloadFile(tmpPath, file, bucket, s3Path) + err := DownloadFile(tmpPath, conf.file, conf.bucket, conf.s3Path) if err != nil { - utils.Fatal("Error download file from s3 %s %v ", file, err) + utils.Fatal("Error download file from s3 %s %v ", conf.file, err) } - RestoreDatabase(db, file) + RestoreDatabase(db, conf) } -func restoreFromRemote(db *dbConfig, file, remotePath string) { +func restoreFromRemote(db *dbConfig, conf *RestoreConfig) { utils.Info("Restore database from remote server") - err := CopyFromRemote(file, remotePath) + err := CopyFromRemote(conf.file, conf.remotePath) if err != nil { - utils.Fatal("Error download file from remote server: %s %v", filepath.Join(remotePath, file), err) + utils.Fatal("Error download file from remote server: %s %v", filepath.Join(conf.remotePath, conf.file), err) } - RestoreDatabase(db, file) + RestoreDatabase(db, conf) } -func restoreFromFTP(db *dbConfig, file, remotePath string) { +func restoreFromFTP(db *dbConfig, conf *RestoreConfig) { utils.Info("Restore database from FTP server") - err := CopyFromFTP(file, remotePath) + err := CopyFromFTP(conf.file, conf.remotePath) if err != nil { - utils.Fatal("Error download file from FTP server: %s %v", filepath.Join(remotePath, file), err) + utils.Fatal("Error download file from FTP server: %s %v", filepath.Join(conf.remotePath, conf.file), err) } - RestoreDatabase(db, file) + RestoreDatabase(db, conf) } // RestoreDatabase restore database -func RestoreDatabase(db *dbConfig, file string) { - gpgPassphrase := os.Getenv("GPG_PASSPHRASE") - if file == "" { +func RestoreDatabase(db *dbConfig, conf *RestoreConfig) { + if conf.file == "" { utils.Fatal("Error, file required") } - extension := filepath.Ext(fmt.Sprintf("%s/%s", tmpPath, file)) + extension := filepath.Ext(filepath.Join(tmpPath, conf.file)) if extension == ".gpg" { - if gpgPassphrase == "" { - utils.Fatal("Error: GPG passphrase is required, your file seems to be a GPG file.\nYou need to provide GPG keys. GPG_PASSPHRASE environment variable is required.") - } else { - //Decrypt file - err := Decrypt(filepath.Join(tmpPath, file), gpgPassphrase) + if conf.usingKey { + utils.Warn("Backup decryption using a private key is not fully supported") + err := decryptWithGPGPrivateKey(filepath.Join(tmpPath, conf.file), conf.privateKey, conf.passphrase) if err != nil { - utils.Fatal("Error decrypting file %s %v", file, err) + utils.Fatal("error during decrypting backup %v", err) + } + } else { + if conf.passphrase == "" { + utils.Error("Error, passphrase or private key required") + utils.Fatal("Your file seems to be a GPG file.\nYou need to provide GPG keys. GPG_PASSPHRASE or GPG_PRIVATE_KEY environment variable is required.") + } else { + //decryptWithGPG file + err := decryptWithGPG(filepath.Join(tmpPath, conf.file), conf.passphrase) + if err != nil { + utils.Fatal("Error decrypting file %s %v", file, err) + } + //Update file name + conf.file = RemoveLastExtension(file) } - //Update file name - file = RemoveLastExtension(file) } } - err := utils.CheckEnvVars(dbHVars) - if err != nil { - utils.Error("Please make sure all required environment variables for database are set") - utils.Fatal("Error checking environment variables: %s", err) - } - - if utils.FileExists(fmt.Sprintf("%s/%s", tmpPath, file)) { + if utils.FileExists(filepath.Join(tmpPath, conf.file)) { err := os.Setenv("PGPASSWORD", db.dbPassword) if err != nil { @@ -101,10 +102,10 @@ func RestoreDatabase(db *dbConfig, file string) { testDatabaseConnection(db) utils.Info("Restoring database...") - extension := filepath.Ext(file) + extension := filepath.Ext(conf.file) // Restore from compressed file / .sql.gz if extension == ".gz" { - str := "zcat " + filepath.Join(tmpPath, file) + " | psql -h " + db.dbHost + " -p " + db.dbPort + " -U " + db.dbUserName + " -v -d " + db.dbName + str := "zcat " + filepath.Join(tmpPath, conf.file) + " | psql -h " + db.dbHost + " -p " + db.dbPort + " -U " + db.dbUserName + " -v -d " + db.dbName _, err := exec.Command("sh", "-c", str).Output() if err != nil { utils.Fatal("Error, in restoring the database %v", err) @@ -116,7 +117,7 @@ func RestoreDatabase(db *dbConfig, file string) { } else if extension == ".sql" { //Restore from sql file - str := "cat " + filepath.Join(tmpPath, file) + " | psql -h " + db.dbHost + " -p " + db.dbPort + " -U " + db.dbUserName + " -v -d " + db.dbName + str := "cat " + filepath.Join(tmpPath, conf.file) + " | psql -h " + db.dbHost + " -p " + db.dbPort + " -U " + db.dbUserName + " -v -d " + db.dbName _, err := exec.Command("sh", "-c", str).Output() if err != nil { utils.Fatal("Error in restoring the database %v", err) @@ -130,6 +131,6 @@ func RestoreDatabase(db *dbConfig, file string) { } } else { - utils.Fatal("File not found in %s", fmt.Sprintf("%s/%s", tmpPath, file)) + utils.Fatal("File not found in %s", filepath.Join(tmpPath, conf.file)) } } diff --git a/pkg/var.go b/pkg/var.go index 2ec9f88..1541f94 100644 --- a/pkg/var.go +++ b/pkg/var.go @@ -16,8 +16,10 @@ var ( file = "" storagePath = "/backup" + workingDir = "/config" disableCompression = false encryption = false + usingKey = false ) // dbHVars Required environment variables for database diff --git a/utils/constant.go b/utils/constant.go index 1bf1437..4cf7fe6 100644 --- a/utils/constant.go +++ b/utils/constant.go @@ -6,11 +6,11 @@ **/ package utils -const RestoreExample = "pg-bkup restore --dbname database --file db_20231219_022941.sql.gz\n" + +const RestoreExample = "restore --dbname database --file db_20231219_022941.sql.gz\n" + "restore --dbname database --storage s3 --path /custom-path --file db_20231219_022941.sql.gz" -const BackupExample = "pg-bkup backup --dbname database --disable-compression\n" + +const BackupExample = "backup --dbname database --disable-compression\n" + "backup --dbname database --storage s3 --path /custom-path --disable-compression" -const MainExample = "pg-bkup backup --dbname database --disable-compression\n" + +const MainExample = "backup --dbname database --disable-compression\n" + "backup --dbname database --storage s3 --path /custom-path\n" + "restore --dbname database --file db_20231219_022941.sql.gz" diff --git a/utils/utils.go b/utils/utils.go index 8ecb140..15392b8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -20,6 +20,7 @@ import ( "strconv" ) +// FileExists checks if the file does exist func FileExists(filename string) bool { info, err := os.Stat(filename) if os.IsNotExist(err) {