mirror of
https://github.com/jkaninda/go-storage.git
synced 2025-12-06 16:49:39 +01:00
Initial commit
This commit is contained in:
Jonas Kaninda
28
.github/workflows/go.yml
vendored
Normal file
28
.github/workflows/go.yml
vendored
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
# This workflow will build a golang project
|
||||||
|
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
|
||||||
|
|
||||||
|
name: Go
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ "main" ]
|
||||||
|
pull_request:
|
||||||
|
branches: [ "main" ]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v4
|
||||||
|
with:
|
||||||
|
go-version: '1.21.0'
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: go build -v ./...
|
||||||
|
|
||||||
|
- name: Test
|
||||||
|
run: go test -v ./...
|
||||||
12
.gitignore
vendored
Normal file
12
.gitignore
vendored
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
/.history
|
||||||
|
backup
|
||||||
|
data
|
||||||
|
compose.yaml
|
||||||
|
.env
|
||||||
|
test.md
|
||||||
|
.DS_Store
|
||||||
|
pg-bkup
|
||||||
|
/.idea
|
||||||
|
bin
|
||||||
|
Makefile
|
||||||
|
tests
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2024 Jonas Kaninda
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
22
go.mod
Normal file
22
go.mod
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
module github.com/jkaninda/go-storage
|
||||||
|
|
||||||
|
go 1.21.0
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/spf13/pflag v1.0.5
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/aws/aws-sdk-go v1.55.3 // indirect
|
||||||
|
github.com/bramvdbogaerde/go-scp v1.5.0 // indirect
|
||||||
|
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||||
|
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||||
|
github.com/hpcloud/tail v1.0.0 // indirect
|
||||||
|
github.com/jlaffaye/ftp v0.2.0 // indirect
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
|
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||||
|
golang.org/x/crypto v0.18.0 // indirect
|
||||||
|
golang.org/x/sys v0.22.0 // indirect
|
||||||
|
gopkg.in/fsnotify.v1 v1.4.7 // indirect
|
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
||||||
|
)
|
||||||
68
go.sum
Normal file
68
go.sum
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
github.com/aws/aws-sdk-go v1.55.3 h1:0B5hOX+mIx7I5XPOrjrHlKSDQV/+ypFZpIHOx5LOk3E=
|
||||||
|
github.com/aws/aws-sdk-go v1.55.3/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
||||||
|
github.com/bramvdbogaerde/go-scp v1.5.0 h1:a9BinAjTfQh273eh7vd3qUgmBC+bx+3TRDtkZWmIpzM=
|
||||||
|
github.com/bramvdbogaerde/go-scp v1.5.0/go.mod h1:on2aH5AxaFb2G0N5Vsdy6B0Ml7k9HuHSwfo1y0QzAbQ=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
|
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||||
|
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||||
|
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||||
|
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
|
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||||
|
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
|
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||||
|
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||||
|
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||||
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
|
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
||||||
|
github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||||
|
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
||||||
|
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||||
|
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||||
|
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||||
|
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||||
|
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||||
|
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
|
||||||
|
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
|
||||||
|
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||||
|
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
||||||
|
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
|
||||||
|
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
|
||||||
|
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
|
||||||
|
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
|
||||||
|
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
|
||||||
|
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
|
||||||
|
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
||||||
|
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||||
|
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||||
|
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||||
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||||
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
117
pkg/ftp/ftp.go
Normal file
117
pkg/ftp/ftp.go
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
package ftp
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/jkaninda/go-storage/pkg"
|
||||||
|
"github.com/jlaffaye/ftp"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ftpStorage struct {
|
||||||
|
*pkg.Backend
|
||||||
|
client *ftp.ServerConn
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config holds the SSH connection details
|
||||||
|
type Config struct {
|
||||||
|
Host string
|
||||||
|
User string
|
||||||
|
Password string
|
||||||
|
Port string
|
||||||
|
LocalPath string
|
||||||
|
RemotePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// createClient creates FTP Client
|
||||||
|
func createClient(conf Config) (*ftp.ServerConn, error) {
|
||||||
|
ftpClient, err := ftp.Dial(fmt.Sprintf("%s:%s", conf.Host, conf.Port), ftp.DialWithTimeout(5*time.Second))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to connect to FTP: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = ftpClient.Login(conf.User, conf.Password)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to log in to FTP: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ftpClient, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStorage creates new Storage
|
||||||
|
func NewStorage(conf Config) (pkg.Storage, error) {
|
||||||
|
client, err := createClient(conf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &ftpStorage{
|
||||||
|
client: client,
|
||||||
|
Backend: &pkg.Backend{
|
||||||
|
RemotePath: conf.RemotePath,
|
||||||
|
LocalPath: conf.LocalPath,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy copies file to the remote server
|
||||||
|
func (s ftpStorage) Copy(fileName string) error {
|
||||||
|
ftpClient := s.client
|
||||||
|
defer ftpClient.Quit()
|
||||||
|
|
||||||
|
filePath := filepath.Join(s.LocalPath, fileName)
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open file %s: %w", fileName, err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
remoteFilePath := filepath.Join(s.RemotePath, fileName)
|
||||||
|
err = ftpClient.Stor(remoteFilePath, file)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to upload file %s: %w", filepath.Join(s.LocalPath, fileName), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CopyFrom copies a file from the remote server to local storage
|
||||||
|
func (s ftpStorage) CopyFrom(fileName string) error {
|
||||||
|
ftpClient := s.client
|
||||||
|
|
||||||
|
defer ftpClient.Quit()
|
||||||
|
|
||||||
|
remoteFilePath := filepath.Join(s.RemotePath, fileName)
|
||||||
|
r, err := ftpClient.Retr(remoteFilePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to retrieve file %s: %w", fileName, err)
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
|
||||||
|
localFilePath := filepath.Join(s.LocalPath, fileName)
|
||||||
|
outFile, err := os.Create(localFilePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create local file %s: %w", fileName, err)
|
||||||
|
}
|
||||||
|
defer outFile.Close()
|
||||||
|
|
||||||
|
_, err = io.Copy(outFile, r)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to copy data to local file %s: %w", fileName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prune deletes old backup created more than specified days
|
||||||
|
func (s ftpStorage) Prune(retentionDays int) error {
|
||||||
|
fmt.Println("Deleting old backup from a remote server is not implemented yet")
|
||||||
|
return nil
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the storage name
|
||||||
|
func (s ftpStorage) Name() string {
|
||||||
|
return "ftp"
|
||||||
|
}
|
||||||
108
pkg/local/local.go
Normal file
108
pkg/local/local.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/jkaninda/go-storage/pkg"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type localStorage struct {
|
||||||
|
*pkg.Backend
|
||||||
|
}
|
||||||
|
type Config struct {
|
||||||
|
LocalPath string
|
||||||
|
RemotePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStorage creates new Storage
|
||||||
|
func NewStorage(conf Config) pkg.Storage {
|
||||||
|
return &localStorage{
|
||||||
|
Backend: &pkg.Backend{
|
||||||
|
LocalPath: conf.LocalPath,
|
||||||
|
RemotePath: conf.RemotePath,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy copies file to the local destination path
|
||||||
|
func (l localStorage) Copy(file string) error {
|
||||||
|
if _, err := os.Stat(filepath.Join(l.LocalPath, file)); os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err := copyFile(filepath.Join(l.LocalPath, file), filepath.Join(l.RemotePath, file))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CopyFrom copies file from a Path to local path
|
||||||
|
func (l localStorage) CopyFrom(file string) error {
|
||||||
|
if _, err := os.Stat(filepath.Join(l.RemotePath, file)); os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err := copyFile(filepath.Join(l.RemotePath, file), filepath.Join(l.LocalPath, file))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prune deletes old backup created more than specified days
|
||||||
|
func (l localStorage) Prune(retentionDays int) error {
|
||||||
|
currentTime := time.Now()
|
||||||
|
// Delete file
|
||||||
|
deleteFile := func(filePath string) error {
|
||||||
|
err := os.Remove(filePath)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Walk through the directory and delete files modified more than specified days ago
|
||||||
|
err := filepath.Walk(l.RemotePath, func(filePath string, fileInfo os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Check if it's a regular file and if it was modified more than specified days ago
|
||||||
|
if fileInfo.Mode().IsRegular() {
|
||||||
|
timeDiff := currentTime.Sub(fileInfo.ModTime())
|
||||||
|
if timeDiff.Hours() > 24*float64(retentionDays) {
|
||||||
|
err := deleteFile(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the storage name
|
||||||
|
func (l localStorage) Name() string {
|
||||||
|
return "local"
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyFile copies file
|
||||||
|
func copyFile(src, dst string) error {
|
||||||
|
in, err := os.Open(src)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer in.Close()
|
||||||
|
|
||||||
|
out, err := os.Create(dst)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = io.Copy(out, in)
|
||||||
|
if err != nil {
|
||||||
|
out.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return out.Close()
|
||||||
|
}
|
||||||
60
pkg/local/local_test.go
Normal file
60
pkg/local/local_test.go
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const content = "Lorem ipsum dolor sit amet. Eum eius voluptas sit vitae vitae aut sequi molestias hic accusamus consequatur"
|
||||||
|
const inputFile = "file.txt"
|
||||||
|
const localPath = "./tests/local"
|
||||||
|
const RemotePath = "./tests/remote"
|
||||||
|
|
||||||
|
func TestCopy(t *testing.T) {
|
||||||
|
|
||||||
|
err := os.MkdirAll(localPath, 0777)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
err = os.MkdirAll(RemotePath, 0777)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = createFile(filepath.Join(localPath, inputFile), content)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
l := NewStorage(Config{
|
||||||
|
LocalPath: "./tests/local",
|
||||||
|
RemotePath: "./tests/remote",
|
||||||
|
})
|
||||||
|
err = l.Copy(inputFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
fmt.Printf("File copied to %s\n", filepath.Join(RemotePath, inputFile))
|
||||||
|
}
|
||||||
|
func createFile(fileName, content string) ([]byte, error) {
|
||||||
|
// Create a file named hello.txt
|
||||||
|
file, err := os.Create(fileName)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error creating file:", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
// Write the message to the file
|
||||||
|
_, err = file.WriteString(content)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error writing to file:", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Successfully wrote to %s\n", fileName)
|
||||||
|
fileBytes, err := os.ReadFile(fileName)
|
||||||
|
return fileBytes, err
|
||||||
|
}
|
||||||
162
pkg/s3/s3.go
Normal file
162
pkg/s3/s3.go
Normal file
@@ -0,0 +1,162 @@
|
|||||||
|
package s3
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
|
"github.com/jkaninda/go-storage/pkg"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type s3Storage struct {
|
||||||
|
*pkg.Backend
|
||||||
|
client *session.Session
|
||||||
|
bucket string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config holds the AWS S3 config
|
||||||
|
type Config struct {
|
||||||
|
Endpoint string
|
||||||
|
Bucket string
|
||||||
|
AccessKey string
|
||||||
|
SecretKey string
|
||||||
|
Region string
|
||||||
|
DisableSsl bool
|
||||||
|
ForcePathStyle bool
|
||||||
|
LocalPath string
|
||||||
|
RemotePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateSession creates a new AWS session
|
||||||
|
func createSession(conf Config) (*session.Session, error) {
|
||||||
|
s3Config := &aws.Config{
|
||||||
|
Credentials: credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""),
|
||||||
|
Endpoint: aws.String(conf.Endpoint),
|
||||||
|
Region: aws.String(conf.Region),
|
||||||
|
DisableSSL: aws.Bool(conf.DisableSsl),
|
||||||
|
S3ForcePathStyle: aws.Bool(conf.ForcePathStyle),
|
||||||
|
}
|
||||||
|
|
||||||
|
return session.NewSession(s3Config)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStorage creates new Storage
|
||||||
|
func NewStorage(conf Config) (pkg.Storage, error) {
|
||||||
|
sess, err := createSession(conf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &s3Storage{
|
||||||
|
client: sess,
|
||||||
|
bucket: conf.Bucket,
|
||||||
|
Backend: &pkg.Backend{
|
||||||
|
RemotePath: conf.RemotePath,
|
||||||
|
LocalPath: conf.LocalPath,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy copies file to S3 storage
|
||||||
|
func (s s3Storage) Copy(fileName string) error {
|
||||||
|
svc := s3.New(s.client)
|
||||||
|
file, err := os.Open(filepath.Join(s.LocalPath, fileName))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
fileInfo, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
objectKey := filepath.Join(s.RemotePath, fileName)
|
||||||
|
buffer := make([]byte, fileInfo.Size())
|
||||||
|
file.Read(buffer)
|
||||||
|
fileBytes := bytes.NewReader(buffer)
|
||||||
|
fileType := http.DetectContentType(buffer)
|
||||||
|
|
||||||
|
_, err = svc.PutObject(&s3.PutObjectInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
Body: fileBytes,
|
||||||
|
ContentLength: aws.Int64(fileInfo.Size()),
|
||||||
|
ContentType: aws.String(fileType),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CopyFrom copies a file from S3 to local storage
|
||||||
|
func (s s3Storage) CopyFrom(fileName string) error {
|
||||||
|
file, err := os.Create(filepath.Join(s.LocalPath, fileName))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
objectKey := filepath.Join(s.RemotePath, fileName)
|
||||||
|
|
||||||
|
downloader := s3manager.NewDownloader(s.client)
|
||||||
|
_, err = downloader.Download(file,
|
||||||
|
&s3.GetObjectInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: aws.String(objectKey),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prune deletes old backup created more than specified days
|
||||||
|
func (s s3Storage) Prune(retentionDays int) error {
|
||||||
|
svc := s3.New(s.client)
|
||||||
|
|
||||||
|
// Get the current time
|
||||||
|
now := time.Now()
|
||||||
|
backupRetentionDays := now.AddDate(0, 0, -retentionDays)
|
||||||
|
|
||||||
|
// List objects in the bucket
|
||||||
|
listObjectsInput := &s3.ListObjectsV2Input{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Prefix: aws.String(s.RemotePath),
|
||||||
|
}
|
||||||
|
err := svc.ListObjectsV2Pages(listObjectsInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
|
||||||
|
for _, object := range page.Contents {
|
||||||
|
if object.LastModified.Before(backupRetentionDays) {
|
||||||
|
// Object is older than retention days, delete it
|
||||||
|
_, err := svc.DeleteObject(&s3.DeleteObjectInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: object.Key,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to delete object %s: %v", *object.Key, err)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("Deleted object %s", *object.Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return !lastPage
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to list objects: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the storage name
|
||||||
|
func (s s3Storage) Name() string {
|
||||||
|
return "s3"
|
||||||
|
}
|
||||||
119
pkg/ssh/ssh.go
Normal file
119
pkg/ssh/ssh.go
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
package ssh
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"github.com/bramvdbogaerde/go-scp"
|
||||||
|
"github.com/bramvdbogaerde/go-scp/auth"
|
||||||
|
"github.com/jkaninda/go-storage/pkg"
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
type sshStorage struct {
|
||||||
|
*pkg.Backend
|
||||||
|
client scp.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config holds the SSH connection details
|
||||||
|
type Config struct {
|
||||||
|
Host string
|
||||||
|
User string
|
||||||
|
Password string
|
||||||
|
Port string
|
||||||
|
IdentifyFile string
|
||||||
|
LocalPath string
|
||||||
|
RemotePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// createClient creates SSH Client
|
||||||
|
func createClient(conf Config) (scp.Client, error) {
|
||||||
|
if _, err := os.Stat(conf.IdentifyFile); os.IsNotExist(err) {
|
||||||
|
clientConfig, err := auth.PrivateKey(conf.User, conf.IdentifyFile, ssh.InsecureIgnoreHostKey())
|
||||||
|
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
|
||||||
|
} else {
|
||||||
|
if conf.Password == "" {
|
||||||
|
return scp.Client{}, errors.New("ssh password required")
|
||||||
|
}
|
||||||
|
clientConfig, err := auth.PasswordKey(conf.User, conf.Password, ssh.InsecureIgnoreHostKey())
|
||||||
|
return scp.NewClient(fmt.Sprintf("%s:%s", conf.Host, conf.Port), &clientConfig), err
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStorage creates new Storage
|
||||||
|
func NewStorage(conf Config) (pkg.Storage, error) {
|
||||||
|
client, err := createClient(conf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &sshStorage{
|
||||||
|
client: client,
|
||||||
|
Backend: &pkg.Backend{
|
||||||
|
RemotePath: conf.RemotePath,
|
||||||
|
LocalPath: conf.LocalPath,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy copies file to the remote server
|
||||||
|
func (s sshStorage) Copy(fileName string) error {
|
||||||
|
client := s.client
|
||||||
|
// Connect to the remote server
|
||||||
|
err := client.Connect()
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("couldn't establish a connection to the remote server")
|
||||||
|
}
|
||||||
|
// Open the local file
|
||||||
|
filePath := filepath.Join(s.LocalPath, fileName)
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open file %s: %w", filePath, err)
|
||||||
|
}
|
||||||
|
defer client.Close()
|
||||||
|
// Copy file to the remote server
|
||||||
|
err = client.CopyFromFile(context.Background(), *file, filepath.Join(s.RemotePath, fileName), "0655")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to copy file to remote server: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CopyFrom copies a file from the remote server to local storage
|
||||||
|
func (s sshStorage) CopyFrom(fileName string) error {
|
||||||
|
// Create a new SCP client
|
||||||
|
client := s.client
|
||||||
|
// Connect to the remote server
|
||||||
|
err := client.Connect()
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("couldn't establish a connection to the remote server")
|
||||||
|
}
|
||||||
|
// Close client connection after the file has been copied
|
||||||
|
defer client.Close()
|
||||||
|
file, err := os.OpenFile(filepath.Join(s.LocalPath, fileName), os.O_RDWR|os.O_CREATE, 0777)
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("couldn't open the output file")
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
err = client.CopyFromRemote(context.Background(), file, filepath.Join(s.RemotePath, fileName))
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prune deletes old backup created more than specified days
|
||||||
|
func (s sshStorage) Prune(retentionDays int) error {
|
||||||
|
fmt.Println("Deleting old backup from a remote server is not implemented yet")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the storage name
|
||||||
|
func (s sshStorage) Name() string {
|
||||||
|
return "ssh"
|
||||||
|
}
|
||||||
14
pkg/storage.go
Normal file
14
pkg/storage.go
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
package pkg
|
||||||
|
|
||||||
|
type Storage interface {
|
||||||
|
Copy(fileName string) error
|
||||||
|
CopyFrom(fileName string) error
|
||||||
|
Prune(retentionDays int) error
|
||||||
|
Name() string
|
||||||
|
}
|
||||||
|
type Backend struct {
|
||||||
|
//Local Path
|
||||||
|
LocalPath string
|
||||||
|
//Remote path or Destination path
|
||||||
|
RemotePath string
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user